[llvm] [X86][X86CompressEVEX] Do not compress when there is a compressed disp8 (PR #221923)
Phoebe Wang via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 01:50:03 PDT 2026
https://github.com/phoebewang created https://github.com/llvm/llvm-project/pull/221923
Fixes: #206963
Assisted-by: Claude Opus 4.8
>From 9e514eff034ac35776a662e6bf88424f73d9eae8 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang at intel.com>
Date: Tue, 8 Sep 2026 01:45:39 -0700
Subject: [PATCH] [X86][X86CompressEVEX] Do not compress when there is a
compressed disp8
Fixes: #206963
Assisted-by: Claude Opus 4.8
---
llvm/lib/Target/X86/X86CompressEVEX.cpp | 39 +
llvm/test/CodeGen/X86/avx512fp16-frem.ll | 20 +-
llvm/test/CodeGen/X86/bfloat.ll | 2 +-
llvm/test/CodeGen/X86/ucmp.ll | 54 +-
llvm/test/CodeGen/X86/vector-fshl-128.ll | 12 +-
.../vector-interleaved-load-i16-stride-3.ll | 56 +-
.../vector-interleaved-load-i16-stride-4.ll | 24 +-
.../vector-interleaved-load-i16-stride-5.ll | 272 ++--
.../vector-interleaved-load-i16-stride-6.ll | 476 +++---
.../vector-interleaved-load-i16-stride-7.ll | 346 ++---
.../vector-interleaved-load-i16-stride-8.ll | 1136 +++++++--------
.../vector-interleaved-load-i32-stride-5.ll | 32 +-
.../vector-interleaved-load-i32-stride-6.ll | 32 +-
.../vector-interleaved-load-i64-stride-5.ll | 24 +-
.../vector-interleaved-load-i64-stride-6.ll | 40 +-
.../vector-interleaved-load-i64-stride-7.ll | 1120 +++++++--------
.../vector-interleaved-load-i64-stride-8.ll | 1280 ++++++++---------
.../vector-interleaved-load-i8-stride-3.ll | 64 +-
.../vector-interleaved-load-i8-stride-4.ll | 16 +-
.../vector-interleaved-load-i8-stride-5.ll | 208 +--
.../vector-interleaved-load-i8-stride-6.ll | 104 +-
.../vector-interleaved-load-i8-stride-7.ll | 190 +--
.../vector-interleaved-load-i8-stride-8.ll | 212 +--
.../vector-interleaved-store-i16-stride-2.ll | 56 +-
.../vector-interleaved-store-i16-stride-5.ll | 16 +-
.../vector-interleaved-store-i16-stride-6.ll | 84 +-
.../vector-interleaved-store-i16-stride-7.ll | 276 ++--
.../vector-interleaved-store-i32-stride-5.ll | 16 +-
.../vector-interleaved-store-i32-stride-6.ll | 32 +-
.../vector-interleaved-store-i32-stride-7.ll | 16 +-
.../vector-interleaved-store-i64-stride-5.ll | 16 +-
.../vector-interleaved-store-i64-stride-6.ll | 248 ++--
.../vector-interleaved-store-i64-stride-7.ll | 596 ++++----
.../vector-interleaved-store-i64-stride-8.ll | 688 ++++-----
.../vector-interleaved-store-i8-stride-3.ll | 16 +-
.../vector-interleaved-store-i8-stride-4.ll | 32 +-
.../vector-interleaved-store-i8-stride-5.ll | 16 +-
.../vector-interleaved-store-i8-stride-7.ll | 20 +-
.../vector-interleaved-store-i8-stride-8.ll | 52 +-
.../CodeGen/X86/vector-replicaton-i1-mask.ll | 20 +-
.../test/CodeGen/X86/vector-shuffle-avx512.ll | 20 +-
llvm/test/CodeGen/X86/vector-shuffle-v192.ll | 44 +-
42 files changed, 4031 insertions(+), 3992 deletions(-)
diff --git a/llvm/lib/Target/X86/X86CompressEVEX.cpp b/llvm/lib/Target/X86/X86CompressEVEX.cpp
index 18d655288f400..fce57ac38f747 100644
--- a/llvm/lib/Target/X86/X86CompressEVEX.cpp
+++ b/llvm/lib/Target/X86/X86CompressEVEX.cpp
@@ -120,6 +120,41 @@ static bool usesExtendedRegister(const MachineInstr &MI) {
return false;
}
+// Return true if the EVEX form of \p MI can encode its memory displacement as
+// a compressed disp8*N (1 byte) while the VEX/legacy twin would be forced to
+// spend a full disp32 (4 bytes). In that window the EVEX encoding is strictly
+// shorter overall, despite its 1-2 byte larger prefix, so compressing it to
+// VEX would grow code size. Mirrors isDispOrCDisp8 in X86MCCodeEmitter.cpp.
+static bool hasShorterEVEXViaCDisp8(const MachineInstr &MI) {
+ uint64_t TSFlags = MI.getDesc().TSFlags;
+ unsigned CD8_Scale =
+ (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
+ CD8_Scale = CD8_Scale ? 1U << (CD8_Scale - 1) : 0U;
+ // Without a CD8 scale > 1 there is no displacement advantage over VEX.
+ if (CD8_Scale <= 1)
+ return false;
+
+ int MemOpIdx = X86::getFirstAddrOperandIdx(MI);
+ if (MemOpIdx < 0)
+ return false;
+
+ const MachineOperand &Disp = MI.getOperand(MemOpIdx + X86::AddrDisp);
+ // Only a constant displacement can be range-checked here; symbolic ones
+ // (globals, constant pool, jump tables, ...) are resolved later.
+ if (!Disp.isImm())
+ return false;
+
+ int64_t Val = Disp.getImm();
+ // VEX can already use a disp8 in this range, so EVEX offers no saving.
+ if (isInt<8>(Val))
+ return false;
+ // EVEX can use disp8*N only when the value is a multiple of N and the scaled
+ // value fits in a signed byte.
+ if (Val % static_cast<int64_t>(CD8_Scale) != 0)
+ return false;
+ return isInt<8>(Val / static_cast<int64_t>(CD8_Scale));
+}
+
// Do any custom cleanup needed to finalize the conversion.
static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {
(void)NewOpc;
@@ -540,6 +575,10 @@ static bool CompressEVEXImpl(MachineInstr &MI, MachineBasicBlock &MBB,
if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))
return false;
+ // Keep the EVEX encoding when there's 1-byte compressed disp8*N.
+ if (hasShorterEVEXViaCDisp8(MI))
+ return 0;
+
// Specialized mask-producing folds to MOVMSK/VBLENDV first.
if (tryCompressMaskProducer(MI, MBB, ST, ToErase))
return true;
diff --git a/llvm/test/CodeGen/X86/avx512fp16-frem.ll b/llvm/test/CodeGen/X86/avx512fp16-frem.ll
index 2164c2460f6d7..edd3273ac381a 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-frem.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-frem.ll
@@ -310,7 +310,7 @@ define <16 x half> @frem_vec16(<16 x half> %x, <16 x half> %y) nounwind {
; CHECK-NEXT: vcvtss2sh %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-NEXT: vpsrldq $10, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = mem[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT: vcvtsh2ss %xmm0, %xmm0, %xmm0
@@ -332,7 +332,7 @@ define <16 x half> @frem_vec16(<16 x half> %x, <16 x half> %y) nounwind {
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
; CHECK-NEXT: vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-NEXT: vpsrldq $14, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = mem[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT: vcvtsh2ss %xmm0, %xmm0, %xmm0
@@ -374,8 +374,8 @@ define <16 x half> @frem_vec16(<16 x half> %x, <16 x half> %y) nounwind {
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
; CHECK-NEXT: vpunpckldq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
-; CHECK-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT: vinserti32x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; CHECK-NEXT: vpsrlq $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: vcvtsh2ss %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpsrlq $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
@@ -556,7 +556,7 @@ define <32 x half> @frem_vec32(<32 x half> %x, <32 x half> %y) nounwind {
; CHECK-NEXT: vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
; CHECK-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
-; CHECK-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; CHECK-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0
; CHECK-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -1008,7 +1008,7 @@ define <16 x half> @frem_strict_vec16(<16 x half> %x, <16 x half> %y) nounwind #
; CHECK-NEXT: vcvtss2sh %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-NEXT: vpsrldq $10, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = mem[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT: vcvtsh2ss %xmm0, %xmm0, %xmm1
@@ -1030,7 +1030,7 @@ define <16 x half> @frem_strict_vec16(<16 x half> %x, <16 x half> %y) nounwind #
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
; CHECK-NEXT: vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-NEXT: vpsrldq $14, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = mem[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT: vcvtsh2ss %xmm0, %xmm0, %xmm1
@@ -1072,8 +1072,8 @@ define <16 x half> @frem_strict_vec16(<16 x half> %x, <16 x half> %y) nounwind #
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
; CHECK-NEXT: vpunpckldq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
-; CHECK-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
-; CHECK-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT: vinserti32x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; CHECK-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; CHECK-NEXT: vpsrlq $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: vcvtsh2ss %xmm0, %xmm0, %xmm1
; CHECK-NEXT: vpsrlq $48, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
@@ -1254,7 +1254,7 @@ define <32 x half> @frem_strict_vec32(<32 x half> %x, <32 x half> %y) nounwind #
; CHECK-NEXT: vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
; CHECK-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
; CHECK-NEXT: vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
-; CHECK-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; CHECK-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm0
; CHECK-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index be217d52fbb9f..4a234a5e80608 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -1523,7 +1523,7 @@ define <8 x bfloat> @fptrunc_v8f64(<8 x double> %a) nounwind {
; X86-NEXT: vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
; X86-NEXT: vmovhps %xmm0, (%esp)
; X86-NEXT: calll __truncdfbf2
-; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT: vmovdqa64 {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/X86/ucmp.ll b/llvm/test/CodeGen/X86/ucmp.ll
index 65836384894ef..cba58168b96bf 100644
--- a/llvm/test/CodeGen/X86/ucmp.ll
+++ b/llvm/test/CodeGen/X86/ucmp.ll
@@ -2573,18 +2573,18 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: andl $127, %r15d
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX512-NEXT: andl $127, %ebx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: vpextrq $1, %xmm2, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX512-NEXT: cmpq %r8, %r11
@@ -2593,7 +2593,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: setb %r9b
; AVX512-NEXT: cmpq %r11, %r8
; AVX512-NEXT: vpextrq $1, %xmm2, %r8
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX512-NEXT: sbbq %rdi, %rsi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
@@ -2603,11 +2603,11 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: movq %r8, %rdi
; AVX512-NEXT: sbbq %rdx, %rdi
; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: setb %r9b
; AVX512-NEXT: cmpq %rsi, %r11
; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r14
; AVX512-NEXT: sbbq %r8, %rdx
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
@@ -2619,7 +2619,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: setb %r9b
; AVX512-NEXT: cmpq %rdx, %r14
; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
; AVX512-NEXT: sbbq %rsi, %rdi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
@@ -2629,11 +2629,11 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: movq %rdx, %rdi
; AVX512-NEXT: sbbq %rcx, %rdi
; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: setb %r9b
; AVX512-NEXT: cmpq %rsi, %r8
; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r8
; AVX512-NEXT: sbbq %rdx, %rcx
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
@@ -2645,7 +2645,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: setb %r9b
; AVX512-NEXT: cmpq %rcx, %r8
; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX512-NEXT: sbbq %rsi, %rdi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
@@ -2655,7 +2655,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: movq %rcx, %rdi
; AVX512-NEXT: sbbq %rax, %rdi
; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: setb %r8b
; AVX512-NEXT: cmpq %rsi, %rdx
; AVX512-NEXT: vpextrq $1, %xmm2, %rdx
@@ -2670,7 +2670,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: setb %r14b
; AVX512-NEXT: cmpq %rax, %rsi
; AVX512-NEXT: vpextrq $1, %xmm1, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
; AVX512-NEXT: sbbq %rdx, %rdi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
@@ -2679,11 +2679,11 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: cmpq %rdx, %rsi
; AVX512-NEXT: movq %rcx, %rdi
; AVX512-NEXT: sbbq %rax, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: setb %r13b
; AVX512-NEXT: cmpq %rsi, %rdx
; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
; AVX512-NEXT: sbbq %rcx, %rax
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
@@ -2696,7 +2696,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: cmpq %rcx, %rax
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: sbbq %rsi, %rdx
; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
@@ -2709,7 +2709,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: cmpq %rsi, %rdx
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: sbbq %rcx, %rax
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
@@ -2723,7 +2723,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
; AVX512-NEXT: sbbq %rax, %rdx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX512-NEXT: sbbb $0, %r10b
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
@@ -2734,7 +2734,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: setb %r9b
; AVX512-NEXT: cmpq %rsi, %rdx
; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm2
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm2
; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
; AVX512-NEXT: sbbq %rax, %rcx
; AVX512-NEXT: vpextrq $1, %xmm2, %rax
@@ -2746,7 +2746,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: sbbq %rdx, %rdi
; AVX512-NEXT: setb %r8b
; AVX512-NEXT: cmpq %rcx, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: sbbq %rax, %rdx
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; AVX512-NEXT: sbbb $0, %r8b
@@ -2755,7 +2755,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: vpextrq $1, %xmm1, %rdx
; AVX512-NEXT: movq %rdx, %rax
; AVX512-NEXT: sbbq %rbx, %rax
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: setb %al
; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
; AVX512-NEXT: cmpq %rcx, %rdi
@@ -2768,9 +2768,9 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: sbbq %r15, %rdx
; AVX512-NEXT: setb %bl
; AVX512-NEXT: cmpq %rcx, %rdi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm1
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm1
; AVX512-NEXT: sbbq %rsi, %r15
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX512-NEXT: sbbb $0, %bl
@@ -2779,7 +2779,7 @@ define <17 x i2> @ucmp_uncommon_vectors(<17 x i71> %x, <17 x i71> %y) nounwind {
; AVX512-NEXT: movq %rcx, %rsi
; AVX512-NEXT: sbbq %r12, %rsi
; AVX512-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512-NEXT: vpand {{[0-9]+}}(%rsp), %xmm0, %xmm0
+; AVX512-NEXT: vpandq {{[0-9]+}}(%rsp), %xmm0, %xmm0
; AVX512-NEXT: setb %r15b
; AVX512-NEXT: cmpq %rdx, %rdi
; AVX512-NEXT: vpextrq $1, %xmm0, %rdx
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index 7b4099d68f9f9..e69cc0f1c68ce 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -1554,9 +1554,9 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
; AVX512VL-NEXT: .p2align 4
; AVX512VL-NEXT: .LBB8_1: # %loop
; AVX512VL-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512VL-NEXT: vmovdqu 1024(%rdi,%rax), %xmm1
+; AVX512VL-NEXT: vmovdqu64 1024(%rdi,%rax), %xmm1
; AVX512VL-NEXT: vprolvd %xmm0, %xmm1, %xmm1
-; AVX512VL-NEXT: vmovdqu %xmm1, 1024(%rdi,%rax)
+; AVX512VL-NEXT: vmovdqu64 %xmm1, 1024(%rdi,%rax)
; AVX512VL-NEXT: addq $16, %rax
; AVX512VL-NEXT: jne .LBB8_1
; AVX512VL-NEXT: # %bb.2: # %end
@@ -1603,9 +1603,9 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
; AVX512VLBW-NEXT: .p2align 4
; AVX512VLBW-NEXT: .LBB8_1: # %loop
; AVX512VLBW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512VLBW-NEXT: vmovdqu 1024(%rdi,%rax), %xmm1
+; AVX512VLBW-NEXT: vmovdqu64 1024(%rdi,%rax), %xmm1
; AVX512VLBW-NEXT: vprolvd %xmm0, %xmm1, %xmm1
-; AVX512VLBW-NEXT: vmovdqu %xmm1, 1024(%rdi,%rax)
+; AVX512VLBW-NEXT: vmovdqu64 %xmm1, 1024(%rdi,%rax)
; AVX512VLBW-NEXT: addq $16, %rax
; AVX512VLBW-NEXT: jne .LBB8_1
; AVX512VLBW-NEXT: # %bb.2: # %end
@@ -1618,9 +1618,9 @@ define void @sink_splatvar(ptr %p, i32 %shift_amt) {
; AVX512VLVBMI2-NEXT: .p2align 4
; AVX512VLVBMI2-NEXT: .LBB8_1: # %loop
; AVX512VLVBMI2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512VLVBMI2-NEXT: vmovdqu 1024(%rdi,%rax), %xmm1
+; AVX512VLVBMI2-NEXT: vmovdqu64 1024(%rdi,%rax), %xmm1
; AVX512VLVBMI2-NEXT: vprolvd %xmm0, %xmm1, %xmm1
-; AVX512VLVBMI2-NEXT: vmovdqu %xmm1, 1024(%rdi,%rax)
+; AVX512VLVBMI2-NEXT: vmovdqu64 %xmm1, 1024(%rdi,%rax)
; AVX512VLVBMI2-NEXT: addq $16, %rax
; AVX512VLVBMI2-NEXT: jne .LBB8_1
; AVX512VLVBMI2-NEXT: # %bb.2: # %end
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
index fbdc3904db4f8..bcc16ec0f76d3 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
@@ -1800,8 +1800,8 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-LABEL: load_i16_stride3_vf32:
; AVX512: # %bb.0:
; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm5
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm5
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512-NEXT: vmovdqa %ymm0, %ymm1
; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm5 ^ (ymm1 & (ymm6 ^ ymm5))
; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
@@ -1878,8 +1878,8 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-LABEL: load_i16_stride3_vf32:
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm5
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm1
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm5 ^ (ymm1 & (ymm6 ^ ymm5))
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
@@ -1956,8 +1956,8 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-LABEL: load_i16_stride3_vf32:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm5
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm1
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm5 ^ (ymm1 & (ymm6 ^ ymm5))
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
@@ -2034,8 +2034,8 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-LABEL: load_i16_stride3_vf32:
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm1
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm5 ^ (ymm1 & (ymm6 ^ ymm5))
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
@@ -3433,8 +3433,8 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
; AVX512-NEXT: vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
; AVX512-NEXT: vpshufb %ymm7, %ymm2, %ymm5
-; AVX512-NEXT: vmovdqa 272(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa 256(%rdi), %xmm3
+; AVX512-NEXT: vmovdqa64 272(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 256(%rdi), %xmm3
; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]
; AVX512-NEXT: vmovdqa64 %xmm1, %xmm19
; AVX512-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
@@ -3451,15 +3451,15 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
; AVX512-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
; AVX512-NEXT: vpshufb %ymm10, %ymm8, %ymm11
-; AVX512-NEXT: vmovdqa 304(%rdi), %xmm8
-; AVX512-NEXT: vmovdqa 288(%rdi), %xmm4
+; AVX512-NEXT: vmovdqa64 304(%rdi), %xmm8
+; AVX512-NEXT: vmovdqa64 288(%rdi), %xmm4
; AVX512-NEXT: vpblendw {{.*#+}} xmm12 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
; AVX512-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm12
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
; AVX512-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm16
; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm23
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm11
; AVX512-NEXT: vmovdqa %ymm0, %ymm5
; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm23 ^ (ymm5 & (ymm11 ^ ymm23))
; AVX512-NEXT: vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
@@ -3597,8 +3597,8 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm5
-; AVX512-FCP-NEXT: vmovdqa 272(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %xmm3
+; AVX512-FCP-NEXT: vmovdqa64 272(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %xmm3
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]
; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm19
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
@@ -3615,15 +3615,15 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm11
-; AVX512-FCP-NEXT: vmovdqa 304(%rdi), %xmm8
-; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %xmm4
+; AVX512-FCP-NEXT: vmovdqa64 304(%rdi), %xmm8
+; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %xmm4
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm12
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm16
; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm23
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm11
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm5
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm23 ^ (ymm5 & (ymm11 ^ ymm23))
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
@@ -3760,8 +3760,8 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
; AVX512DQ-NEXT: vpshufb %ymm7, %ymm2, %ymm5
-; AVX512DQ-NEXT: vmovdqa 272(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %xmm3
+; AVX512DQ-NEXT: vmovdqa64 272(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %xmm3
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]
; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm19
; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
@@ -3778,15 +3778,15 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
; AVX512DQ-NEXT: vpshufb %ymm10, %ymm8, %ymm11
-; AVX512DQ-NEXT: vmovdqa 304(%rdi), %xmm8
-; AVX512DQ-NEXT: vmovdqa 288(%rdi), %xmm4
+; AVX512DQ-NEXT: vmovdqa64 304(%rdi), %xmm8
+; AVX512DQ-NEXT: vmovdqa64 288(%rdi), %xmm4
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm12 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm12
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm16
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm23
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm11
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm11
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm5
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm23 ^ (ymm5 & (ymm11 ^ ymm23))
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
@@ -3924,8 +3924,8 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 272(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 272(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %xmm3
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm19
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
@@ -3942,15 +3942,15 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa 304(%rdi), %xmm8
-; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 304(%rdi), %xmm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %xmm4
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm12
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm16
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm23
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm5
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm23 ^ (ymm5 & (ymm11 ^ ymm23))
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
index ce9f73a79b7dd..0c295016bada2 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
@@ -2670,12 +2670,12 @@ define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512: # %bb.0:
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm2
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm3
; AVX512-NEXT: vpmovqw %ymm3, %xmm4
; AVX512-NEXT: vmovdqa64 240(%rdi), %xmm16
; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm16[0,2,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,0,2,4,5,6,7]
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm9
+; AVX512-NEXT: vmovdqa64 224(%rdi), %xmm9
; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm9[0,2,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm8[0,1,0,2,4,5,6,7]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
@@ -2772,11 +2772,11 @@ define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]
; AVX512-FCP-NEXT: # ymm2 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,2,2,3,0,2,4,6]
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm5
; AVX512-FCP-NEXT: vpermd %ymm5, %ymm4, %ymm6
; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm6, %ymm7
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm8
; AVX512-FCP-NEXT: vpermd %ymm8, %ymm4, %ymm9
; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm10
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,2,2,3,4,6,12,14]
@@ -2849,12 +2849,12 @@ define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm2
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm3
; AVX512DQ-NEXT: vpmovqw %ymm3, %xmm4
; AVX512DQ-NEXT: vmovdqa64 240(%rdi), %xmm16
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm16[0,2,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm9
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %xmm9
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm9[0,2,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm8[0,1,0,2,4,5,6,7]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
@@ -2951,11 +2951,11 @@ define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,2,2,3,0,2,4,6]
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpermd %ymm5, %ymm4, %ymm6
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm6, %ymm7
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vpermd %ymm8, %ymm4, %ymm9
; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm10
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,2,2,3,4,6,12,14]
@@ -5593,7 +5593,7 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 448(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 448(%rdi), %ymm8
; AVX512-NEXT: vpmovqw %ymm8, %xmm9
; AVX512-NEXT: vmovdqa64 496(%rdi), %xmm17
; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm17[0,2,2,3]
@@ -5605,7 +5605,7 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %ymm12, %ymm2, %ymm9
; AVX512-NEXT: vpmovqw %zmm24, %xmm12
; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm12[0,1,2,3],ymm9[4,5,6,7]
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm9
+; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm9
; AVX512-NEXT: vpmovqw %ymm9, %xmm3
; AVX512-NEXT: vmovdqa64 368(%rdi), %xmm19
; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[0,2,2,3]
@@ -5934,7 +5934,7 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %ymm8
; AVX512DQ-NEXT: vpmovqw %ymm8, %xmm9
; AVX512DQ-NEXT: vmovdqa64 496(%rdi), %xmm17
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm17[0,2,2,3]
@@ -5946,7 +5946,7 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %ymm12, %ymm2, %ymm9
; AVX512DQ-NEXT: vpmovqw %zmm24, %xmm12
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm12[0,1,2,3],ymm9[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm9
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm9
; AVX512DQ-NEXT: vpmovqw %ymm9, %xmm3
; AVX512DQ-NEXT: vmovdqa64 368(%rdi), %xmm19
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[0,2,2,3]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
index 5d7627d42d67a..cb8ac46da4115 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
@@ -1929,8 +1929,8 @@ define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
; AVX512-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u]
; AVX512-NEXT: vpor %ymm4, %ymm5, %ymm5
-; AVX512-NEXT: vmovdqa 144(%rdi), %xmm6
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm4
+; AVX512-NEXT: vmovdqa64 144(%rdi), %xmm6
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm4
; AVX512-NEXT: vpblendd {{.*#+}} xmm7 = xmm4[0],xmm6[1],xmm4[2,3]
; AVX512-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
@@ -2018,7 +2018,7 @@ define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %ymm4, %ymm5, %ymm5
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,0,3,5,0]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512-FCP-NEXT: vpermd %ymm4, %ymm6, %ymm6
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm7 = [0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27]
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm6, %ymm6
@@ -2106,8 +2106,8 @@ define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u]
; AVX512DQ-NEXT: vpor %ymm4, %ymm5, %ymm5
-; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm6
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm4
+; AVX512DQ-NEXT: vmovdqa64 144(%rdi), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm7 = xmm4[0],xmm6[1],xmm4[2,3]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
@@ -2195,7 +2195,7 @@ define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %ymm4, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [0,0,0,0,0,3,5,0]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm6, %ymm6
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm7 = [0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27,0,0,18,19,20,21,26,27]
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm6, %ymm6
@@ -2275,7 +2275,7 @@ define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,17,22,27]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm3
; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
@@ -2308,7 +2308,7 @@ define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,17,22,27]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm3
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
@@ -2341,7 +2341,7 @@ define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,17,22,27]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512DQ-BW-NEXT: vpermi2w %ymm4, %ymm2, %ymm3
; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
@@ -2374,7 +2374,7 @@ define void @load_i16_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,5,10,15,20,25,30,35,40,45,50,55,60,0,0,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,8,9,10,11,12,17,22,27]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm4, %ymm2, %ymm3
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,6,11,16,21,26,31,36,41,46,51,56,61,0,0,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm2
@@ -3875,23 +3875,23 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512-LABEL: load_i16_stride5_vf32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm0
-; AVX512-NEXT: vmovdqa 288(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 288(%rdi), %ymm1
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4],xmm2[5,6,7]
; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,4,5,14,15,8,9,2,3,12,13,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm7
-; AVX512-NEXT: vmovdqa 224(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm7
+; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm8
; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm8[0],ymm7[1],ymm8[2,3],ymm7[4],ymm8[5],ymm7[6],ymm8[7,8],ymm7[9],ymm8[10,11],ymm7[12],ymm8[13],ymm7[14],ymm8[15]
; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5,6,7]
; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23]
-; AVX512-NEXT: vmovdqa 176(%rdi), %xmm9
+; AVX512-NEXT: vmovdqa64 176(%rdi), %xmm9
; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm9[3,1,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
-; AVX512-NEXT: vmovdqa 160(%rdi), %xmm10
+; AVX512-NEXT: vmovdqa64 160(%rdi), %xmm10
; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm10[0,2,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
@@ -3910,8 +3910,8 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1,2,3],xmm12[4,5],xmm13[6,7]
; AVX512-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[u,u,u,u,u,u]
; AVX512-NEXT: vpor %ymm11, %ymm12, %ymm13
-; AVX512-NEXT: vmovdqa 144(%rdi), %xmm12
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm11
+; AVX512-NEXT: vmovdqa64 144(%rdi), %xmm12
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm11
; AVX512-NEXT: vpblendd {{.*#+}} xmm14 = xmm11[0],xmm12[1],xmm11[2,3]
; AVX512-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm15
@@ -4054,13 +4054,13 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512-FCP-LABEL: load_i16_stride5_vf32:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 176(%rdi), %xmm0
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[4,5,14,15,4,5,6,7,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %xmm1
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[0,1,10,11,8,9,10,11,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm3
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [2,4,7,1,4,6,0,0]
; AVX512-FCP-NEXT: vpermd %ymm5, %ymm6, %ymm5
@@ -4068,7 +4068,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,9,3,2,4,0,0,0]
; AVX512-FCP-NEXT: vpermi2d %ymm4, %ymm5, %ymm6
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,3,1,0,0,3,5,0]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512-FCP-NEXT: vpermd %ymm4, %ymm5, %ymm5
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm5, %ymm5
@@ -4088,8 +4088,8 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpor %ymm6, %ymm7, %ymm7
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm19 & (zmm7 ^ zmm5))
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm5
-; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %ymm6
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm6[0,1],ymm5[2],ymm6[3],ymm5[4],ymm6[5,6],ymm5[7],ymm6[8,9],ymm5[10],ymm6[11],ymm5[12],ymm6[13,14],ymm5[15]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm12, %xmm14
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm14[3,4],xmm12[5,6,7]
@@ -4233,23 +4233,23 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512DQ-LABEL: load_i16_stride5_vf32:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqa 288(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 288(%rdi), %ymm1
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4],xmm2[5,6,7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,4,5,14,15,8,9,2,3,12,13,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm7
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm7
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm8
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm8[0],ymm7[1],ymm8[2,3],ymm7[4],ymm8[5],ymm7[6],ymm8[7,8],ymm7[9],ymm8[10,11],ymm7[12],ymm8[13],ymm7[14],ymm8[15]
; AVX512DQ-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5,6,7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23]
-; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm9
+; AVX512DQ-NEXT: vmovdqa64 176(%rdi), %xmm9
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm9[3,1,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm10
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %xmm10
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm10[0,2,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
@@ -4268,8 +4268,8 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1,2,3],xmm12[4,5],xmm13[6,7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[u,u,u,u,u,u]
; AVX512DQ-NEXT: vpor %ymm11, %ymm12, %ymm13
-; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm12
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm11
+; AVX512DQ-NEXT: vmovdqa64 144(%rdi), %xmm12
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm11
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm14 = xmm11[0],xmm12[1],xmm11[2,3]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm15
@@ -4412,13 +4412,13 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512DQ-FCP-LABEL: load_i16_stride5_vf32:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 176(%rdi), %xmm0
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[4,5,14,15,4,5,6,7,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm1[0,1,10,11,8,9,10,11,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [2,4,7,1,4,6,0,0]
; AVX512DQ-FCP-NEXT: vpermd %ymm5, %ymm6, %ymm5
@@ -4426,7 +4426,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,9,3,2,4,0,0,0]
; AVX512DQ-FCP-NEXT: vpermi2d %ymm4, %ymm5, %ymm6
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,3,1,0,0,3,5,0]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm5, %ymm5
@@ -4446,8 +4446,8 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpor %ymm6, %ymm7, %ymm7
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm19 & (zmm7 ^ zmm5))
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm6[0,1],ymm5[2],ymm6[3],ymm5[4],ymm6[5,6],ymm5[7],ymm6[8,9],ymm5[10],ymm6[11],ymm5[12],ymm6[13,14],ymm5[15]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm12, %xmm14
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm14[3,4],xmm12[5,6,7]
@@ -7897,29 +7897,29 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-LABEL: load_i16_stride5_vf64:
; AVX512: # %bb.0:
; AVX512-NEXT: subq $552, %rsp # imm = 0x228
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm6
-; AVX512-NEXT: vmovdqa 416(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 416(%rdi), %ymm11
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0],ymm11[1,2],ymm6[3],ymm11[4],ymm6[5],ymm11[6,7],ymm6[8],ymm11[9,10],ymm6[11],ymm11[12],ymm6[13],ymm11[14,15]
-; AVX512-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6],ymm1[7]
; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,6,7,16,17,26,27,20,21,30,31,24,25,128,128,128,128,128,128]
; AVX512-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512-NEXT: vmovdqa 352(%rdi), %ymm8
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm7
+; AVX512-NEXT: vmovdqa64 352(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm7
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm7[0],ymm8[1],ymm7[2,3],ymm8[4],ymm7[5],ymm8[6],ymm7[7,8],ymm8[9],ymm7[10,11],ymm8[12],ymm7[13],ymm8[14],ymm7[15]
-; AVX512-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3],xmm1[4,5],xmm3[6,7]
; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
; AVX512-NEXT: vpshufb %ymm1, %ymm3, %ymm3
; AVX512-NEXT: vporq %ymm2, %ymm3, %ymm19
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm15
-; AVX512-NEXT: vmovdqa 224(%rdi), %ymm13
-; AVX512-NEXT: vmovdqa 176(%rdi), %xmm12
-; AVX512-NEXT: vmovdqa 160(%rdi), %xmm14
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm15
+; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm13
+; AVX512-NEXT: vmovdqa64 176(%rdi), %xmm12
+; AVX512-NEXT: vmovdqa64 160(%rdi), %xmm14
; AVX512-NEXT: vmovdqa (%rdi), %ymm4
; AVX512-NEXT: vmovdqa 32(%rdi), %ymm5
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm10
@@ -7972,8 +7972,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %xmm6, %xmm1, %xmm1
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4,5,6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vmovdqa 144(%rdi), %xmm11
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 144(%rdi), %xmm11
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm7
; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm11[0,1],xmm7[2],xmm11[3]
; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
@@ -7987,8 +7987,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %ymm6, %ymm1, %ymm1
; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm12
-; AVX512-NEXT: vmovdqa 288(%rdi), %ymm15
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm12
+; AVX512-NEXT: vmovdqa64 288(%rdi), %ymm15
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm12[0],ymm15[1],ymm12[2,3],ymm15[4],ymm12[5],ymm15[6],ymm12[7,8],ymm15[9],ymm12[10,11],ymm15[12],ymm12[13],ymm15[14],ymm12[15]
; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
@@ -8000,10 +8000,10 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 464(%rdi), %xmm8
+; AVX512-NEXT: vmovdqa64 464(%rdi), %xmm8
; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
-; AVX512-NEXT: vmovdqa 448(%rdi), %xmm3
+; AVX512-NEXT: vmovdqa64 448(%rdi), %xmm3
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -8022,22 +8022,22 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovdqa64 %xmm1, %xmm23
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm10
-; AVX512-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 608(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 608(%rdi), %ymm2
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1],ymm1[2],ymm2[3],ymm1[4],ymm2[5,6],ymm1[7],ymm2[8,9],ymm1[10],ymm2[11],ymm1[12],ymm2[13,14],ymm1[15]
; AVX512-NEXT: vmovdqa64 %ymm2, %ymm20
; AVX512-NEXT: vmovdqa64 %ymm1, %ymm17
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0,1,2],xmm1[3,4],xmm0[5,6,7]
-; AVX512-NEXT: vmovdqa 512(%rdi), %ymm5
-; AVX512-NEXT: vmovdqa 544(%rdi), %ymm13
+; AVX512-NEXT: vmovdqa64 512(%rdi), %ymm5
+; AVX512-NEXT: vmovdqa64 544(%rdi), %ymm13
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm13[0],ymm5[1],ymm13[2,3],ymm5[4],ymm13[5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10,11],ymm5[12],ymm13[13],ymm5[14],ymm13[15]
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5,6,7]
; AVX512-NEXT: vmovdqa64 496(%rdi), %xmm21
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm21[3,1,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[2,1,2,3,4,5,6,7]
-; AVX512-NEXT: vmovdqa 480(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 480(%rdi), %xmm7
; AVX512-NEXT: vpshufd {{.*#+}} xmm14 = xmm7[0,2,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[0,3,2,3,4,5,6,7]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
@@ -8140,7 +8140,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm27 & (zmm0 ^ zmm28))
; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Reload
; AVX512-NEXT: vmovdqa64 %ymm18, %ymm0
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm12[1],ymm0[2,3],ymm12[4],ymm0[5],ymm12[6],ymm0[7,8],ymm12[9],ymm0[10,11],ymm12[12],ymm0[13],ymm12[14],ymm0[15]
@@ -8369,16 +8369,16 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: subq $648, %rsp # imm = 0x288
; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,0,0,0,0,0,0,0,0]
-; AVX512-FCP-NEXT: vmovdqa 496(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 496(%rdi), %xmm1
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm2
; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm31
; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm1 = [0,1,10,11,8,9,10,11,0,0,0,0,0,0,0,0]
-; AVX512-FCP-NEXT: vmovdqa 480(%rdi), %xmm4
+; AVX512-FCP-NEXT: vmovdqa64 480(%rdi), %xmm4
; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm4, %xmm3
; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm21
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 544(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 544(%rdi), %ymm5
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm29
; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm30
@@ -8395,8 +8395,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm2
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %ymm11
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm4
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm11[1],ymm4[2,3],ymm11[4],ymm4[5],ymm11[6],ymm4[7,8],ymm11[9],ymm4[10,11],ymm11[12],ymm4[13],ymm11[14],ymm4[15]
; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm17
; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -8405,8 +8405,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm7
-; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 416(%rdi), %ymm8
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm7[0],ymm8[1,2],ymm7[3],ymm8[4],ymm7[5],ymm8[6,7],ymm7[8],ymm8[9,10],ymm7[11],ymm8[12],ymm7[13],ymm8[14,15]
; AVX512-FCP-NEXT: vmovdqu %ymm8, (%rsp) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -8416,15 +8416,15 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
; AVX512-FCP-NEXT: vpor %ymm3, %ymm2, %ymm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 176(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 176(%rdi), %xmm2
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm0
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm18
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %xmm2
; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm1
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm22
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm15
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm15
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm3
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm3[0],ymm15[1],ymm3[2,3],ymm15[4],ymm3[5],ymm15[6],ymm3[7,8],ymm15[9],ymm3[10,11],ymm15[12],ymm3[13],ymm15[14],ymm3[15]
; AVX512-FCP-NEXT: vpermd %ymm12, %ymm9, %ymm9
; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm9, %ymm5
@@ -8438,15 +8438,15 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm10
; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm9
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm10[0],ymm9[1,2],ymm10[3],ymm9[4],ymm10[5],ymm9[6,7],ymm10[8],ymm9[9,10],ymm10[11],ymm9[12],ymm10[13],ymm9[14,15]
-; AVX512-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermd %ymm0, %ymm13, %ymm0
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm1
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm2
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vextracti128 $1, %ymm5, %xmm6
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm4
@@ -8477,8 +8477,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3],ymm5[4,5,6,7]
; AVX512-FCP-NEXT: vbroadcasti32x4 {{.*#+}} ymm25 = [1,4,6,3,1,4,6,3]
; AVX512-FCP-NEXT: # ymm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4],ymm1[5],ymm0[6,7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12],ymm1[13],ymm0[14,15]
; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm23
@@ -8514,8 +8514,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm9[0],xmm4[1],xmm9[1]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm1[3,4,5,6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %ymm5
-; AVX512-FCP-NEXT: vmovdqa 608(%rdi), %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 608(%rdi), %ymm7
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm5[0],ymm7[1,2],ymm5[3],ymm7[4],ymm5[5],ymm7[6,7],ymm5[8],ymm7[9,10],ymm5[11],ymm7[12],ymm5[13],ymm7[14,15]
; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm30
; AVX512-FCP-NEXT: vpermd %ymm4, %ymm25, %ymm4
@@ -8622,14 +8622,14 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: # zmm29 = mem ^ (zmm26 & (zmm29 ^ mem))
; AVX512-FCP-NEXT: vmovdqa64 %zmm29, %zmm30
; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm18, %zmm30
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm6[0,1],ymm0[2],ymm6[3],ymm0[4],ymm6[5,6],ymm0[7],ymm6[8,9],ymm0[10],ymm6[11],ymm0[12],ymm6[13,14],ymm0[15]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm4, %xmm7
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[3,4],xmm4[5,6,7]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [0,2,0,0,5,7,2,4]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0],ymm1[1],ymm15[2,3],ymm1[4],ymm15[5],ymm1[6],ymm15[7,8],ymm1[9],ymm15[10,11],ymm1[12],ymm15[13],ymm1[14],ymm15[15]
; AVX512-FCP-NEXT: vpermd %ymm8, %ymm7, %ymm8
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,6,7,8,9,14,15,4,5,6,7,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
@@ -8801,29 +8801,29 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-LABEL: load_i16_stride5_vf64:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: subq $552, %rsp # imm = 0x228
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm6
-; AVX512DQ-NEXT: vmovdqa 416(%rdi), %ymm11
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 416(%rdi), %ymm11
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0],ymm11[1,2],ymm6[3],ymm11[4],ymm6[5],ymm11[6,7],ymm6[8],ymm11[9,10],ymm6[11],ymm11[12],ymm6[13],ymm11[14,15]
-; AVX512DQ-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6],ymm1[7]
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,6,7,16,17,26,27,20,21,30,31,24,25,128,128,128,128,128,128]
; AVX512DQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %ymm8
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm7
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm7
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm7[0],ymm8[1],ymm7[2,3],ymm8[4],ymm7[5],ymm8[6],ymm7[7,8],ymm8[9],ymm7[10,11],ymm8[12],ymm7[13],ymm8[14],ymm7[15]
-; AVX512DQ-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm3
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3],xmm1[4,5],xmm3[6,7]
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpshufb %ymm1, %ymm3, %ymm3
; AVX512DQ-NEXT: vporq %ymm2, %ymm3, %ymm19
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm15
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm13
-; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm12
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm14
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm15
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 176(%rdi), %xmm12
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %xmm14
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm4
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm5
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm10
@@ -8876,8 +8876,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb %xmm6, %xmm1, %xmm1
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4,5,6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm11
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 144(%rdi), %xmm11
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm7
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm11[0,1],xmm7[2],xmm11[3]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
@@ -8891,8 +8891,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm1, %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm12
-; AVX512DQ-NEXT: vmovdqa 288(%rdi), %ymm15
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm12
+; AVX512DQ-NEXT: vmovdqa64 288(%rdi), %ymm15
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm12[0],ymm15[1],ymm12[2,3],ymm15[4],ymm12[5],ymm15[6],ymm12[7,8],ymm15[9],ymm12[10,11],ymm15[12],ymm12[13],ymm15[14],ymm12[15]
; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
@@ -8904,10 +8904,10 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm8
+; AVX512DQ-NEXT: vmovdqa64 464(%rdi), %xmm8
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %xmm3
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %xmm3
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
@@ -8926,22 +8926,22 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm0, %xmm0
; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm23
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm10
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 608(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 608(%rdi), %ymm2
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0,1],ymm1[2],ymm2[3],ymm1[4],ymm2[5,6],ymm1[7],ymm2[8,9],ymm1[10],ymm2[11],ymm1[12],ymm2[13,14],ymm1[15]
; AVX512DQ-NEXT: vmovdqa64 %ymm2, %ymm20
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm17
; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0,1,2],xmm1[3,4],xmm0[5,6,7]
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %ymm5
-; AVX512DQ-NEXT: vmovdqa 544(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 544(%rdi), %ymm13
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm13[0],ymm5[1],ymm13[2,3],ymm5[4],ymm13[5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10,11],ymm5[12],ymm13[13],ymm5[14],ymm13[15]
; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 496(%rdi), %xmm21
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm21[3,1,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[2,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 480(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 480(%rdi), %xmm7
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm7[0,2,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm14 = xmm14[0,3,2,3,4,5,6,7]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
@@ -9044,7 +9044,7 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm27 & (zmm0 ^ zmm28))
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %ymm18, %ymm0
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm12[1],ymm0[2,3],ymm12[4],ymm0[5],ymm12[6],ymm0[7,8],ymm12[9],ymm0[10,11],ymm12[12],ymm0[13],ymm12[14],ymm0[15]
@@ -9273,16 +9273,16 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: subq $648, %rsp # imm = 0x288
; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,0,0,0,0,0,0,0,0]
-; AVX512DQ-FCP-NEXT: vmovdqa 496(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 496(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm31
; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm1 = [0,1,10,11,8,9,10,11,0,0,0,0,0,0,0,0]
-; AVX512DQ-FCP-NEXT: vmovdqa 480(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 480(%rdi), %xmm4
; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm4, %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm21
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 544(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 544(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, %ymm29
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm30
@@ -9299,8 +9299,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm2
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm11[1],ymm4[2,3],ymm11[4],ymm4[5],ymm11[6],ymm4[7,8],ymm11[9],ymm4[10,11],ymm11[12],ymm4[13],ymm11[14],ymm4[15]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm17
; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -9309,8 +9309,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm7
-; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 416(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm7[0],ymm8[1,2],ymm7[3],ymm8[4],ymm7[5],ymm8[6,7],ymm7[8],ymm8[9,10],ymm7[11],ymm8[12],ymm7[13],ymm8[14,15]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm8, (%rsp) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -9320,15 +9320,15 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
; AVX512DQ-FCP-NEXT: vpor %ymm3, %ymm2, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 176(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 176(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm18
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm22
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm15
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm3[0],ymm15[1],ymm3[2,3],ymm15[4],ymm3[5],ymm15[6],ymm3[7,8],ymm15[9],ymm3[10,11],ymm15[12],ymm3[13],ymm15[14],ymm3[15]
; AVX512DQ-FCP-NEXT: vpermd %ymm12, %ymm9, %ymm9
; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm9, %ymm5
@@ -9342,15 +9342,15 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm10
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm9
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm10[0],ymm9[1,2],ymm10[3],ymm9[4],ymm10[5],ymm9[6,7],ymm10[8],ymm9[9,10],ymm10[11],ymm9[12],ymm10[13],ymm9[14,15]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm13, %ymm0
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm5, %xmm6
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1,2,3],xmm5[4,5],xmm6[6,7]
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm4
@@ -9381,8 +9381,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3],ymm5[4,5,6,7]
; AVX512DQ-FCP-NEXT: vbroadcasti32x4 {{.*#+}} ymm25 = [1,4,6,3,1,4,6,3]
; AVX512DQ-FCP-NEXT: # ymm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4],ymm1[5],ymm0[6,7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12],ymm1[13],ymm0[14,15]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm23
@@ -9418,8 +9418,8 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm9[0],xmm4[1],xmm9[1]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm1[3,4,5,6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 608(%rdi), %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 608(%rdi), %ymm7
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm5[0],ymm7[1,2],ymm5[3],ymm7[4],ymm5[5],ymm7[6,7],ymm5[8],ymm7[9,10],ymm5[11],ymm7[12],ymm5[13],ymm7[14,15]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, %ymm30
; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm25, %ymm4
@@ -9526,14 +9526,14 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: # zmm29 = mem ^ (zmm26 & (zmm29 ^ mem))
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm29, %zmm30
; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm18, %zmm30
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm6[0,1],ymm0[2],ymm6[3],ymm0[4],ymm6[5,6],ymm0[7],ymm6[8,9],ymm0[10],ymm6[11],ymm0[12],ymm6[13,14],ymm0[15]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm4, %xmm7
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[3,4],xmm4[5,6,7]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [0,2,0,0,5,7,2,4]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm15[0],ymm1[1],ymm15[2,3],ymm1[4],ymm15[5],ymm1[6],ymm15[7,8],ymm1[9],ymm15[10,11],ymm1[12],ymm15[13],ymm1[14],ymm15[15]
; AVX512DQ-FCP-NEXT: vpermd %ymm8, %ymm7, %ymm8
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,6,7,8,9,14,15,4,5,6,7,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
index 441fa54f98264..8d7bb62cbaf3a 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-6.ll
@@ -2408,11 +2408,11 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512-LABEL: load_i16_stride6_vf16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm0
; AVX512-NEXT: vmovdqa (%rdi), %ymm3
; AVX512-NEXT: vmovdqa 32(%rdi), %ymm4
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[2,2,2,2,4,5,6,7]
; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm8
@@ -2525,7 +2525,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm2
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6],ymm3[7]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm5
; AVX512-FCP-NEXT: vextracti128 $1, %ymm8, %xmm6
@@ -2537,7 +2537,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0],ymm6[1],ymm7[2,3,4,5],ymm6[6],ymm7[7]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm5[3,4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm5
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm1[0,1],ymm5[2],ymm1[3,4],ymm5[5],ymm1[6,7]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm11 = [0,1,4,5,4,5,4,5,0,1,12,13,8,9,4,5]
; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm10, %xmm12
@@ -2631,11 +2631,11 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
;
; AVX512DQ-LABEL: load_i16_stride6_vf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm3
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm4
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[2,2,2,2,4,5,6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm5, %xmm8
@@ -2748,7 +2748,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6],ymm3[7]
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm5
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm8, %xmm6
@@ -2760,7 +2760,7 @@ define void @load_i16_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm7[0],ymm6[1],ymm7[2,3,4,5],ymm6[6],ymm7[7]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm5[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm1[0,1],ymm5[2],ymm1[3,4],ymm5[5],ymm1[6,7]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm11 = [0,1,4,5,4,5,4,5,0,1,12,13,8,9,4,5]
; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm10, %xmm12
@@ -4978,19 +4978,19 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512: # %bb.0:
; AVX512-NEXT: subq $72, %rsp
; AVX512-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
-; AVX512-NEXT: vmovdqa 224(%rdi), %ymm14
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm14
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm11
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm11[0],ymm14[1],ymm11[2,3],ymm14[4],ymm11[5,6],ymm14[7]
; AVX512-NEXT: vpshufb %xmm5, %xmm2, %xmm1
; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,2,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1],xmm3[2],xmm1[3],xmm3[4,5],xmm1[6,7]
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm4
; AVX512-NEXT: vmovdqa (%rdi), %ymm13
; AVX512-NEXT: vmovdqa 32(%rdi), %ymm10
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm6
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm7
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm7
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1],ymm4[2],ymm7[3,4],ymm4[5],ymm7[6,7]
; AVX512-NEXT: vmovdqa64 %ymm7, %ymm24
; AVX512-NEXT: vmovdqa64 %ymm4, %ymm26
@@ -5017,8 +5017,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb {{.*#+}} ymm8 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3,4,5,6,7]
; AVX512-NEXT: vmovdqu64 %zmm7, (%rsp) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 352(%rdi), %ymm8
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm10
+; AVX512-NEXT: vmovdqa64 352(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm10
; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1],ymm8[2],ymm10[3,4],ymm8[5],ymm10[6,7]
; AVX512-NEXT: vmovdqa64 %ymm10, %ymm18
; AVX512-NEXT: vmovdqa64 %ymm8, %ymm20
@@ -5027,9 +5027,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm10 = xmm8[0,1,2],xmm10[3,4],xmm8[5,6,7]
; AVX512-NEXT: vpshufb %xmm9, %xmm10, %xmm9
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm10
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm9
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm12 = ymm9[2,3],mem[2,3]
-; AVX512-NEXT: vinserti128 $1, 288(%rdi), %ymm9, %ymm15
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm9
+; AVX512-NEXT: vshufi64x2 {{.*#+}} ymm12 = ymm9[2,3],mem[2,3]
+; AVX512-NEXT: vinserti32x4 $1, 288(%rdi), %ymm9, %ymm15
; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm15[0],ymm12[1],ymm15[2,3,4,5],ymm12[6],ymm15[7]
; AVX512-NEXT: vmovdqa64 %ymm15, %ymm25
; AVX512-NEXT: vmovdqa64 %ymm12, %ymm27
@@ -5261,8 +5261,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-LABEL: load_i16_stride6_vf32:
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,12,13,0,1,4,5,8,9,12,13,12,13,14,15]
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm11
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm11[1],ymm1[2,3],ymm11[4],ymm1[5,6],ymm11[7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm23
; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm0
@@ -5270,11 +5270,11 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,1,0,3]
; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm9, %xmm1
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm5
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm1
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm10
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm25
; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm26
@@ -5300,8 +5300,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm28
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm2[0,1,2],ymm5[3,4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm5
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm0[2],ymm5[3,4],ymm0[5],ymm5[6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm20
; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm21
@@ -5310,9 +5310,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm6
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm10[3],xmm6[4,5],xmm10[6],xmm6[7]
; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm8
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm6
-; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm6[2,3],mem[2,3]
-; AVX512-FCP-NEXT: vinserti128 $1, 288(%rdi), %ymm6, %ymm10
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm6
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} ymm0 = ymm6[2,3],mem[2,3]
+; AVX512-FCP-NEXT: vinserti32x4 $1, 288(%rdi), %ymm6, %ymm10
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm10[0],ymm0[1],ymm10[2,3,4,5],ymm0[6],ymm10[7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm10, %ymm22
; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm24
@@ -5523,8 +5523,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: pushq %rax
; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm13
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0],ymm13[1],ymm2[2,3],ymm13[4],ymm2[5,6],ymm13[7]
; AVX512DQ-NEXT: vmovdqa64 %ymm2, %ymm25
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm1
@@ -5532,11 +5532,11 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm9[0,2,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm2
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm5
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm6
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm4
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm7
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm7
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm14 = ymm7[0,1],ymm2[2],ymm7[3,4],ymm2[5],ymm7[6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm7, %ymm20
; AVX512DQ-NEXT: vmovdqa64 %ymm2, %ymm22
@@ -5561,8 +5561,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm28
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm5 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm0[0,1,2],ymm5[3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm5
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1],ymm0[2],ymm5[3,4],ymm0[5],ymm5[6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm5, %ymm21
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm23
@@ -5571,9 +5571,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm5[0,1,2],xmm11[3,4],xmm5[5,6,7]
; AVX512DQ-NEXT: vpshufb %xmm7, %xmm11, %xmm7
; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm8
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm7
-; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm7[2,3],mem[2,3]
-; AVX512DQ-NEXT: vinserti128 $1, 288(%rdi), %ymm7, %ymm11
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm7
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} ymm0 = ymm7[2,3],mem[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $1, 288(%rdi), %ymm7, %ymm11
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm11[0],ymm0[1],ymm11[2,3,4,5],ymm0[6],ymm11[7]
; AVX512DQ-NEXT: vmovdqa64 %ymm11, %ymm24
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm26
@@ -5798,8 +5798,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-LABEL: load_i16_stride6_vf32:
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,12,13,0,1,4,5,8,9,12,13,12,13,14,15]
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm12
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm12[1],ymm1[2,3],ymm12[4],ymm1[5,6],ymm12[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm18
; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm0
@@ -5807,11 +5807,11 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,1,0,3]
; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm9, %xmm1
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm10
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm24
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, %ymm26
@@ -5837,8 +5837,8 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm29
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm2[0,1,2],ymm5[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm0[2],ymm5[3,4],ymm0[5],ymm5[6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, %ymm22
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm23
@@ -5847,9 +5847,9 @@ define void @load_i16_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm6
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm11[3],xmm6[4,5],xmm11[6],xmm6[7]
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm8
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm6
-; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm6[2,3],mem[2,3]
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 288(%rdi), %ymm6, %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} ymm0 = ymm6[2,3],mem[2,3]
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 288(%rdi), %ymm6, %ymm11
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm11[0],ymm0[1],ymm11[2,3,4,5],ymm0[6],ymm11[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm11, %ymm25
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm27
@@ -10380,10 +10380,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512: # %bb.0:
; AVX512-NEXT: subq $1480, %rsp # imm = 0x5C8
; AVX512-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
-; AVX512-NEXT: vmovdqa 608(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 608(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
; AVX512-NEXT: vpshufb %xmm9, %xmm1, %xmm0
; AVX512-NEXT: vextracti32x4 $1, %ymm1, %xmm20
@@ -10391,10 +10391,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm20[0,2,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
-; AVX512-NEXT: vmovdqa 544(%rdi), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 512(%rdi), %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 544(%rdi), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 512(%rdi), %ymm2
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm13, %xmm2
; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[2,2,2,2,4,5,6,7]
@@ -10405,32 +10405,32 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 448(%rdi), %ymm0
-; AVX512-NEXT: vmovdqa 416(%rdi), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 448(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 416(%rdi), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm2
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm12 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]
; AVX512-NEXT: vpshufb %xmm9, %xmm12, %xmm1
; AVX512-NEXT: vextracti32x4 $1, %ymm12, %xmm22
; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm22[0,2,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3],xmm2[4,5],xmm1[6,7]
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],mem[2,3]
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vinserti128 $1, 480(%rdi), %ymm0, %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vshufi64x2 {{.*#+}} ymm2 = ymm0[2,3],mem[2,3]
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vinserti32x4 $1, 480(%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2,3,4,5],ymm2[6],ymm0[7]
; AVX512-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,12,13,8,9,12,13,8,9,12,13,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512-NEXT: vpshufb %ymm5, %ymm2, %ymm0
; AVX512-NEXT: vmovdqa64 %ymm2, %ymm23
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 640(%rdi), %ymm0
-; AVX512-NEXT: vmovdqa 736(%rdi), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 704(%rdi), %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 640(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 736(%rdi), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 704(%rdi), %ymm2
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm11, %xmm2
; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[2,2,2,2,4,5,6,7]
@@ -10438,10 +10438,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %xmm2, %xmm28
; AVX512-NEXT: vpshufb %xmm10, %xmm1, %xmm1
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm0[2,3],mem[2,3]
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vinserti128 $1, 672(%rdi), %ymm0, %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vshufi64x2 {{.*#+}} ymm2 = ymm0[2,3],mem[2,3]
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vinserti32x4 $1, 672(%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2,3,4,5],ymm2[6],ymm0[7]
; AVX512-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u]
; AVX512-NEXT: vpshufb %ymm3, %ymm2, %ymm0
@@ -10450,10 +10450,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7],ymm0[8,9,10],ymm1[11,12,13,14,15]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 224(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm14 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
; AVX512-NEXT: vextracti128 $1, %ymm14, %xmm15
@@ -10461,8 +10461,8 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vpshufb %xmm9, %xmm14, %xmm1
; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512-NEXT: vmovdqa64 %ymm1, %ymm30
@@ -10476,7 +10476,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa (%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]
; AVX512-NEXT: vpshufb %xmm9, %xmm3, %xmm0
; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm9
@@ -10492,19 +10492,19 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %ymm5, %ymm8, %ymm2
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 352(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 352(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm1
; AVX512-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm6
; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[2,2,2,2,4,5,6,7]
; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0,1,2],xmm2[3,4],xmm6[5,6,7]
; AVX512-NEXT: vpshufb %xmm10, %xmm2, %xmm10
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm2
-; AVX512-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],mem[2,3]
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm2
+; AVX512-NEXT: vshufi64x2 {{.*#+}} ymm1 = ymm2[2,3],mem[2,3]
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vinserti128 $1, 288(%rdi), %ymm2, %ymm0
+; AVX512-NEXT: vinserti32x4 $1, 288(%rdi), %ymm2, %ymm0
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm31
; AVX512-NEXT: vmovdqa64 %ymm17, %ymm0
@@ -10513,7 +10513,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm0[0,1,2],ymm10[3,4,5,6,7],ymm0[8,9,10],ymm10[11,12,13,14,15]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,6]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4,5,6,7]
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa {{.*#+}} xmm10 = [2,3,14,15,10,11,6,7,2,3,14,15,12,13,14,15]
; AVX512-NEXT: vpshufb %xmm10, %xmm15, %xmm0
; AVX512-NEXT: vpshufb %xmm10, %xmm14, %xmm14
@@ -10645,7 +10645,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpmovsxdq {{.*#+}} zmm22 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm22 & (zmm3 ^ zmm1))
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
@@ -10655,7 +10655,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,6,6]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[2,1,2,0,4,5,6,7]
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512-NEXT: # ymm2 = mem[0,1],ymm2[2],mem[3,4],ymm2[5],mem[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
@@ -10667,7 +10667,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5,6],xmm3[7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm23
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
@@ -10677,13 +10677,13 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,3,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[2,1,2,0,4,5,6,7]
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-NEXT: vpblendd $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
; AVX512-NEXT: # ymm5 = ymm2[0,1],mem[2],ymm2[3],mem[4],ymm2[5,6],mem[7]
; AVX512-NEXT: vpshufb %ymm0, %ymm5, %ymm0
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm0[5,6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm0
@@ -10693,7 +10693,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,1,2,1]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm14 = xmm3[0,1,2,3,6,5,6,4]
; AVX512-NEXT: vpblendw {{.*#+}} xmm14 = xmm1[0,1,2,3],xmm14[4],xmm1[5,6],xmm14[7]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
; AVX512-NEXT: vmovdqa64 %ymm24, %ymm0
@@ -10844,10 +10844,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %ymm5, %ymm19
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm3, %ymm23
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
; AVX512-NEXT: # ymm5 = mem[0],ymm3[1],mem[2,3,4,5],ymm3[6],mem[7]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm15 # 32-byte Folded Reload
; AVX512-NEXT: # ymm15 = mem[0,1],ymm3[2],mem[3,4],ymm3[5],mem[6,7]
; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm3
@@ -10857,14 +10857,14 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %ymm0, %ymm5, %ymm0
; AVX512-NEXT: vmovdqa64 %ymm5, %ymm17
; AVX512-NEXT: vpternlogq {{.*#+}} ymm4 = (ymm4 & ymm29) | ymm0
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
; AVX512-NEXT: # ymm5 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm1
; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm9
; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm9[2,2,2,2,4,5,6,7]
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3],xmm3[4],xmm1[5,6,7]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpblendd $146, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
; AVX512-NEXT: # ymm3 = ymm0[0],mem[1],ymm0[2,3],mem[4],ymm0[5,6],mem[7]
; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm12
@@ -10876,10 +10876,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512-NEXT: vinserti32x4 $2, %xmm1, %zmm3, %zmm31
; AVX512-NEXT: vmovdqa32 %zmm4, %zmm31 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
; AVX512-NEXT: # ymm8 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm7
@@ -11007,10 +11007,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: subq $872, %rsp # imm = 0x368
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,12,13,0,1,4,5,8,9,12,13,12,13,14,15]
-; AVX512-FCP-NEXT: vmovdqa 608(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 608(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm0
; AVX512-FCP-NEXT: vextracti128 $1, %ymm2, %xmm1
@@ -11019,10 +11019,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm1
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm25
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
-; AVX512-FCP-NEXT: vmovdqa 544(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 544(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,4,5,4,5,4,5,0,1,12,13,8,9,4,5]
; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm1
@@ -11033,11 +11033,11 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 416(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]
; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm15, %xmm2
; AVX512-FCP-NEXT: vextracti128 $1, %ymm15, %xmm3
@@ -11045,31 +11045,31 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm4, %xmm3
; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm22
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3],xmm3[4,5],xmm2[6,7]
-; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],mem[2,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vinserti128 $1, 480(%rdi), %ymm1, %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} ymm3 = ymm1[2,3],mem[2,3]
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vinserti32x4 $1, 480(%rdi), %ymm1, %ymm1
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm3[1],ymm1[2,3,4,5],ymm3[6],ymm1[7]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,12,13,8,9,12,13,8,9,12,13,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm21
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1,2],ymm1[3,4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 736(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 704(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 736(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 704(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm12, %xmm1
; AVX512-FCP-NEXT: vextracti128 $1, %ymm12, %xmm4
; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm2
; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm20
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %ymm2
; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm2[2,3],mem[2,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vinserti128 $1, 672(%rdi), %ymm2, %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} ymm4 = ymm2[2,3],mem[2,3]
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vinserti32x4 $1, 672(%rdi), %ymm2, %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0],ymm4[1],ymm2[2,3,4,5],ymm4[6],ymm2[7]
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u]
; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm2
@@ -11083,19 +11083,19 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,19,20,21,22,23]
; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm3
; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm13, %xmm0
; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm13, %xmm1
; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm14 = xmm0[2,1,0,3]
; AVX512-FCP-NEXT: vpshufb %xmm9, %xmm14, %xmm0
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512-FCP-NEXT: vmovdqu %ymm2, (%rsp) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm1
@@ -11122,18 +11122,18 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm31
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm9, %ymm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm1
; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm2
; AVX512-FCP-NEXT: vextracti128 $1, %ymm4, %xmm8
; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm8, %xmm10
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm2[3],xmm10[4,5],xmm2[6],xmm10[7]
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm2
-; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],mem[2,3]
-; AVX512-FCP-NEXT: vinserti128 $1, 288(%rdi), %ymm2, %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm2
+; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} ymm1 = ymm2[2,3],mem[2,3]
+; AVX512-FCP-NEXT: vinserti32x4 $1, 288(%rdi), %ymm2, %ymm0
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm29
; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm30
@@ -11278,7 +11278,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpmovsxdq {{.*#+}} zmm27 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm27 & (zmm5 ^ zmm4))
; AVX512-FCP-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm3 = mem[0,1],ymm3[2],mem[3,4],ymm3[5],mem[6,7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
@@ -11288,7 +11288,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm13 = xmm3[2,1,2,3]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[2,1,2,0,4,5,6,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0],xmm0[1,2],xmm3[3],xmm0[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm3 = ymm3[0,1],mem[2],ymm3[3,4],mem[5],ymm3[6,7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
@@ -11299,7 +11299,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4],xmm3[5,6],xmm4[7]
; AVX512-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm9
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm3
@@ -11308,13 +11308,13 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[2,1,2,3]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm7[2,1,2,0,4,5,6,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2],xmm0[3],xmm2[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm6 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm6 = ymm2[0,1],mem[2],ymm2[3],mem[4],ymm2[5,6],mem[7]
; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm6, %ymm1
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm2
@@ -11323,7 +11323,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[0,1,2,1]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm10 = xmm4[0,1,2,3,6,5,6,4]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm10 = xmm0[0,1,2,3],xmm10[4],xmm0[5,6],xmm10[7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
@@ -11459,10 +11459,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm29
; AVX512-FCP-NEXT: vpermt2d %ymm2, %ymm22, %ymm3
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm24
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm5 = mem[0],ymm2[1],mem[2,3,4,5],ymm2[6],mem[7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm4 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm4 = mem[0,1],ymm2[2],mem[3,4],ymm2[5],mem[6,7]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm2
@@ -11472,14 +11472,14 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm30
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm21) | ymm1
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm6 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm6 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm6, %xmm0
; AVX512-FCP-NEXT: vextracti128 $1, %ymm6, %xmm8
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm8[2,2,2,2,4,5,6,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6,7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm3 = mem[0],ymm3[1],mem[2,3],ymm3[4],mem[5,6],ymm3[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm1
@@ -11491,10 +11491,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm25
; AVX512-FCP-NEXT: vmovdqa32 %zmm2, %zmm25 {%k1}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm3
@@ -11588,10 +11588,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: subq $840, %rsp # imm = 0x348
; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,12,13,u,u,4,5,u,u,u,u,12,13,14,15]
-; AVX512DQ-NEXT: vmovdqa 608(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 608(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
; AVX512DQ-NEXT: vpshufb %xmm9, %xmm1, %xmm0
; AVX512DQ-NEXT: vextracti32x4 $1, %ymm1, %xmm24
@@ -11599,10 +11599,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm24[0,2,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
-; AVX512DQ-NEXT: vmovdqa 544(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 544(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[2,2,2,2,4,5,6,7]
@@ -11613,31 +11613,31 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb %xmm10, %xmm1, %xmm1
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512DQ-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 416(%rdi), %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 416(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm15 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]
; AVX512DQ-NEXT: vpshufb %xmm9, %xmm15, %xmm2
; AVX512DQ-NEXT: vextracti32x4 $1, %ymm15, %xmm21
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm21[0,2,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3],xmm3[4,5],xmm2[6,7]
-; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],mem[2,3]
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 480(%rdi), %ymm1, %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} ymm3 = ymm1[2,3],mem[2,3]
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vinserti32x4 $1, 480(%rdi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm3[1],ymm1[2,3,4,5],ymm3[6],ymm1[7]
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,12,13,8,9,12,13,8,9,12,13,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512DQ-NEXT: vpshufb %ymm5, %ymm3, %ymm1
; AVX512DQ-NEXT: vmovdqa64 %ymm3, %ymm20
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1,2],ymm1[3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 640(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 736(%rdi), %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 704(%rdi), %ymm4
-; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 736(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 704(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm12 = ymm4[0,1],ymm2[2],ymm4[3,4],ymm2[5],ymm4[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm12, %xmm4
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[2,2,2,2,4,5,6,7]
@@ -11645,10 +11645,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %xmm4, %xmm19
; AVX512DQ-NEXT: vpshufb %xmm10, %xmm2, %xmm2
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],mem[2,3]
-; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 672(%rdi), %ymm1, %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} ymm4 = ymm1[2,3],mem[2,3]
+; AVX512DQ-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vinserti32x4 $1, 672(%rdi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0],ymm4[1],ymm1[2,3,4,5],ymm4[6],ymm1[7]
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u]
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm4, %ymm1
@@ -11663,9 +11663,9 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: kmovw %eax, %k1
; AVX512DQ-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm3 {%k1}
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm13, %xmm14
@@ -11673,9 +11673,9 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vpshufb %xmm9, %xmm13, %xmm1
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm4, %xmm11
@@ -11702,18 +11702,18 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm30
; AVX512DQ-NEXT: vpshufb %ymm5, %ymm6, %ymm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm1
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm5, %xmm8
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[2,2,2,2,4,5,6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0,1,2],xmm2[3,4],xmm8[5,6,7]
; AVX512DQ-NEXT: vpshufb %xmm10, %xmm2, %xmm10
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm2
-; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],mem[2,3]
-; AVX512DQ-NEXT: vinserti128 $1, 288(%rdi), %ymm2, %ymm0
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm2
+; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} ymm1 = ymm2[2,3],mem[2,3]
+; AVX512DQ-NEXT: vinserti32x4 $1, 288(%rdi), %ymm2, %ymm0
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm26
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm29
@@ -11858,7 +11858,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpmovsxdq {{.*#+}} zmm18 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm18 & (zmm3 ^ zmm1))
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[0,1],ymm1[2],mem[3,4],ymm1[5],mem[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm2
@@ -11868,7 +11868,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,6,6,6]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[2,1,2,0,4,5,6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm2 = ymm2[0,1],mem[2],ymm2[3,4],mem[5],ymm2[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
@@ -11880,7 +11880,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5,6],xmm3[7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm19
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = ymm1[0,1],mem[2],ymm1[3,4],mem[5],ymm1[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm2
@@ -11890,13 +11890,13 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,3,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[2,1,2,0,4,5,6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2],xmm2[3],xmm1[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm5 = ymm2[0,1],mem[2],ymm2[3],mem[4],ymm2[5,6],mem[7]
; AVX512DQ-NEXT: vpshufb %ymm0, %ymm5, %ymm0
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm0[5,6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm0
@@ -11906,7 +11906,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,1,2,1]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm14 = xmm3[0,1,2,3,6,5,6,4]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm14 = xmm1[0,1,2,3],xmm14[4],xmm1[5,6],xmm14[7]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
; AVX512DQ-NEXT: vmovdqa64 %ymm27, %ymm0
@@ -12047,10 +12047,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %ymm5, %ymm17
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm24
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm14 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm14 = mem[0],ymm2[1],mem[2,3,4,5],ymm2[6],mem[7]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm15 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm15 = mem[0,1],ymm2[2],mem[3,4],ymm2[5],mem[6,7]
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm2
@@ -12059,14 +12059,14 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3],xmm4[4],xmm2[5,6,7]
; AVX512DQ-NEXT: vpshufb %ymm0, %ymm14, %ymm0
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm2 & ymm22) | ymm0
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm6 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm1
; AVX512DQ-NEXT: vextracti128 $1, %ymm6, %xmm10
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[2,2,2,2,4,5,6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2,3],xmm4[4],xmm1[5,6,7]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm4 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm4, %xmm8
@@ -12078,10 +12078,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-NEXT: vinserti32x4 $2, %xmm1, %zmm4, %zmm25
; AVX512DQ-NEXT: vinserti32x8 $0, %ymm2, %zmm0, %zmm25 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm5 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm4
@@ -12186,10 +12186,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: subq $872, %rsp # imm = 0x368
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,12,13,0,1,4,5,8,9,12,13,12,13,14,15]
-; AVX512DQ-FCP-NEXT: vmovdqa 608(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 608(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm0
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm2, %xmm1
@@ -12198,10 +12198,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm25
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3],xmm1[4,5],xmm0[6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 544(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 544(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,4,5,4,5,4,5,0,1,12,13,8,9,4,5]
; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm1
@@ -12212,11 +12212,11 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 416(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]
; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm15, %xmm2
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm15, %xmm3
@@ -12224,31 +12224,31 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm4, %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm22
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3],xmm3[4,5],xmm2[6,7]
-; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm1[2,3],mem[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 480(%rdi), %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} ymm3 = ymm1[2,3],mem[2,3]
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 480(%rdi), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm3[1],ymm1[2,3,4,5],ymm3[6],ymm1[7]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,12,13,8,9,12,13,8,9,12,13,8,9,4,5,16,17,28,29,24,25,28,29,24,25,28,29,24,25,20,21]
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm21
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1,2],ymm1[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 736(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 704(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 736(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 704(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm12, %xmm1
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm12, %xmm4
; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm20
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm2[2,3],mem[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 672(%rdi), %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} ymm4 = ymm2[2,3],mem[2,3]
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 672(%rdi), %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm2[0],ymm4[1],ymm2[2,3,4,5],ymm4[6],ymm2[7]
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,12,13,8,9,12,13,8,9,12,13,4,5,u,u,16,17,28,29,24,25,28,29,24,25,28,29,20,21,u,u]
; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm2
@@ -12262,9 +12262,9 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,2,3,4,5,6,7,8,9,10,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm3
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm13, %xmm0
@@ -12272,9 +12272,9 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm14 = xmm0[2,1,0,3]
; AVX512DQ-FCP-NEXT: vpshufb %xmm9, %xmm14, %xmm0
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4,5],xmm1[6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, (%rsp) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm5, %xmm1
@@ -12301,18 +12301,18 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm31
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm9, %ymm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]
; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm2
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm4, %xmm8
; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm8, %xmm10
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm2[3],xmm10[4,5],xmm2[6],xmm10[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],mem[2,3]
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 288(%rdi), %ymm2, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} ymm1 = ymm2[2,3],mem[2,3]
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 288(%rdi), %ymm2, %ymm0
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3,4,5],ymm1[6],ymm0[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm30
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm29
@@ -12457,7 +12457,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpmovsxdq {{.*#+}} zmm27 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,65535,0,0]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm27 & (zmm5 ^ zmm4))
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1],ymm3[2],mem[3,4],ymm3[5],mem[6,7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
@@ -12467,7 +12467,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm13 = xmm3[2,1,2,3]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[2,1,2,0,4,5,6,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0],xmm0[1,2],xmm3[3],xmm0[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm3 = ymm3[0,1],mem[2],ymm3[3,4],mem[5],ymm3[6,7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm4
@@ -12478,7 +12478,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4],xmm3[5,6],xmm4[7]
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm9
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm3
@@ -12487,13 +12487,13 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[2,1,2,3]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm7[2,1,2,0,4,5,6,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2],xmm0[3],xmm2[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm6 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm6 = ymm2[0,1],mem[2],ymm2[3],mem[4],ymm2[5,6],mem[7]
; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm6, %ymm1
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5,6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm2
@@ -12502,7 +12502,7 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[0,1,2,1]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm10 = xmm4[0,1,2,3,6,5,6,4]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm10 = xmm0[0,1,2,3],xmm10[4],xmm0[5,6],xmm10[7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $107, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6],ymm0[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm18, %ymm0
@@ -12638,10 +12638,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm31
; AVX512DQ-FCP-NEXT: vpermt2d %ymm0, %ymm21, %ymm3
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm24
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $189, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm6 = mem[0],ymm0[1],mem[2,3,4,5],ymm0[6],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $219, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1],ymm0[2],mem[3,4],ymm0[5],mem[6,7]
; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm3
@@ -12652,14 +12652,14 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm6, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm16
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = (ymm3 & ymm20) | ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $36, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm2 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm1
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm2, %xmm8
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm8[2,2,2,2,4,5,6,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2,3],xmm4[4],xmm1[5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm4 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm4, %xmm7
@@ -12670,10 +12670,10 @@ define void @load_i16_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm1, %zmm4, %zmm25
; AVX512DQ-FCP-NEXT: vinserti32x8 $0, %ymm3, %zmm0, %zmm25 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $66, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm4 = ymm0[0],mem[1],ymm0[2,3,4,5],mem[6],ymm0[7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd $109, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm1 = mem[0],ymm0[1],mem[2,3],ymm0[4],mem[5,6],ymm0[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm1, %xmm3
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
index 4129b6d9ed1ca..41c8935535950 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
@@ -3223,8 +3223,8 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm6[4],xmm5[5],xmm6[6],xmm5[7]
; AVX512-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vpor %ymm4, %ymm5, %ymm4
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm5
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm5
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm6[0,1,2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,4,6,4,7]
; AVX512-NEXT: vextracti128 $1, %ymm7, %xmm7
@@ -3232,7 +3232,7 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,7,7]
; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm9[0,1,0,2]
; AVX512-NEXT: vpshufd {{.*#+}} ymm10 = ymm8[0,1,2,1,4,5,6,5]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
@@ -3324,8 +3324,8 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1],xmm11[2],xmm12[3],xmm11[4],xmm12[5,6,7]
; AVX512-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[8,9,8,9,6,7,4,5,2,3,u,u,u,u,u,u]
; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm12
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm13
+; AVX512-NEXT: vmovdqa64 208(%rdi), %xmm12
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm13
; AVX512-NEXT: vpblendw {{.*#+}} xmm14 = xmm13[0,1,2,3,4,5],xmm12[6],xmm13[7]
; AVX512-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[0,1,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,4,7,6]
@@ -3395,7 +3395,7 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermd %zmm0, %zmm3, %zmm3
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,6,10,13,3,6,10,13]
; AVX512-FCP-NEXT: vpermd %zmm2, %zmm4, %zmm4
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm5
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm5[0,1,0,2]
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27]
; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm14, %ymm6
@@ -3410,8 +3410,8 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm13 = ymm13[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %ymm3, %ymm13, %ymm3
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm12[5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm12
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm13[0,1],ymm12[2],ymm13[3,4,5],ymm12[6],ymm13[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm15, %xmm1
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm15[0,1,2,3],xmm1[4],xmm15[5],xmm1[6],xmm15[7]
@@ -3542,8 +3542,8 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm6[4],xmm5[5],xmm6[6],xmm5[7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpor %ymm4, %ymm5, %ymm4
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm5
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm6[0,1,2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,4,6,4,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm7, %xmm7
@@ -3551,7 +3551,7 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,7,7]
; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm9[0,1,0,2]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm10 = ymm8[0,1,2,1,4,5,6,5]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
@@ -3643,8 +3643,8 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1],xmm11[2],xmm12[3],xmm11[4],xmm12[5,6,7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[8,9,8,9,6,7,4,5,2,3,u,u,u,u,u,u]
; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm12
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm13
+; AVX512DQ-NEXT: vmovdqa64 208(%rdi), %xmm12
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm13
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm14 = xmm13[0,1,2,3,4,5],xmm12[6],xmm13[7]
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[0,1,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,4,7,6]
@@ -3714,7 +3714,7 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermd %zmm0, %zmm3, %zmm3
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,6,10,13,3,6,10,13]
; AVX512DQ-FCP-NEXT: vpermd %zmm2, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm5[0,1,0,2]
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27]
; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm14, %ymm6
@@ -3729,8 +3729,8 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm13 = ymm13[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %ymm3, %ymm13, %ymm3
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm12[5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm12
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm13[0,1],ymm12[2],ymm13[3,4,5],ymm12[6],ymm13[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm15, %xmm1
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm15[0,1,2,3],xmm1[4],xmm15[5],xmm1[6],xmm15[7]
@@ -6767,8 +6767,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vporq %ymm0, %ymm1, %ymm21
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm9
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm10
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm9
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm10
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm9[3],ymm10[4,5],ymm9[6],ymm10[7]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,6,4,7]
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
@@ -6776,22 +6776,22 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm2
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm0
; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm1[0,1,2,1,4,5,6,5]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm3[7]
; AVX512-NEXT: vpbroadcastw 252(%rdi), %xmm3
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm13
+; AVX512-NEXT: vmovdqa64 224(%rdi), %xmm13
; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm13[0,1,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
; AVX512-NEXT: movw $992, %ax # imm = 0x3E0
; AVX512-NEXT: kmovw %eax, %k1
; AVX512-NEXT: vinserti32x4 $2, %xmm3, %zmm2, %zmm21 {%k1}
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm5
-; AVX512-NEXT: vmovdqa 288(%rdi), %ymm6
-; AVX512-NEXT: vmovdqa 240(%rdi), %xmm14
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm5
+; AVX512-NEXT: vmovdqa64 288(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 240(%rdi), %xmm14
; AVX512-NEXT: vmovdqa 80(%rdi), %xmm2
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm11[0,1],ymm12[2],ymm11[3,4],ymm12[5],ymm11[6,7]
; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6,7,8,9,10],ymm2[11],ymm3[12,13,14,15]
@@ -6867,8 +6867,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,8,9,6,7,4,5,2,3,u,u,u,u,u,u]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm2
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 208(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm1
; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,5],xmm2[6],xmm1[7]
; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,7,6]
@@ -6882,8 +6882,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm3
; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm0[3],xmm3[4],xmm0[5],xmm3[6,7]
-; AVX512-NEXT: vmovdqa 352(%rdi), %ymm15
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 352(%rdi), %ymm15
+; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm4
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm4[0,1,2],ymm15[3],ymm4[4,5],ymm15[6],ymm4[7]
; AVX512-NEXT: vpermq {{.*#+}} ymm13 = ymm8[2,3,0,1]
; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm13[6],ymm8[7,8,9,10,11,12,13],ymm13[14],ymm8[15]
@@ -6892,8 +6892,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[0,1,14,15,12,13,10,11,4,5,6,7,8,9,2,3,16,17,30,31,28,29,26,27,20,21,22,23,24,25,18,19]
; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm8[0,1,2],xmm3[3,4,5,6],xmm8[7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm8[4,5,6,7]
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm8
-; AVX512-NEXT: vmovdqa 416(%rdi), %ymm13
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 416(%rdi), %ymm13
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1],ymm8[2],ymm13[3,4],ymm8[5],ymm13[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm7
; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
@@ -7144,14 +7144,14 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm11[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vporq %ymm8, %ymm11, %ymm23
; AVX512-FCP-NEXT: vpbroadcastw 252(%rdi), %xmm8
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %xmm12
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %xmm12
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm11 = xmm12[u,u,u,u,u,u,u,u,0,1,14,15,12,13,14,15]
; AVX512-FCP-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm11[2],xmm8[2],xmm11[3],xmm8[3]
; AVX512-FCP-NEXT: movw $992, %ax # imm = 0x3E0
; AVX512-FCP-NEXT: kmovw %eax, %k1
; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm8, %zmm7, %zmm23 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm7
-; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %ymm8
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm11, %xmm13
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3],xmm13[4],xmm11[5],xmm13[6,7]
@@ -7163,7 +7163,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm0
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm29
-; AVX512-FCP-NEXT: vmovdqa 240(%rdi), %xmm15
+; AVX512-FCP-NEXT: vmovdqa64 240(%rdi), %xmm15
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm6[3],ymm5[4,5],ymm6[6],ymm5[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm11
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm11[0],xmm0[1],xmm11[2,3,4,5],xmm0[6],xmm11[7]
@@ -7171,8 +7171,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm0, %ymm0
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[0,1,6,7,8,9,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vporq %ymm10, %ymm0, %ymm22
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm11
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm10
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm10
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm11[2],ymm10[3,4,5],ymm11[6],ymm10[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm14
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm14[4],xmm0[5],xmm14[6],xmm0[7]
@@ -7246,8 +7246,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2d %ymm9, %ymm26, %ymm4
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm21
-; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 416(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm4
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm4[0,1],ymm3[2],ymm4[3,4,5],ymm3[6],ymm4[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm9, %xmm14
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2,3],xmm14[4],xmm9[5],xmm14[6],xmm9[7]
@@ -7411,8 +7411,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vporq %ymm0, %ymm1, %ymm18
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm8
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm9
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm9
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2],ymm8[3],ymm9[4,5],ymm8[6],ymm9[7]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,6,4,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm0
@@ -7420,22 +7420,22 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7]
; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm2
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm0
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm1[0,1,2,1,4,5,6,5]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm3[7]
; AVX512DQ-NEXT: vpbroadcastw 252(%rdi), %xmm3
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm13
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %xmm13
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm13[0,1,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
; AVX512DQ-NEXT: movw $992, %ax # imm = 0x3E0
; AVX512DQ-NEXT: kmovw %eax, %k1
; AVX512DQ-NEXT: vinserti32x4 $2, %xmm3, %zmm2, %zmm18 {%k1}
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm6
-; AVX512DQ-NEXT: vmovdqa 288(%rdi), %ymm5
-; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm14
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 288(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 240(%rdi), %xmm14
; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm10[0,1],ymm4[2],ymm10[3,4],ymm4[5],ymm10[6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6,7,8,9,10],ymm2[11],ymm3[12,13,14,15]
@@ -7509,8 +7509,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[8,9,8,9,6,7,4,5,2,3,u,u,u,u,u,u]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm2
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 208(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1,2,3,4,5],xmm2[6],xmm7[7]
; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm29
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
@@ -7526,8 +7526,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm6[2,3],ymm5[4,5],ymm6[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4],xmm0[5],xmm1[6,7]
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm14
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm14[0,1,2],ymm1[3],ymm14[4,5],ymm1[6],ymm14[7]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6],ymm2[7,8,9,10,11,12,13],ymm3[14],ymm2[15]
@@ -7536,8 +7536,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[0,1,14,15,12,13,10,11,4,5,6,7,8,9,2,3,16,17,30,31,28,29,26,27,20,21,22,23,24,25,18,19]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4,5,6],xmm2[7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm5
-; AVX512DQ-NEXT: vmovdqa 416(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 416(%rdi), %ymm13
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm13[0,1],ymm5[2],ymm13[3,4],ymm5[5],ymm13[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
@@ -7789,14 +7789,14 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm11 = ymm11[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vporq %ymm8, %ymm11, %ymm23
; AVX512DQ-FCP-NEXT: vpbroadcastw 252(%rdi), %xmm8
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %xmm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %xmm12
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm11 = xmm12[u,u,u,u,u,u,u,u,0,1,14,15,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpunpckhdq {{.*#+}} xmm8 = xmm11[2],xmm8[2],xmm11[3],xmm8[3]
; AVX512DQ-FCP-NEXT: movw $992, %ax # imm = 0x3E0
; AVX512DQ-FCP-NEXT: kmovw %eax, %k1
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm8, %zmm7, %zmm23 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm7
-; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm11, %xmm13
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3],xmm13[4],xmm11[5],xmm13[6,7]
@@ -7808,7 +7808,7 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm29
-; AVX512DQ-FCP-NEXT: vmovdqa 240(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 240(%rdi), %xmm15
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm6[3],ymm5[4,5],ymm6[6],ymm5[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm11
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm11[0],xmm0[1],xmm11[2,3,4,5],xmm0[6],xmm11[7]
@@ -7816,8 +7816,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[0,1,6,7,8,9,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vporq %ymm10, %ymm0, %ymm22
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm10
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm11[2],ymm10[3,4,5],ymm11[6],ymm10[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm14
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm14[4],xmm0[5],xmm14[6],xmm0[7]
@@ -7891,8 +7891,8 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2d %ymm9, %ymm26, %ymm4
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm21
-; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 416(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm4[0,1],ymm3[2],ymm4[3,4,5],ymm3[6],ymm4[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm9, %xmm14
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1,2,3],xmm14[4],xmm9[5],xmm14[6],xmm9[7]
@@ -13974,8 +13974,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-LABEL: load_i16_stride7_vf64:
; AVX512: # %bb.0:
; AVX512-NEXT: subq $1864, %rsp # imm = 0x748
-; AVX512-NEXT: vmovdqa 480(%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 480(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm1[2],ymm2[3,4,5],ymm1[6],ymm2[7]
; AVX512-NEXT: vmovdqa64 %ymm2, %ymm16
; AVX512-NEXT: vmovdqa64 %ymm1, %ymm18
@@ -13983,8 +13983,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,14,15,12,13,10,11,8,9,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512-NEXT: vmovdqa 512(%rdi), %ymm13
-; AVX512-NEXT: vmovdqa 544(%rdi), %ymm10
+; AVX512-NEXT: vmovdqa64 512(%rdi), %ymm13
+; AVX512-NEXT: vmovdqa64 544(%rdi), %ymm10
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm10[0],ymm13[1],ymm10[2,3,4],ymm13[5],ymm10[6,7]
; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,2,3]
; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,128,128,128,128,6,7,12,13,2,3,16,17,30,31,128,128,128,128,128,128,128,128,128,128,128,128]
@@ -14011,16 +14011,16 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %ymm0, %ymm3, %ymm0
; AVX512-NEXT: vpor %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm9
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm9
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm11
; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm21
; AVX512-NEXT: vpermq {{.*#+}} ymm24 = ymm21[0,1,0,2]
; AVX512-NEXT: vpbroadcastw 252(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm4
+; AVX512-NEXT: vmovdqa64 224(%rdi), %xmm4
; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[0,1,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7]
; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 80(%rdi), %xmm0
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm15[0,1],ymm14[2],ymm15[3,4],ymm14[5],ymm15[6,7]
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5,6,7,8,9,10],ymm0[11],ymm3[12,13,14,15]
@@ -14045,14 +14045,14 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufd {{.*#+}} ymm7 = ymm24[0,1,1,3,4,5,5,7]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14]
; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm0[0,1,2,3,4,5,6],ymm7[7]
-; AVX512-NEXT: vmovdqa 240(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 240(%rdi), %xmm0
; AVX512-NEXT: vpblendw {{.*#+}} xmm8 = xmm0[0],xmm4[1],xmm0[2,3,4,5,6,7]
; AVX512-NEXT: vmovdqa64 %xmm0, %xmm23
; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,3,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,0,3,3,4,5,6,7]
; AVX512-NEXT: vinserti32x4 $2, %xmm8, %zmm7, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 528(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 528(%rdi), %xmm7
; AVX512-NEXT: vmovdqa %ymm10, %ymm12
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm10[0,1],ymm13[2],ymm10[3,4],ymm13[5],ymm10[6,7]
; AVX512-NEXT: vpblendw {{.*#+}} ymm7 = ymm8[0,1,2],ymm7[3],ymm8[4,5,6,7,8,9,10],ymm7[11],ymm8[12,13,14,15]
@@ -14065,8 +14065,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %ymm6, %ymm7, %ymm6
; AVX512-NEXT: vpor %ymm3, %ymm6, %ymm3
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 576(%rdi), %ymm0
-; AVX512-NEXT: vmovdqa 608(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 608(%rdi), %ymm6
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0,1],ymm6[2],ymm0[3,4,5],ymm6[6],ymm0[7]
; AVX512-NEXT: vmovdqa64 %ymm6, %ymm26
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm30
@@ -14079,7 +14079,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufd {{.*#+}} ymm5 = ymm18[0,1,1,3,4,5,5,7]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14]
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm3[0,1,2,3,4,5,6],ymm5[7]
-; AVX512-NEXT: vmovdqa 688(%rdi), %xmm3
+; AVX512-NEXT: vmovdqa64 688(%rdi), %xmm3
; AVX512-NEXT: vmovdqa64 %xmm22, %xmm2
; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm2[1],xmm3[2,3,4,5,6,7]
; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,3,2,3]
@@ -14087,8 +14087,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti32x4 $2, %xmm6, %zmm5, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm15[0,1],ymm14[2,3],ymm15[4,5],ymm14[6,7]
-; AVX512-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa64 %ymm14, %ymm22
; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm6
; AVX512-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4],ymm5[5,6,7,8,9,10,11],ymm6[12],ymm5[13,14,15]
@@ -14119,20 +14119,20 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5,6],ymm9[7]
; AVX512-NEXT: vmovdqa64 %xmm23, %xmm0
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm0[2,1,2,3]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,3,2,3,4,5,6,7]
; AVX512-NEXT: vinserti32x4 $2, %xmm9, %zmm8, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm12[0,1],ymm13[2,3],ymm12[4,5],ymm13[6,7]
-; AVX512-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vextracti128 $1, %ymm8, %xmm9
; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4],ymm8[5,6,7,8,9,10,11],ymm9[12],ymm8[13,14,15]
; AVX512-NEXT: vpshufb %ymm7, %ymm8, %ymm7
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5,6,7]
-; AVX512-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vextracti128 $1, %ymm8, %xmm9
; AVX512-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm9[1],xmm8[2,3,4,5],xmm9[6],xmm8[7]
; AVX512-NEXT: vpshufb %ymm5, %ymm8, %ymm5
@@ -14216,8 +14216,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [8,9,8,9,6,7,4,5,2,3,2,3,2,3,2,3]
; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm5
; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm6
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 208(%rdi), %xmm6
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512-NEXT: vpblendw {{.*#+}} xmm8 = xmm7[0,1,2,3,4,5],xmm6[6],xmm7[7]
; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,7,6]
@@ -14234,8 +14234,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3],xmm5[4],xmm8[5,6,7]
; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm1
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm8
-; AVX512-NEXT: vmovdqa 656(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa 640(%rdi), %xmm5
+; AVX512-NEXT: vmovdqa64 656(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 640(%rdi), %xmm5
; AVX512-NEXT: vpblendw {{.*#+}} xmm9 = xmm5[0,1,2,3,4,5],xmm1[6],xmm5[7]
; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,1,0,3]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,4,7,6]
@@ -14293,13 +14293,13 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]
; AVX512-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512-NEXT: vmovdqa 736(%rdi), %ymm5
+; AVX512-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512-NEXT: vmovdqa64 736(%rdi), %ymm5
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm7[2,3],ymm5[4,5],ymm7[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4],xmm0[5],xmm1[6,7]
-; AVX512-NEXT: vmovdqa 800(%rdi), %ymm3
-; AVX512-NEXT: vmovdqa 768(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 800(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 768(%rdi), %ymm4
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2],ymm3[3],ymm4[4,5],ymm3[6],ymm4[7]
; AVX512-NEXT: vmovdqa64 %ymm3, %ymm20
; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
@@ -14310,8 +14310,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %ymm1, %ymm2, %ymm2
; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4,5,6],xmm2[7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT: vmovdqa 832(%rdi), %ymm3
-; AVX512-NEXT: vmovdqa 864(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 832(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 864(%rdi), %ymm8
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1],ymm3[2],ymm8[3,4],ymm3[5],ymm8[6,7]
; AVX512-NEXT: vmovdqa64 %ymm3, %ymm21
; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm3
@@ -14337,14 +14337,14 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 16-byte Folded Reload
; AVX512-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm2
-; AVX512-NEXT: vmovdqa 288(%rdi), %ymm15
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 288(%rdi), %ymm15
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1],ymm2[2,3],ymm15[4,5],ymm2[6,7]
; AVX512-NEXT: vmovdqa64 %ymm2, %ymm25
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3],xmm2[4],xmm0[5],xmm2[6,7]
-; AVX512-NEXT: vmovdqa 352(%rdi), %ymm14
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm10
+; AVX512-NEXT: vmovdqa64 352(%rdi), %ymm14
+; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm10
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm14[3],ymm10[4,5],ymm14[6],ymm10[7]
; AVX512-NEXT: vpermq {{.*#+}} ymm13 = ymm0[2,3,0,1]
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm13[6],ymm0[7,8,9,10,11,12,13],ymm13[14],ymm0[15]
@@ -14353,8 +14353,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7]
; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2],xmm1[3,4,5,6],xmm0[7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 416(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 416(%rdi), %ymm3
; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm3[0,1],ymm1[2],ymm3[3,4],ymm1[5],ymm3[6,7]
; AVX512-NEXT: vextracti128 $1, %ymm13, %xmm12
; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,1,3,1,4,5,6,7]
@@ -14363,7 +14363,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
; AVX512-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm12[6,7]
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa64 %ymm25, %ymm2
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1,2],ymm2[3],ymm15[4,5],ymm2[6],ymm15[7]
; AVX512-NEXT: vmovdqa64 %ymm15, %ymm24
@@ -14388,7 +14388,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm11
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm11[6,7]
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm7[3],ymm5[4,5],ymm7[6],ymm5[7]
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm11
; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm11[0,1,2,3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
@@ -14434,7 +14434,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0],ymm4[1],ymm6[2,3,4],ymm4[5],ymm6[6,7]
; AVX512-NEXT: vpshufb %ymm1, %ymm0, %ymm1
; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
@@ -14452,7 +14452,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm11[4],xmm0[5],xmm11[5],xmm0[6],xmm11[6],xmm0[7],xmm11[7]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa64 %ymm22, %ymm0
; AVX512-NEXT: vmovdqa64 %ymm23, %ymm1
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6,7]
@@ -14531,7 +14531,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2],ymm11[3,4,5,6,7],ymm1[8,9,10],ymm11[11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
; AVX512-NEXT: vmovdqa64 %ymm16, %ymm0
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5],ymm1[6],ymm0[7]
@@ -14540,7 +14540,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufb %ymm3, %ymm0, %ymm11
; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6],ymm11[7,8,9,10,11,12,13],ymm0[14],ymm11[15]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm20 # 32-byte Reload
; AVX512-NEXT: vmovdqa64 %ymm20, %ymm1
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm6[0,1],ymm1[2],ymm6[3,4],ymm1[5],ymm6[6,7]
@@ -14578,15 +14578,15 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpor %ymm11, %ymm13, %ymm11
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1],ymm10[2],ymm15[3,4,5],ymm10[6],ymm15[7]
; AVX512-NEXT: vmovdqa64 %ymm10, %ymm29
; AVX512-NEXT: vpshufb %ymm3, %ymm0, %ymm3
; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6],ymm3[7,8,9,10,11,12,13],ymm0[14],ymm3[15]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm13[0,1],ymm10[2],ymm13[3,4],ymm10[5],ymm13[6,7]
; AVX512-NEXT: vmovdqa64 %ymm10, %ymm30
; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm11
@@ -14877,8 +14877,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [8,1,12,5,12,5,14,15]
; AVX512-FCP-NEXT: vpermd %zmm21, %zmm3, %zmm2
; AVX512-FCP-NEXT: vpermd %zmm11, %zmm3, %zmm4
-; AVX512-FCP-NEXT: vmovdqa 480(%rdi), %ymm12
-; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 480(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %ymm6
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1],ymm12[2],ymm6[3,4,5],ymm12[6],ymm6[7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm17
; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm5
@@ -14888,7 +14888,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [128,128,128,128,128,128,128,128,128,128,6,7,12,13,2,3,16,17,30,31,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
; AVX512-FCP-NEXT: vporq %ymm4, %ymm6, %ymm16
-; AVX512-FCP-NEXT: vmovdqa 672(%rdi), %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 672(%rdi), %xmm7
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,2,3,4,5,6,7,0,1,14,15,12,13,14,15]
; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm7, %xmm6
; AVX512-FCP-NEXT: vmovdqa64 %xmm7, %xmm19
@@ -14906,12 +14906,12 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm6[4],xmm5[5],xmm6[6],xmm5[7]
; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm5, %ymm3
; AVX512-FCP-NEXT: vporq %ymm2, %ymm3, %ymm25
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %xmm5
; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm5, %xmm2
; AVX512-FCP-NEXT: vpbroadcastw 252(%rdi), %xmm3
; AVX512-FCP-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 240(%rdi), %xmm15
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 240(%rdi), %xmm15
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,0,1,6,7,8,9,18,19,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1,2],ymm7[3],ymm8[4,5],ymm7[6],ymm8[7]
@@ -14923,8 +14923,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm4
; AVX512-FCP-NEXT: vpor %ymm0, %ymm4, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm10
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm10
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1],ymm10[2],ymm13[3,4,5],ymm10[6],ymm13[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm4
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
@@ -14952,8 +14952,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
; AVX512-FCP-NEXT: vpor %ymm1, %ymm3, %ymm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 608(%rdi), %ymm11
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 608(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm11[2],ymm2[3,4,5],ymm11[6],ymm2[7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
; AVX512-FCP-NEXT: vextracti128 $1, %ymm1, %xmm2
@@ -14964,7 +14964,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm2
; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm28
; AVX512-FCP-NEXT: vpermt2d %ymm2, %ymm23, %ymm1
-; AVX512-FCP-NEXT: vmovdqa 688(%rdi), %xmm8
+; AVX512-FCP-NEXT: vmovdqa64 688(%rdi), %xmm8
; AVX512-FCP-NEXT: vmovdqa64 %xmm19, %xmm12
; AVX512-FCP-NEXT: vmovdqa64 %xmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0],xmm12[1],xmm8[2,3,4,5,6,7]
@@ -15130,8 +15130,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]
; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm4, %zmm3, %zmm3
; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 704(%rdi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 736(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 704(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 736(%rdi), %ymm8
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1],ymm4[2,3],ymm8[4,5],ymm4[6,7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm8, %ymm20
; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm19
@@ -15185,8 +15185,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm0
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm10
-; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm10
+; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %ymm4
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm10[2,3],ymm4[4,5],ymm10[6,7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm14
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0,1,2],xmm0[3],xmm14[4],xmm0[5],xmm14[6,7]
@@ -15278,8 +15278,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm1
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm7
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm9
+; AVX512-FCP-NEXT: vmovdqa64 416(%rdi), %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm9
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1],ymm7[2],ymm9[3,4,5],ymm7[6],ymm9[7]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm6
; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm6[4],xmm0[5],xmm6[6],xmm0[7]
@@ -15303,8 +15303,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: # zmm1 = zmm1 ^ (zmm28 & (zmm1 ^ mem))
; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm29, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 864(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 832(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 864(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 832(%rdi), %ymm5
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm1[2],ymm5[3,4,5],ymm1[6],ymm5[7]
; AVX512-FCP-NEXT: vmovdqa64 %ymm5, %ymm22
; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm21
@@ -15583,8 +15583,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-LABEL: load_i16_stride7_vf64:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: subq $1592, %rsp # imm = 0x638
-; AVX512DQ-NEXT: vmovdqa 480(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 480(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm1[2],ymm2[3,4,5],ymm1[6],ymm2[7]
; AVX512DQ-NEXT: vmovdqa64 %ymm2, %ymm30
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm29
@@ -15592,8 +15592,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [0,1,14,15,12,13,10,11,8,9,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %ymm12
-; AVX512DQ-NEXT: vmovdqa 544(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %ymm12
+; AVX512DQ-NEXT: vmovdqa64 544(%rdi), %ymm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0],ymm12[1],ymm4[2,3,4],ymm12[5],ymm4[6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm4, %ymm17
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,2,3]
@@ -15601,11 +15601,11 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb %ymm1, %ymm3, %ymm3
; AVX512DQ-NEXT: vporq %ymm3, %ymm2, %ymm26
; AVX512DQ-NEXT: vpbroadcastw 700(%rdi), %xmm3
-; AVX512DQ-NEXT: vmovdqa 672(%rdi), %xmm14
+; AVX512DQ-NEXT: vmovdqa64 672(%rdi), %xmm14
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm4 = xmm14[0,1,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,7]
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm6
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm11
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm8
@@ -15619,12 +15619,12 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb %ymm0, %ymm3, %ymm0
; AVX512DQ-NEXT: vpor %ymm1, %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm9
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm9
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm13
; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm18
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm28 = ymm18[0,1,0,2]
; AVX512DQ-NEXT: vpbroadcastw 252(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm15
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %xmm15
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm15[0,1,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7]
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]
@@ -15654,13 +15654,13 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm7 = ymm28[0,1,1,3,4,5,5,7]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm0[0,1,2,3,4,5,6],ymm7[7]
-; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm10
+; AVX512DQ-NEXT: vmovdqa64 240(%rdi), %xmm10
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0],xmm15[1],xmm10[2,3,4,5,6,7]
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,3,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,0,3,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti32x4 $2, %xmm8, %zmm7, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 528(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 528(%rdi), %xmm7
; AVX512DQ-NEXT: vmovdqa64 %ymm17, %ymm11
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm11[0,1],ymm12[2],ymm11[3,4],ymm12[5],ymm11[6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm7 = ymm8[0,1,2],ymm7[3],ymm8[4,5,6,7,8,9,10],ymm7[11],ymm8[12,13,14,15]
@@ -15673,8 +15673,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm7, %ymm6
; AVX512DQ-NEXT: vpor %ymm3, %ymm6, %ymm3
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %ymm2
-; AVX512DQ-NEXT: vmovdqa 608(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 608(%rdi), %ymm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1],ymm1[2],ymm2[3,4,5],ymm1[6],ymm2[7]
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm23
; AVX512DQ-NEXT: vextracti128 $1, %ymm3, %xmm6
@@ -15686,7 +15686,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm6 = ymm20[0,1,1,3,4,5,5,7]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm6[7]
-; AVX512DQ-NEXT: vmovdqa 688(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 688(%rdi), %xmm0
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0],xmm14[1],xmm0[2,3,4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %xmm14, %xmm17
; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm25
@@ -15729,7 +15729,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3]
; AVX512DQ-NEXT: vmovdqa64 %xmm10, %xmm21
; AVX512DQ-NEXT: vmovdqa64 %xmm15, %xmm24
-; AVX512DQ-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm3[2,1,2,3]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,3,2,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti32x4 $2, %xmm9, %zmm8, %zmm3
@@ -15828,8 +15828,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm1 = [8,9,8,9,6,7,4,5,2,3,2,3,2,3,2,3]
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm3
; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm5
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 208(%rdi), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm6
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5],xmm5[6],xmm6[7]
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,7,6]
@@ -15847,8 +15847,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0,1],xmm3[2],xmm7[3],xmm3[4],xmm7[5,6,7]
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm3, %xmm1
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm7
-; AVX512DQ-NEXT: vmovdqa 656(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa 640(%rdi), %xmm3
+; AVX512DQ-NEXT: vmovdqa64 656(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %xmm3
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm8 = xmm3[0,1,2,3,4,5],xmm1[6],xmm3[7]
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,0,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,7,6]
@@ -15906,13 +15906,13 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti32x4 $2, %xmm1, %zmm0, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512DQ-NEXT: vmovdqa 736(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512DQ-NEXT: vmovdqa64 736(%rdi), %ymm6
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm7[2,3],ymm6[4,5],ymm7[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4],xmm0[5],xmm1[6,7]
-; AVX512DQ-NEXT: vmovdqa 800(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqa 768(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 800(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqa64 768(%rdi), %ymm5
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7]
; AVX512DQ-NEXT: vmovdqa64 %ymm3, %ymm24
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
@@ -15923,8 +15923,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufb %ymm1, %ymm2, %ymm2
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4,5,6],xmm2[7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 832(%rdi), %ymm9
-; AVX512DQ-NEXT: vmovdqa 864(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 832(%rdi), %ymm9
+; AVX512DQ-NEXT: vmovdqa64 864(%rdi), %ymm8
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm8[0,1],ymm9[2],ymm8[3,4],ymm9[5],ymm8[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
@@ -15949,13 +15949,13 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-NEXT: vinserti32x4 $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 16-byte Folded Reload
; AVX512DQ-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm11
-; AVX512DQ-NEXT: vmovdqa 288(%rdi), %ymm15
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm11
+; AVX512DQ-NEXT: vmovdqa64 288(%rdi), %ymm15
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1],ymm11[2,3],ymm15[4,5],ymm11[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3],xmm2[4],xmm0[5],xmm2[6,7]
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %ymm14
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm3
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm3[0,1,2],ymm14[3],ymm3[4,5],ymm14[6],ymm3[7]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm12 = ymm10[2,3,0,1]
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm12[6],ymm10[7,8,9,10,11,12,13],ymm12[14],ymm10[15]
@@ -15964,8 +15964,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0,1,2],xmm2[3,4,5,6],xmm1[7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 416(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 416(%rdi), %ymm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm12 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]
; AVX512DQ-NEXT: vextracti128 $1, %ymm12, %xmm13
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[0,1,3,1,4,5,6,7]
@@ -16461,8 +16461,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [8,1,12,5,12,5,14,15]
; AVX512DQ-FCP-NEXT: vpermd %zmm21, %zmm3, %zmm2
; AVX512DQ-FCP-NEXT: vpermd %zmm11, %zmm3, %zmm4
-; AVX512DQ-FCP-NEXT: vmovdqa 480(%rdi), %ymm12
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 480(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1],ymm12[2],ymm6[3,4,5],ymm12[6],ymm6[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm17
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm5
@@ -16472,7 +16472,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [128,128,128,128,128,128,128,128,128,128,6,7,12,13,2,3,16,17,30,31,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
; AVX512DQ-FCP-NEXT: vporq %ymm4, %ymm6, %ymm16
-; AVX512DQ-FCP-NEXT: vmovdqa 672(%rdi), %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 672(%rdi), %xmm7
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,2,3,4,5,6,7,0,1,14,15,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm7, %xmm6
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm7, %xmm19
@@ -16490,12 +16490,12 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm6[4],xmm5[5],xmm6[6],xmm5[7]
; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm5, %ymm3
; AVX512DQ-FCP-NEXT: vporq %ymm2, %ymm3, %ymm25
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm5, %xmm2
; AVX512DQ-FCP-NEXT: vpbroadcastw 252(%rdi), %xmm3
; AVX512DQ-FCP-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 240(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 240(%rdi), %xmm15
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,0,1,6,7,8,9,18,19,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1,2],ymm7[3],ymm8[4,5],ymm7[6],ymm8[7]
@@ -16507,8 +16507,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm4
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm4, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm10
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1],ymm10[2],ymm13[3,4,5],ymm10[6],ymm13[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm4
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
@@ -16536,8 +16536,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm3, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 608(%rdi), %ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 608(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm11[2],ymm2[3,4,5],ymm11[6],ymm2[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm27
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm1, %xmm2
@@ -16548,7 +16548,7 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm28
; AVX512DQ-FCP-NEXT: vpermt2d %ymm2, %ymm23, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 688(%rdi), %xmm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 688(%rdi), %xmm8
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm19, %xmm12
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0],xmm12[1],xmm8[2,3,4,5,6,7]
@@ -16714,8 +16714,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm4, %zmm3, %zmm3
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 704(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 736(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 704(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 736(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1],ymm4[2,3],ymm8[4,5],ymm4[6,7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm8, %ymm20
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm19
@@ -16769,8 +16769,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm10
-; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm10[2,3],ymm4[4,5],ymm10[6,7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm14
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0,1,2],xmm0[3],xmm14[4],xmm0[5],xmm14[6,7]
@@ -16862,8 +16862,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm1, %xmm1
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm7
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 416(%rdi), %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm9
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1],ymm7[2],ymm9[3,4,5],ymm7[6],ymm9[7]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm6
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm6[4],xmm0[5],xmm6[6],xmm0[7]
@@ -16887,8 +16887,8 @@ define void @load_i16_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: # zmm1 = zmm1 ^ (zmm28 & (zmm1 ^ mem))
; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm29, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 864(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 832(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 864(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 832(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm1[2],ymm5[3,4,5],ymm1[6],ymm5[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, %ymm22
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm21
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-8.ll
index d276d46831341..e3d3913907511 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-8.ll
@@ -5719,14 +5719,14 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-LABEL: load_i16_stride8_vf32:
; AVX512: # %bb.0:
; AVX512-NEXT: subq $616, %rsp # imm = 0x268
-; AVX512-NEXT: vmovdqa 368(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 368(%rdi), %xmm0
; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 352(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 352(%rdi), %xmm1
; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT: vmovdqa 336(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 336(%rdi), %xmm0
; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 320(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 320(%rdi), %xmm1
; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm1 = [0,0,0,4]
@@ -5735,36 +5735,36 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2d %xmm2, %xmm1, %xmm0
; AVX512-NEXT: vmovdqa64 %xmm2, %xmm24
; AVX512-NEXT: vmovdqa %xmm1, %xmm3
-; AVX512-NEXT: vmovdqa 304(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 304(%rdi), %xmm1
; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 288(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 288(%rdi), %xmm2
; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vmovdqa 272(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 272(%rdi), %xmm1
; AVX512-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 256(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 256(%rdi), %xmm2
; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
; AVX512-NEXT: vpblendd {{.*#+}} xmm7 = xmm1[0,1],xmm0[2,3]
-; AVX512-NEXT: vmovdqa 480(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 480(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm22 = ymm0[0,1,0,2]
; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm22[0,1,2,0,4,5,6,4]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-NEXT: vmovdqa64 %ymm1, %ymm26
-; AVX512-NEXT: vmovdqa 448(%rdi), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 448(%rdi), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm23 = ymm1[0,1,0,2]
; AVX512-NEXT: vpshufd {{.*#+}} ymm2 = ymm23[0,1,2,0,4,5,6,4]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm1 = ymm2[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-NEXT: vmovdqa64 %ymm2, %ymm27
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
-; AVX512-NEXT: vmovdqa 416(%rdi), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 416(%rdi), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm28 = ymm1[0,1,0,2]
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm30 = ymm1[0,1,0,2]
; AVX512-NEXT: vpshufd {{.*#+}} ymm12 = ymm28[0,2,2,3,4,6,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm12[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
@@ -5802,11 +5802,11 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]
; AVX512-NEXT: vmovdqa64 %xmm3, %xmm20
; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm1[0,1],xmm0[2,3]
-; AVX512-NEXT: vmovdqa 224(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm31 = ymm0[0,1,0,2]
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm18 = ymm0[0,1,0,2]
; AVX512-NEXT: vpshufd {{.*#+}} ymm6 = ymm31[0,1,2,0,4,5,6,4]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm10 = ymm6[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
@@ -5815,8 +5815,8 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm14[0,1,2,3,4,5,6],ymm10[7]
; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm25
; AVX512-NEXT: vpermq {{.*#+}} ymm17 = ymm25[0,1,0,2]
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm16 = ymm0[0,1,0,2]
; AVX512-NEXT: vpshufd {{.*#+}} ymm2 = ymm17[0,2,2,3,4,6,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} ymm0 = ymm2[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
@@ -6104,14 +6104,14 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-LABEL: load_i16_stride8_vf32:
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: subq $616, %rsp # imm = 0x268
-; AVX512-FCP-NEXT: vmovdqa 368(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 368(%rdi), %xmm0
; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %xmm1
; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT: vmovdqa 336(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 336(%rdi), %xmm0
; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %xmm1
; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} xmm1 = [0,0,0,4]
@@ -6120,36 +6120,36 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2d %xmm2, %xmm1, %xmm0
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm22
; AVX512-FCP-NEXT: vmovdqa %xmm1, %xmm2
-; AVX512-FCP-NEXT: vmovdqa 304(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 304(%rdi), %xmm1
; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %xmm3
+; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %xmm3
; AVX512-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-FCP-NEXT: vmovdqa 272(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 272(%rdi), %xmm1
; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %xmm3
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %xmm3
; AVX512-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm13[0],xmm4[0],xmm13[1],xmm4[1]
; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm24
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm1[0,1],xmm0[2,3]
-; AVX512-FCP-NEXT: vmovdqa 480(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 480(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm21 = ymm0[0,1,0,2]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm21[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm25
-; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm27 = ymm1[0,1,0,2]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm12 = ymm27[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm12[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm0[7]
-; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 416(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm28 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm30 = ymm0[0,1,0,2]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm9 = ymm28[0,2,2,3,4,6,6,7]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm5 = ymm9[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
@@ -6186,19 +6186,19 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm19
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm31 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm17 = ymm0[0,1,0,2]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm6 = ymm31[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm11 = ymm6[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm14 = ymm17[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm4 = ymm14[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm11[7]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm16 = ymm0[0,1,0,2]
; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm26
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm20 = ymm26[0,1,0,2]
@@ -6482,14 +6482,14 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-LABEL: load_i16_stride8_vf32:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: subq $616, %rsp # imm = 0x268
-; AVX512DQ-NEXT: vmovdqa 368(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 368(%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %xmm1
; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-NEXT: vmovdqa 336(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 336(%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %xmm1
; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} xmm1 = [0,0,0,4]
@@ -6498,36 +6498,36 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2d %xmm2, %xmm1, %xmm0
; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm24
; AVX512DQ-NEXT: vmovdqa %xmm1, %xmm3
-; AVX512DQ-NEXT: vmovdqa 304(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 304(%rdi), %xmm1
; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 288(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 288(%rdi), %xmm2
; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vmovdqa 272(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 272(%rdi), %xmm1
; AVX512DQ-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %xmm2
; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm15[0],xmm5[0],xmm15[1],xmm5[1]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm7 = xmm1[0,1],xmm0[2,3]
-; AVX512DQ-NEXT: vmovdqa 480(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 480(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm22 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm22[0,1,2,0,4,5,6,4]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm26
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm23 = ymm1[0,1,0,2]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm23[0,1,2,0,4,5,6,4]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm1 = ymm2[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-NEXT: vmovdqa64 %ymm2, %ymm27
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-NEXT: vmovdqa 416(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 416(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm28 = ymm1[0,1,0,2]
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm30 = ymm1[0,1,0,2]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm12 = ymm28[0,2,2,3,4,6,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm1 = ymm12[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
@@ -6565,11 +6565,11 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]
; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm20
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm1[0,1],xmm0[2,3]
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm31 = ymm0[0,1,0,2]
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm18 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm6 = ymm31[0,1,2,0,4,5,6,4]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm10 = ymm6[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
@@ -6578,8 +6578,8 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm14[0,1,2,3,4,5,6],ymm10[7]
; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm25
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm17 = ymm25[0,1,0,2]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm16 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm17[0,2,2,3,4,6,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm2[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
@@ -6867,14 +6867,14 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-LABEL: load_i16_stride8_vf32:
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: subq $616, %rsp # imm = 0x268
-; AVX512DQ-FCP-NEXT: vmovdqa 368(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 368(%rdi), %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-FCP-NEXT: vmovdqa 336(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 336(%rdi), %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} xmm1 = [0,0,0,4]
@@ -6883,36 +6883,36 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2d %xmm2, %xmm1, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm22
; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa 304(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 304(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-FCP-NEXT: vmovdqa 272(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 272(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm13[0],xmm4[0],xmm13[1],xmm4[1]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm24
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm1[0,1],xmm0[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa 480(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 480(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm21 = ymm0[0,1,0,2]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm21[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm25
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm27 = ymm1[0,1,0,2]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm12 = ymm27[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm12[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 416(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm28 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm30 = ymm0[0,1,0,2]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm9 = ymm28[0,2,2,3,4,6,6,7]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm5 = ymm9[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
@@ -6949,19 +6949,19 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm19
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm31 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm17 = ymm0[0,1,0,2]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm6 = ymm31[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm11 = ymm6[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm14 = ymm17[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm4 = ymm14[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm11[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm16 = ymm0[0,1,0,2]
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm26
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm20 = ymm26[0,1,0,2]
@@ -12631,42 +12631,42 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-LABEL: load_i16_stride8_vf64:
; AVX512: # %bb.0:
; AVX512-NEXT: subq $2392, %rsp # imm = 0x958
-; AVX512-NEXT: vmovdqa 368(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 352(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 368(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 352(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT: vmovdqa 336(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 320(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 336(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 320(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm8 = [0,0,0,4]
; AVX512-NEXT: vpermt2d %xmm2, %xmm8, %xmm0
; AVX512-NEXT: vmovdqa64 %xmm2, %xmm18
-; AVX512-NEXT: vmovdqa 304(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 304(%rdi), %xmm1
; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 288(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 288(%rdi), %xmm2
; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 272(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 256(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 272(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 256(%rdi), %xmm2
; AVX512-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; AVX512-NEXT: vmovdqa64 %xmm2, %xmm17
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX512-NEXT: vmovdqa 480(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 480(%rdi), %ymm1
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
-; AVX512-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -12674,19 +12674,19 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT: vmovdqa 416(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 416(%rdi), %ymm2
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm2
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufd {{.*#+}} ymm2 = ymm3[0,2,2,3,4,6,6,7]
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm5[0,2,2,3,4,6,6,7]
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
@@ -12694,37 +12694,37 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: # zmm23 = mem[0,1,2,3,0,1,2,3]
; AVX512-NEXT: vpermt2q %zmm1, %zmm23, %zmm0
; AVX512-NEXT: vmovdqa 112(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 96(%rdi), %xmm2
-; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 80(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpermt2d %xmm3, %xmm8, %xmm1
; AVX512-NEXT: vmovdqa (%rdi), %xmm2
-; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 16(%rdi), %xmm3
-; AVX512-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 32(%rdi), %xmm4
-; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 48(%rdi), %xmm5
; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
; AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT: vmovdqa 224(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm2
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -12735,11 +12735,11 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufhw {{.*#+}} ymm3 = ymm4[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-NEXT: vmovdqa64 %ymm4, %ymm16
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm3
; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm3[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm3[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -12752,39 +12752,39 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 880(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 864(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 880(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 864(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT: vmovdqa 848(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 832(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 848(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 832(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpermt2d %xmm2, %xmm8, %xmm0
; AVX512-NEXT: vmovdqa64 %xmm2, %xmm28
; AVX512-NEXT: vmovdqa %xmm8, %xmm11
-; AVX512-NEXT: vmovdqa 816(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 800(%rdi), %xmm2
-; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 816(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 800(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT: vmovdqa 784(%rdi), %xmm1
-; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 768(%rdi), %xmm2
-; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 784(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 768(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; AVX512-NEXT: vmovdqa64 %xmm2, %xmm27
; AVX512-NEXT: vmovdqa64 %xmm3, %xmm20
; AVX512-NEXT: vpblendd {{.*#+}} xmm10 = xmm1[0,1],xmm0[2,3]
-; AVX512-NEXT: vmovdqa 992(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 992(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 960(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 960(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -12793,11 +12793,11 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm2[0,1,2,0,4,5,6,4]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm1 = ymm3[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm1[0,1,2,3,4,5,6],ymm0[7]
-; AVX512-NEXT: vmovdqa 928(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 928(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 896(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 896(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -12808,39 +12808,39 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
; AVX512-NEXT: vpermt2q %zmm7, %zmm23, %zmm10
-; AVX512-NEXT: vmovdqa 624(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 608(%rdi), %xmm7
-; AVX512-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 624(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 608(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
-; AVX512-NEXT: vmovdqa 592(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 576(%rdi), %xmm7
-; AVX512-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 592(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 576(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
; AVX512-NEXT: vmovdqa %xmm0, %xmm7
; AVX512-NEXT: vmovdqa64 %xmm0, %xmm25
; AVX512-NEXT: vpermt2d %xmm8, %xmm11, %xmm7
; AVX512-NEXT: vmovdqa64 %xmm8, %xmm26
-; AVX512-NEXT: vmovdqa 560(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 544(%rdi), %xmm8
-; AVX512-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 560(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 544(%rdi), %xmm8
+; AVX512-NEXT: vmovdqa64 %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
-; AVX512-NEXT: vmovdqa 528(%rdi), %xmm0
-; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 512(%rdi), %xmm8
-; AVX512-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 528(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovdqa64 512(%rdi), %xmm8
+; AVX512-NEXT: vmovdqa64 %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]
; AVX512-NEXT: vmovdqa64 %xmm0, %xmm19
; AVX512-NEXT: vmovdqa64 %xmm9, %xmm24
; AVX512-NEXT: vpblendd {{.*#+}} xmm14 = xmm8[0,1],xmm7[2,3]
-; AVX512-NEXT: vmovdqa 736(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 736(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm7 = ymm0[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 704(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 704(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,2]
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -12851,10 +12851,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufhw {{.*#+}} ymm12 = ymm0[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm21
; AVX512-NEXT: vpblendd {{.*#+}} ymm15 = ymm12[0,1,2,3,4,5,6],ymm11[7]
-; AVX512-NEXT: vmovdqa 672(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 672(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm31 = ymm0[0,1,0,2]
-; AVX512-NEXT: vmovdqa 640(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 640(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpermq {{.*#+}} ymm29 = ymm0[0,1,0,2]
; AVX512-NEXT: vpshufd {{.*#+}} ymm11 = ymm31[0,2,2,3,4,6,6,7]
@@ -12870,7 +12870,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm17[1,1,1,1]
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm9[1],xmm0[2,3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm15[0],xmm18[0],xmm15[1],xmm18[1]
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3]
; AVX512-NEXT: vpshufhw $116, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
@@ -12896,9 +12896,9 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm10[6,7]
; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm16[1,1,1,1]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0],xmm14[1],xmm6[2,3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm22 # 16-byte Reload
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm7[0],xmm22[0],xmm7[1],xmm22[1]
; AVX512-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1],xmm10[2,3]
@@ -12910,7 +12910,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %xmm20, %xmm8
; AVX512-NEXT: vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm8[1],xmm0[2,3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm28[0],xmm6[1],xmm28[1]
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,7,5,8,9,10,11,12,13,15,13]
@@ -12947,11 +12947,11 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[0,1,1,3,4,5,5,7]
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,4,6,8,9,10,11,12,13,12,14]
; AVX512-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512-NEXT: # ymm2 = mem[0,1,1,3,4,5,5,7]
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
@@ -13038,7 +13038,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm18[2],xmm21[2],xmm18[3],xmm21[3]
; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm17[2],xmm22[2],xmm17[3],xmm22[3]
; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm14 = [3,7,0,0]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
; AVX512-NEXT: vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm5 # 16-byte Folded Reload
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm5[0,1],xmm0[2,3]
; AVX512-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -13067,7 +13067,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm20[2],xmm28[2],xmm20[3],xmm28[3]
; AVX512-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm16[2],xmm26[2],xmm16[3],xmm26[3]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
; AVX512-NEXT: vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm4 # 16-byte Folded Reload
; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
; AVX512-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -13095,10 +13095,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm7 # 16-byte Folded Reload
; AVX512-NEXT: # xmm7 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -13143,19 +13143,19 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
; AVX512-NEXT: vpermt2q %zmm1, %zmm23, %zmm0
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm2 # 16-byte Folded Reload
; AVX512-NEXT: # xmm2 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm5 # 16-byte Folded Reload
; AVX512-NEXT: # xmm5 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
; AVX512-NEXT: # xmm3 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm4 # 16-byte Folded Reload
; AVX512-NEXT: # xmm4 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512-NEXT: vmovdqa %xmm5, %xmm1
@@ -13189,16 +13189,16 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
; AVX512-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm9 # 16-byte Folded Reload
; AVX512-NEXT: # xmm9 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512-NEXT: vmovdqa %xmm2, %xmm0
@@ -13234,16 +13234,16 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4],ymm2[5],ymm5[6,7]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
; AVX512-NEXT: vpermt2q %zmm1, %zmm23, %zmm3
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
; AVX512-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm13 # 16-byte Folded Reload
; AVX512-NEXT: # xmm13 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512-NEXT: vmovdqa %xmm11, %xmm0
@@ -13548,62 +13548,62 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-LABEL: load_i16_stride8_vf64:
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: subq $2440, %rsp # imm = 0x988
-; AVX512-FCP-NEXT: vmovdqa 368(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 368(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT: vmovdqa 336(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 336(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} xmm8 = [0,0,0,4]
; AVX512-FCP-NEXT: vpermt2d %xmm2, %xmm8, %xmm0
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm16
-; AVX512-FCP-NEXT: vmovdqa 304(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 304(%rdi), %xmm1
; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %xmm2
; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 272(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %xmm2
-; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 272(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm17
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX512-FCP-NEXT: vmovdqa 480(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 480(%rdi), %ymm1
; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
-; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 416(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,1,0,2]
; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm3[0,2,2,3,4,6,6,7]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm5[0,2,2,3,4,6,6,7]
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
@@ -13611,14 +13611,14 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm26, %zmm0
; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm2
-; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-FCP-NEXT: vmovdqa 80(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-FCP-NEXT: vmovdqa %xmm2, %xmm1
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm21
@@ -13627,40 +13627,40 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm2
; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm3
-; AVX512-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm4
-; AVX512-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa 48(%rdi), %xmm5
-; AVX512-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm20
; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm19
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,2]
; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm3[0,1,2,0,4,5,6,4]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm4[0,1,2,0,4,5,6,4]
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm3[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm3[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm10 = ymm4[0,2,2,3,4,6,6,7]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm10[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm7 = ymm5[0,2,2,3,4,6,6,7]
@@ -13670,56 +13670,56 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 880(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 864(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 880(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 864(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT: vmovdqa 848(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 832(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 848(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 832(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpermt2d %xmm2, %xmm8, %xmm0
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm24
; AVX512-FCP-NEXT: vmovdqa %xmm8, %xmm11
-; AVX512-FCP-NEXT: vmovdqa 816(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 800(%rdi), %xmm2
-; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 816(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 800(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-FCP-NEXT: vmovdqa 784(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 768(%rdi), %xmm2
-; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 784(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 768(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm28
; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm23
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm1[0,1],xmm0[2,3]
-; AVX512-FCP-NEXT: vmovdqa 992(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 992(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 960(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 960(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm30
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm4 = ymm2[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm4[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm1[0,1,2,3,4,5,6],ymm0[7]
-; AVX512-FCP-NEXT: vmovdqa 928(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 928(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 896(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 896(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm1[0,2,2,3,4,6,6,7]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm9 = ymm3[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm0[0,2,2,3,4,6,6,7]
@@ -13727,55 +13727,55 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3,4],ymm9[5],ymm12[6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
; AVX512-FCP-NEXT: vpermt2q %zmm8, %zmm26, %zmm6
-; AVX512-FCP-NEXT: vmovdqa 624(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 608(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 624(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 608(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT: vmovdqa 592(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 592(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512-FCP-NEXT: vmovdqa %xmm0, %xmm9
; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm27
; AVX512-FCP-NEXT: vpermt2d %xmm8, %xmm11, %xmm9
; AVX512-FCP-NEXT: vmovdqa64 %xmm8, %xmm29
-; AVX512-FCP-NEXT: vmovdqa 560(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 544(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 560(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 544(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT: vmovdqa 528(%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 528(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm31
; AVX512-FCP-NEXT: vmovdqa64 %xmm8, %xmm25
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm12[0,1],xmm9[2,3]
-; AVX512-FCP-NEXT: vmovdqa 736(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 736(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm9 = ymm1[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm14 = ymm9[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm12 = ymm0[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm15 = ymm12[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0,1,2,3,4,5,6],ymm14[7]
-; AVX512-FCP-NEXT: vmovdqa 672(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 672(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm8 = ymm0[0,1,0,2]
-; AVX512-FCP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm14 = ymm5[0,2,2,3,4,6,6,7]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm14[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm15 = ymm8[0,2,2,3,4,6,6,7]
@@ -13864,11 +13864,11 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm1[0,1],xmm0[2,3]
; AVX512-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,4,6,8,9,10,11,12,13,12,14]
; AVX512-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm1 = mem[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
; AVX512-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
@@ -13911,7 +13911,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm0[2,3]
; AVX512-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,5,7]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5,4,6,8,9,10,11,12,13,12,14]
; AVX512-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,5,7]
@@ -14010,17 +14010,17 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512-FCP-NEXT: vmovdqa %xmm4, %xmm0
@@ -14033,15 +14033,15 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm1 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm2 = mem[0,1,1,3]
; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
@@ -14057,15 +14057,15 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm27, %zmm0
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm2 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm2 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm6 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm6 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm3 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
@@ -14079,12 +14079,12 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm2 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm3 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -14092,10 +14092,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm3 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm4 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm6 = ymm3[0,2,2,3,4,6,6,7]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm6[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm24
@@ -14106,16 +14106,16 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512-FCP-NEXT: vmovdqa %xmm2, %xmm0
@@ -14128,10 +14128,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1],xmm0[2,3]
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm2 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm23
@@ -14140,10 +14140,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm3 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm10 = ymm0[0,2,2,3,4,6,6,7]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm10[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm6 = ymm3[0,2,2,3,4,6,6,7]
@@ -14151,16 +14151,16 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm27, %zmm4
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm13 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm13 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512-FCP-NEXT: vmovdqa %xmm5, %xmm0
@@ -14172,10 +14172,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm15 = xmm1[0,1],xmm0[2,3]
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm2 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm9 = ymm0[0,1,2,0,4,5,6,4]
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm5 = ymm2[0,1,2,0,4,5,6,4]
@@ -14183,10 +14183,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm2 = mem[0,1,1,3]
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[0,2,2,3,4,6,6,7]
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm3[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,2,2,3,4,6,6,7]
@@ -14221,7 +14221,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2d %xmm17, %xmm18, %xmm1
; AVX512-FCP-NEXT: vmovdqa64 %xmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
; AVX512-FCP-NEXT: vpshufhw $116, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -14365,7 +14365,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm23[2],xmm19[2],xmm23[3],xmm19[3]
; AVX512-FCP-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm21[2],xmm16[2],xmm21[3],xmm16[3]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} xmm17 = [3,7,0,0]
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
; AVX512-FCP-NEXT: vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm17, %xmm4 # 16-byte Folded Reload
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
; AVX512-FCP-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -14380,7 +14380,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3,4,5],ymm4[6,7]
; AVX512-FCP-NEXT: vpermt2q %zmm4, %zmm27, %zmm0
-; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
; AVX512-FCP-NEXT: vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm17, %xmm4 # 16-byte Folded Reload
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm4
@@ -14457,42 +14457,42 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-LABEL: load_i16_stride8_vf64:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: subq $2392, %rsp # imm = 0x958
-; AVX512DQ-NEXT: vmovdqa 368(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 368(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-NEXT: vmovdqa 336(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 336(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} xmm8 = [0,0,0,4]
; AVX512DQ-NEXT: vpermt2d %xmm2, %xmm8, %xmm0
; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm18
-; AVX512DQ-NEXT: vmovdqa 304(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 304(%rdi), %xmm1
; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 288(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 288(%rdi), %xmm2
; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 272(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 272(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %xmm2
; AVX512DQ-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm17
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX512DQ-NEXT: vmovdqa 480(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 480(%rdi), %ymm1
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -14500,19 +14500,19 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT: vmovdqa 416(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 416(%rdi), %ymm2
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm2
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm3[0,2,2,3,4,6,6,7]
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm5[0,2,2,3,4,6,6,7]
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
@@ -14520,37 +14520,37 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: # zmm23 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-NEXT: vpermt2q %zmm1, %zmm23, %zmm0
; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpermt2d %xmm3, %xmm8, %xmm1
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm3
-; AVX512DQ-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm4
-; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm5
; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm2
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -14561,11 +14561,11 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm3 = ymm4[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-NEXT: vmovdqa64 %ymm4, %ymm16
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm3
; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm3[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm3[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -14578,39 +14578,39 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 880(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 864(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 880(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 864(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-NEXT: vmovdqa 848(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 832(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 848(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 832(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpermt2d %xmm2, %xmm8, %xmm0
; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm28
; AVX512DQ-NEXT: vmovdqa %xmm8, %xmm11
-; AVX512DQ-NEXT: vmovdqa 816(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 800(%rdi), %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 816(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 800(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT: vmovdqa 784(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 768(%rdi), %xmm2
-; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 784(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 768(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm27
; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm20
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm10 = xmm1[0,1],xmm0[2,3]
-; AVX512DQ-NEXT: vmovdqa 992(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 992(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 960(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 960(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -14619,11 +14619,11 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm2[0,1,2,0,4,5,6,4]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm1 = ymm3[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm1[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-NEXT: vmovdqa 928(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 928(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 896(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 896(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -14634,39 +14634,39 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
; AVX512DQ-NEXT: vpermt2q %zmm7, %zmm23, %zmm10
-; AVX512DQ-NEXT: vmovdqa 624(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 608(%rdi), %xmm7
-; AVX512DQ-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 624(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 608(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
-; AVX512DQ-NEXT: vmovdqa 592(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %xmm7
-; AVX512DQ-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 592(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
; AVX512DQ-NEXT: vmovdqa %xmm0, %xmm7
; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm25
; AVX512DQ-NEXT: vpermt2d %xmm8, %xmm11, %xmm7
; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm26
-; AVX512DQ-NEXT: vmovdqa 560(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 544(%rdi), %xmm8
-; AVX512DQ-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 560(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 544(%rdi), %xmm8
+; AVX512DQ-NEXT: vmovdqa64 %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
-; AVX512DQ-NEXT: vmovdqa 528(%rdi), %xmm0
-; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %xmm8
-; AVX512DQ-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 528(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %xmm8
+; AVX512DQ-NEXT: vmovdqa64 %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]
; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm19
; AVX512DQ-NEXT: vmovdqa64 %xmm9, %xmm24
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm14 = xmm8[0,1],xmm7[2,3]
-; AVX512DQ-NEXT: vmovdqa 736(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 736(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 704(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 704(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -14677,10 +14677,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm12 = ymm0[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm21
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm15 = ymm12[0,1,2,3,4,5,6],ymm11[7]
-; AVX512DQ-NEXT: vmovdqa 672(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 672(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm31 = ymm0[0,1,0,2]
-; AVX512DQ-NEXT: vmovdqa 640(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm29 = ymm0[0,1,0,2]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm11 = ymm31[0,2,2,3,4,6,6,7]
@@ -14696,7 +14696,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm17[1,1,1,1]
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm9[1],xmm0[2,3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm15[0],xmm18[0],xmm15[1],xmm18[1]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3]
; AVX512DQ-NEXT: vpshufhw $116, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
@@ -14722,9 +14722,9 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm10[6,7]
; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm16[1,1,1,1]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0],xmm14[1],xmm6[2,3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm22 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm7[0],xmm22[0],xmm7[1],xmm22[1]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm6 = xmm6[0,1],xmm10[2,3]
@@ -14736,7 +14736,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm8
; AVX512DQ-NEXT: vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm8[1],xmm0[2,3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm28[0],xmm6[1],xmm28[1]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,7,5,8,9,10,11,12,13,15,13]
@@ -14773,11 +14773,11 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,4,6,8,9,10,11,12,13,12,14]
; AVX512DQ-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm2 = mem[0,1,1,3,4,5,5,7]
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
@@ -14864,7 +14864,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm18[2],xmm21[2],xmm18[3],xmm21[3]
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm17[2],xmm22[2],xmm17[3],xmm22[3]
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} xmm14 = [3,7,0,0]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
; AVX512DQ-NEXT: vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm5 # 16-byte Folded Reload
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm5[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -14893,7 +14893,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm20[2],xmm28[2],xmm20[3],xmm28[3]
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm16[2],xmm26[2],xmm16[3],xmm26[3]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
; AVX512DQ-NEXT: vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm4 # 16-byte Folded Reload
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
; AVX512DQ-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -14921,10 +14921,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm7 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm7 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -14969,19 +14969,19 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
; AVX512DQ-NEXT: vpermt2q %zmm1, %zmm23, %zmm0
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm2 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm2 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm5 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm5 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm3 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512DQ-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm4 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm4 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512DQ-NEXT: vmovdqa %xmm5, %xmm1
@@ -15015,16 +15015,16 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm9 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm9 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512DQ-NEXT: vmovdqa %xmm2, %xmm0
@@ -15060,16 +15060,16 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4],ymm2[5],ymm5[6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
; AVX512DQ-NEXT: vpermt2q %zmm1, %zmm23, %zmm3
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm13 # 16-byte Folded Reload
; AVX512DQ-NEXT: # xmm13 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512DQ-NEXT: vmovdqa %xmm11, %xmm0
@@ -15374,62 +15374,62 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-LABEL: load_i16_stride8_vf64:
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: subq $2440, %rsp # imm = 0x988
-; AVX512DQ-FCP-NEXT: vmovdqa 368(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 368(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-FCP-NEXT: vmovdqa 336(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 336(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} xmm8 = [0,0,0,4]
; AVX512DQ-FCP-NEXT: vpermt2d %xmm2, %xmm8, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm16
-; AVX512DQ-FCP-NEXT: vmovdqa 304(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 304(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 272(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 272(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm17
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa 480(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 480(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 416(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,1,0,2]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm3[0,2,2,3,4,6,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm5[0,2,2,3,4,6,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
@@ -15437,14 +15437,14 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm26, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-FCP-NEXT: vmovdqa 80(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm21
@@ -15453,40 +15453,40 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm3
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 48(%rdi), %xmm5
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm20
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm19
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,2]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm3[0,1,2,0,4,5,6,4]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm4[0,1,2,0,4,5,6,4]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm3[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm3[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm10 = ymm4[0,2,2,3,4,6,6,7]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm10[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm7 = ymm5[0,2,2,3,4,6,6,7]
@@ -15496,56 +15496,56 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 880(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 864(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 880(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 864(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-FCP-NEXT: vmovdqa 848(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 832(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 848(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 832(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2d %xmm2, %xmm8, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm24
; AVX512DQ-FCP-NEXT: vmovdqa %xmm8, %xmm11
-; AVX512DQ-FCP-NEXT: vmovdqa 816(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 800(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 816(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 800(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-FCP-NEXT: vmovdqa 784(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 768(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 784(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 768(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm28
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm23
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm1[0,1],xmm0[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa 992(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 992(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 960(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 960(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm30
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm4 = ymm2[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm4[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm1[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 928(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 928(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 896(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 896(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm1[0,2,2,3,4,6,6,7]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm9 = ymm3[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm0[0,2,2,3,4,6,6,7]
@@ -15553,55 +15553,55 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3,4],ymm9[5],ymm12[6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm8, %zmm26, %zmm6
-; AVX512DQ-FCP-NEXT: vmovdqa 624(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 608(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 624(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 608(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-FCP-NEXT: vmovdqa 592(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 592(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, %xmm9
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm27
; AVX512DQ-FCP-NEXT: vpermt2d %xmm8, %xmm11, %xmm9
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm8, %xmm29
-; AVX512DQ-FCP-NEXT: vmovdqa 560(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 544(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 560(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 544(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-FCP-NEXT: vmovdqa 528(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 528(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm31
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm8, %xmm25
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm13 = xmm12[0,1],xmm9[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa 736(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 736(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm9 = ymm1[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm14 = ymm9[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm12 = ymm0[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm15 = ymm12[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm15[0,1,2,3,4,5,6],ymm14[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 672(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 672(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm8 = ymm0[0,1,0,2]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm14 = ymm5[0,2,2,3,4,6,6,7]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm14[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm15 = ymm8[0,2,2,3,4,6,6,7]
@@ -15690,11 +15690,11 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,4,6,8,9,10,11,12,13,12,14]
; AVX512DQ-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm1 = mem[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5,6],ymm0[7]
; AVX512DQ-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
@@ -15737,7 +15737,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm0[2,3]
; AVX512DQ-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,5,7]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5,4,6,8,9,10,11,12,13,12,14]
; AVX512DQ-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,5,7]
@@ -15836,17 +15836,17 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, %xmm0
@@ -15859,15 +15859,15 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm1 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,1,3]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,6,4,6,7,8,9,10,11,14,12,14,15]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
@@ -15883,15 +15883,15 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm27, %zmm0
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm2 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm2 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm6 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm6 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm3 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
@@ -15905,12 +15905,12 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -15918,10 +15918,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm4 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm6 = ymm3[0,2,2,3,4,6,6,7]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm6[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm24
@@ -15932,16 +15932,16 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, %xmm0
@@ -15954,10 +15954,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm23
@@ -15966,10 +15966,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm10 = ymm0[0,2,2,3,4,6,6,7]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm10[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm6 = ymm3[0,2,2,3,4,6,6,7]
@@ -15977,16 +15977,16 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm27, %zmm4
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm13 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm13 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]
; AVX512DQ-FCP-NEXT: vmovdqa %xmm5, %xmm0
@@ -15998,10 +15998,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm15 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm9 = ymm0[0,1,2,0,4,5,6,4]
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm5 = ymm2[0,1,2,0,4,5,6,4]
@@ -16009,10 +16009,10 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermq $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,1,3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[0,2,2,3,4,6,6,7]
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm3[0,1,0,2,4,5,6,7,8,9,8,10,12,13,14,15]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,2,2,3,4,6,6,7]
@@ -16047,7 +16047,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2d %xmm17, %xmm18, %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
; AVX512DQ-FCP-NEXT: vpshufhw $116, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -16191,7 +16191,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm23[2],xmm19[2],xmm23[3],xmm19[3]
; AVX512DQ-FCP-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm21[2],xmm16[2],xmm21[3],xmm16[3]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} xmm17 = [3,7,0,0]
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm17, %xmm4 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
; AVX512DQ-FCP-NEXT: vpshufhw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
@@ -16206,7 +16206,7 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5],ymm12[6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3,4,5],ymm4[6,7]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm4, %zmm27, %zmm0
-; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %xmm17, %xmm4 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm4
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-5.ll
index b988749fd86f1..e10dbf2592c1a 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-5.ll
@@ -1005,7 +1005,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 64(%rdi), %zmm2
; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,5,10,15,20,25,30,0]
; AVX512-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm3
+; AVX512-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm3
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6],ymm3[7]
; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm3 = [17,22,27,0,5,10,15,0]
; AVX512-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
@@ -1014,7 +1014,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,7,12,17,22,27,0,0]
; AVX512-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,1,2,3,4,5,8,13]
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512-NEXT: vpermi2d %ymm6, %ymm4, %ymm5
; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,8,13,18,23,28,0,0]
; AVX512-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
@@ -1038,7 +1038,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm2
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,5,10,15,20,25,30,0]
; AVX512-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm3
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm3
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6],ymm3[7]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [17,22,27,0,5,10,15,0]
; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
@@ -1047,7 +1047,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,7,12,17,22,27,0,0]
; AVX512-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,1,2,3,4,5,8,13]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512-FCP-NEXT: vpermi2d %ymm6, %ymm4, %ymm5
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,8,13,18,23,28,0,0]
; AVX512-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
@@ -1071,7 +1071,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm2
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,5,10,15,20,25,30,0]
; AVX512DQ-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm3
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm3
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6],ymm3[7]
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm3 = [17,22,27,0,5,10,15,0]
; AVX512DQ-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
@@ -1080,7 +1080,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,7,12,17,22,27,0,0]
; AVX512DQ-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,1,2,3,4,5,8,13]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-NEXT: vpermi2d %ymm6, %ymm4, %ymm5
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,8,13,18,23,28,0,0]
; AVX512DQ-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
@@ -1104,7 +1104,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm2
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,5,10,15,20,25,30,0]
; AVX512DQ-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm3
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm3
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6],ymm3[7]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [17,22,27,0,5,10,15,0]
; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
@@ -1113,7 +1113,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,7,12,17,22,27,0,0]
; AVX512DQ-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,1,2,3,4,5,8,13]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpermi2d %ymm6, %ymm4, %ymm5
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,8,13,18,23,28,0,0]
; AVX512DQ-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
@@ -1137,7 +1137,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm2
; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,5,10,15,20,25,30,0]
; AVX512BW-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512BW-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm3
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm3
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6],ymm3[7]
; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm3 = [17,22,27,0,5,10,15,0]
; AVX512BW-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
@@ -1146,7 +1146,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,7,12,17,22,27,0,0]
; AVX512BW-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,1,2,3,4,5,8,13]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512BW-NEXT: vpermi2d %ymm6, %ymm4, %ymm5
; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,8,13,18,23,28,0,0]
; AVX512BW-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
@@ -1170,7 +1170,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm2
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,5,10,15,20,25,30,0]
; AVX512BW-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm3
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm3
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6],ymm3[7]
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [17,22,27,0,5,10,15,0]
; AVX512BW-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
@@ -1179,7 +1179,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,7,12,17,22,27,0,0]
; AVX512BW-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,1,2,3,4,5,8,13]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpermi2d %ymm6, %ymm4, %ymm5
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,8,13,18,23,28,0,0]
; AVX512BW-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
@@ -1203,7 +1203,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm2
; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,5,10,15,20,25,30,0]
; AVX512DQ-BW-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm3
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm3
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6],ymm3[7]
; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm3 = [17,22,27,0,5,10,15,0]
; AVX512DQ-BW-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
@@ -1212,7 +1212,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,7,12,17,22,27,0,0]
; AVX512DQ-BW-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,1,2,3,4,5,8,13]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpermi2d %ymm6, %ymm4, %ymm5
; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,8,13,18,23,28,0,0]
; AVX512DQ-BW-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
@@ -1236,7 +1236,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm2
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,5,10,15,20,25,30,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm3
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm3
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6],ymm3[7]
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [17,22,27,0,5,10,15,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm3
@@ -1245,7 +1245,7 @@ define void @load_i32_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,7,12,17,22,27,0,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,1,2,3,4,5,8,13]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpermi2d %ymm6, %ymm4, %ymm5
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [3,8,13,18,23,28,0,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm2, %zmm1, %zmm4
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-6.ll
index f8863e16f68d1..2d65e0688d1eb 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-6.ll
@@ -1295,8 +1295,8 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-LABEL: load_i32_stride6_vf8:
; AVX512: # %bb.0:
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,6,12,18,24,30,0,0]
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -1340,8 +1340,8 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-LABEL: load_i32_stride6_vf8:
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,6,12,18,24,30,0,0]
; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -1385,8 +1385,8 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-LABEL: load_i32_stride6_vf8:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,6,12,18,24,30,0,0]
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -1430,8 +1430,8 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-LABEL: load_i32_stride6_vf8:
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,6,12,18,24,30,0,0]
; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -1475,8 +1475,8 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-LABEL: load_i32_stride6_vf8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,6,12,18,24,30,0,0]
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -1520,8 +1520,8 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-LABEL: load_i32_stride6_vf8:
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,6,12,18,24,30,0,0]
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -1565,8 +1565,8 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-LABEL: load_i32_stride6_vf8:
; AVX512DQ-BW: # %bb.0:
; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,6,12,18,24,30,0,0]
; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -1610,8 +1610,8 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-LABEL: load_i32_stride6_vf8:
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,6,12,18,24,30,0,0]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm4
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-5.ll
index 4631b7d126902..0dacc0cf92947 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-5.ll
@@ -467,7 +467,7 @@ define void @load_i64_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpermi2q %zmm1, %zmm0, %zmm2
; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,6,11,0]
; AVX512-NEXT: vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm4
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm4
; AVX512-NEXT: vpbroadcastq %xmm4, %ymm5
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
@@ -498,9 +498,9 @@ define void @load_i64_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,6,11,0]
; AVX512-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm3
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [0,1,2,4]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm5
; AVX512-FCP-NEXT: vpermi2q %ymm5, %ymm3, %ymm4
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %xmm3
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %xmm3
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm6 = [2,7,12,0]
; AVX512-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm6
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm7 = [0,1,2,5]
@@ -528,7 +528,7 @@ define void @load_i64_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpermi2q %zmm1, %zmm0, %zmm2
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,6,11,0]
; AVX512DQ-NEXT: vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm4
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm4
; AVX512DQ-NEXT: vpbroadcastq %xmm4, %ymm5
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
@@ -559,9 +559,9 @@ define void @load_i64_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,6,11,0]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm3
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [0,1,2,4]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpermi2q %ymm5, %ymm3, %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %xmm3
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm6 = [2,7,12,0]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm6
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm7 = [0,1,2,5]
@@ -589,7 +589,7 @@ define void @load_i64_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm2
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,6,11,0]
; AVX512BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm4
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm4
; AVX512BW-NEXT: vpbroadcastq %xmm4, %ymm5
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
@@ -620,9 +620,9 @@ define void @load_i64_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,6,11,0]
; AVX512BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm3
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [0,1,2,4]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm5
; AVX512BW-FCP-NEXT: vpermi2q %ymm5, %ymm3, %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm3
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm3
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm6 = [2,7,12,0]
; AVX512BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm6
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm7 = [0,1,2,5]
@@ -650,7 +650,7 @@ define void @load_i64_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm2
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,6,11,0]
; AVX512DQ-BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm3
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm4
; AVX512DQ-BW-NEXT: vpbroadcastq %xmm4, %ymm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
@@ -681,9 +681,9 @@ define void @load_i64_stride5_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,6,11,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm3
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [0,1,2,4]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm5
; AVX512DQ-BW-FCP-NEXT: vpermi2q %ymm5, %ymm3, %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm3
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm6 = [2,7,12,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm6
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm7 = [0,1,2,5]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-6.ll
index 753faf94641d4..4efd003a2a417 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-6.ll
@@ -569,18 +569,18 @@ define void @load_i64_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,7,13,0]
; AVX512-NEXT: vpermi2q %zmm3, %zmm2, %zmm1
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm5 = [10,0,6,0]
; AVX512-NEXT: vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512-NEXT: vmovdqa 160(%rdi), %xmm6
+; AVX512-NEXT: vmovdqa64 160(%rdi), %xmm6
; AVX512-NEXT: vpbroadcastq %xmm6, %ymm7
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm7[6,7]
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm7 = [11,1,7,0]
; AVX512-NEXT: vpermi2q %zmm2, %zmm3, %zmm7
; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm7
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm7
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[2],ymm7[2]
; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,3]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm8 = [4,10]
@@ -615,9 +615,9 @@ define void @load_i64_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [10,0,6,0]
; AVX512-FCP-NEXT: vpermi2q %zmm2, %zmm3, %zmm4
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm5 = [0,1,2,4]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512-FCP-NEXT: vpermi2q %ymm6, %ymm4, %ymm5
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %xmm4
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %xmm4
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm7 = [11,1,7,0]
; AVX512-FCP-NEXT: vpermi2q %zmm2, %zmm3, %zmm7
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,1,2,5]
@@ -652,18 +652,18 @@ define void @load_i64_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,7,13,0]
; AVX512DQ-NEXT: vpermi2q %zmm3, %zmm2, %zmm1
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm5 = [10,0,6,0]
; AVX512DQ-NEXT: vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %xmm6
; AVX512DQ-NEXT: vpbroadcastq %xmm6, %ymm7
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm7[6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm7 = [11,1,7,0]
; AVX512DQ-NEXT: vpermi2q %zmm2, %zmm3, %zmm7
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm7
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm7
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[2],ymm7[2]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,3]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm8 = [4,10]
@@ -698,9 +698,9 @@ define void @load_i64_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [10,0,6,0]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm2, %zmm3, %zmm4
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm5 = [0,1,2,4]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpermi2q %ymm6, %ymm4, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %xmm4
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm7 = [11,1,7,0]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm2, %zmm3, %zmm7
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,1,2,5]
@@ -735,18 +735,18 @@ define void @load_i64_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,7,13,0]
; AVX512BW-NEXT: vpermi2q %zmm3, %zmm2, %zmm1
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm5 = [10,0,6,0]
; AVX512BW-NEXT: vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm6
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm6
; AVX512BW-NEXT: vpbroadcastq %xmm6, %ymm7
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm7[6,7]
; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm7 = [11,1,7,0]
; AVX512BW-NEXT: vpermi2q %zmm2, %zmm3, %zmm7
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %ymm7
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %ymm7
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[2],ymm7[2]
; AVX512BW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,3]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm8 = [4,10]
@@ -781,9 +781,9 @@ define void @load_i64_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [10,0,6,0]
; AVX512BW-FCP-NEXT: vpermi2q %zmm2, %zmm3, %zmm4
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm5 = [0,1,2,4]
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpermi2q %ymm6, %ymm4, %ymm5
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %xmm4
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %xmm4
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm7 = [11,1,7,0]
; AVX512BW-FCP-NEXT: vpermi2q %zmm2, %zmm3, %zmm7
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,1,2,5]
@@ -818,18 +818,18 @@ define void @load_i64_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,7,13,0]
; AVX512DQ-BW-NEXT: vpermi2q %zmm3, %zmm2, %zmm1
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm5 = [10,0,6,0]
; AVX512DQ-BW-NEXT: vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %xmm6
; AVX512DQ-BW-NEXT: vpbroadcastq %xmm6, %ymm7
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm7[6,7]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm7 = [11,1,7,0]
; AVX512DQ-BW-NEXT: vpermi2q %zmm2, %zmm3, %zmm7
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %ymm7
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %ymm7
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[2],ymm7[2]
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,3]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm8 = [4,10]
@@ -864,9 +864,9 @@ define void @load_i64_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [10,0,6,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm2, %zmm3, %zmm4
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm5 = [0,1,2,4]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpermi2q %ymm6, %ymm4, %ymm5
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %xmm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %xmm4
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm7 = [11,1,7,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm2, %zmm3, %zmm7
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,1,2,5]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-7.ll
index aa27979f12e9b..ff129f04c4e67 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-7.ll
@@ -644,7 +644,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm3
; AVX512-NEXT: vmovdqa64 64(%rdi), %zmm4
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512-NEXT: vinserti128 $1, 160(%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vinserti32x4 $1, 160(%rdi), %ymm0, %ymm0
; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,14,0]
; AVX512-NEXT: vpermi2q %zmm4, %zmm5, %zmm1
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
@@ -652,7 +652,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpermi2q %zmm5, %zmm4, %zmm1
; AVX512-NEXT: vpbroadcastq 176(%rdi), %ymm2
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm6
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm6
; AVX512-NEXT: vpbroadcastq %xmm6, %ymm2
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512-NEXT: vmovdqa 16(%rdi), %xmm7
@@ -660,16 +660,16 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm7
; AVX512-NEXT: vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm8
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm8
; AVX512-NEXT: vpalignr {{.*#+}} xmm6 = xmm6[8,9,10,11,12,13,14,15],xmm8[0,1,2,3,4,5,6,7]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm7[2,3],ymm6[0,1]
; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm7
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm8
; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm9 = [4,11]
; AVX512-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512-NEXT: vpalignr {{.*#+}} ymm8 = ymm8[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm8[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm9 = [5,12]
; AVX512-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
@@ -696,7 +696,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %zmm3
; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm4
; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %xmm0
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,14,0]
; AVX512-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm1
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm6 = [0,1,2,5]
@@ -712,17 +712,17 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %xmm7
-; AVX512-FCP-NEXT: vpalignr {{.*#+}} xmm8 = mem[8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7]
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm7
+; AVX512-FCP-NEXT: valignq {{.*#+}} xmm8 = mem[1],xmm7[0]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm2[2,3],ymm8[0,1]
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [2,5,2,5]
; AVX512-FCP-NEXT: # ymm8 = mem[0,1,0,1]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm9
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm9
; AVX512-FCP-NEXT: vpermi2q %ymm7, %ymm9, %ymm8
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} xmm7 = [4,11]
; AVX512-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm7
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm8
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm8 = ymm9[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],ymm9[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} xmm9 = [5,12]
; AVX512-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
@@ -749,7 +749,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm3
; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm4
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512DQ-NEXT: vinserti128 $1, 160(%rdi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vinserti32x4 $1, 160(%rdi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,14,0]
; AVX512DQ-NEXT: vpermi2q %zmm4, %zmm5, %zmm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
@@ -757,7 +757,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpermi2q %zmm5, %zmm4, %zmm1
; AVX512DQ-NEXT: vpbroadcastq 176(%rdi), %ymm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm6
; AVX512DQ-NEXT: vpbroadcastq %xmm6, %ymm2
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm7
@@ -765,16 +765,16 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm7
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm8
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm8
; AVX512DQ-NEXT: vpalignr {{.*#+}} xmm6 = xmm6[8,9,10,11,12,13,14,15],xmm8[0,1,2,3,4,5,6,7]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm7[2,3],ymm6[0,1]
; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm7
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm8
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm9 = [4,11]
; AVX512DQ-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm8 = ymm8[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm8[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm9 = [5,12]
; AVX512DQ-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
@@ -801,7 +801,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %zmm3
; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm4
; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %xmm0
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,14,0]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm1
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm6 = [0,1,2,5]
@@ -817,17 +817,17 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %xmm7
-; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} xmm8 = mem[8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm7
+; AVX512DQ-FCP-NEXT: valignq {{.*#+}} xmm8 = mem[1],xmm7[0]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm2[2,3],ymm8[0,1]
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [2,5,2,5]
; AVX512DQ-FCP-NEXT: # ymm8 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm9
; AVX512DQ-FCP-NEXT: vpermi2q %ymm7, %ymm9, %ymm8
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} xmm7 = [4,11]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm7
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm8 = ymm9[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],ymm9[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} xmm9 = [5,12]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
@@ -854,7 +854,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm3
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm4
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512BW-NEXT: vinserti128 $1, 160(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vinserti32x4 $1, 160(%rdi), %ymm0, %ymm0
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,14,0]
; AVX512BW-NEXT: vpermi2q %zmm4, %zmm5, %zmm1
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
@@ -862,7 +862,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpermi2q %zmm5, %zmm4, %zmm1
; AVX512BW-NEXT: vpbroadcastq 176(%rdi), %ymm2
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm6
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm6
; AVX512BW-NEXT: vpbroadcastq %xmm6, %ymm2
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm7
@@ -870,16 +870,16 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
; AVX512BW-NEXT: vmovdqa 64(%rdi), %ymm7
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm8
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %xmm8
; AVX512BW-NEXT: vpalignr {{.*#+}} xmm6 = xmm6[8,9,10,11,12,13,14,15],xmm8[0,1,2,3,4,5,6,7]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm7[2,3],ymm6[0,1]
; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm7
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm8
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm8
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm9 = [4,11]
; AVX512BW-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm8 = ymm8[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm8[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm9 = [5,12]
; AVX512BW-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
@@ -906,7 +906,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm3
; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm4
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %xmm0
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,14,0]
; AVX512BW-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm1
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm6 = [0,1,2,5]
@@ -922,17 +922,17 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm7
-; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} xmm8 = mem[8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7]
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm7
+; AVX512BW-FCP-NEXT: valignq {{.*#+}} xmm8 = mem[1],xmm7[0]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm2[2,3],ymm8[0,1]
; AVX512BW-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [2,5,2,5]
; AVX512BW-FCP-NEXT: # ymm8 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm9
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm9
; AVX512BW-FCP-NEXT: vpermi2q %ymm7, %ymm9, %ymm8
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm7 = [4,11]
; AVX512BW-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm7
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm8
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm8 = ymm9[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],ymm9[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm9 = [5,12]
; AVX512BW-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
@@ -959,7 +959,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm3
; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm4
; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512DQ-BW-NEXT: vinserti128 $1, 160(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 160(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,14,0]
; AVX512DQ-BW-NEXT: vpermi2q %zmm4, %zmm5, %zmm1
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
@@ -967,7 +967,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpermi2q %zmm5, %zmm4, %zmm1
; AVX512DQ-BW-NEXT: vpbroadcastq 176(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm6
; AVX512DQ-BW-NEXT: vpbroadcastq %xmm6, %ymm2
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm7
@@ -975,16 +975,16 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %ymm7
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm8
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %xmm8
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} xmm6 = xmm6[8,9,10,11,12,13,14,15],xmm8[0,1,2,3,4,5,6,7]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm6 = ymm7[2,3],ymm6[0,1]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm7
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm8
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm8
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm9 = [4,11]
; AVX512DQ-BW-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm8 = ymm8[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm8[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm9 = [5,12]
; AVX512DQ-BW-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
@@ -1011,7 +1011,7 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm3
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm5
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,14,0]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm6 = [0,1,2,5]
@@ -1027,17 +1027,17 @@ define void @load_i64_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm7
-; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} xmm8 = mem[8,9,10,11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm7
+; AVX512DQ-BW-FCP-NEXT: valignq {{.*#+}} xmm8 = mem[1],xmm7[0]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm2[2,3],ymm8[0,1]
; AVX512DQ-BW-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [2,5,2,5]
; AVX512DQ-BW-FCP-NEXT: # ymm8 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm9
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm9
; AVX512DQ-BW-FCP-NEXT: vpermi2q %ymm7, %ymm9, %ymm8
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm7 = [4,11]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm7
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm8
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm8 = ymm9[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],ymm9[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm9 = [5,12]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm4, %zmm5, %zmm9
@@ -1628,8 +1628,8 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm11
; AVX512-NEXT: vpalignr {{.*#+}} ymm11 = mem[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm11, %xmm11
-; AVX512-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm12
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm12
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm14 = [4,11]
; AVX512-NEXT: vpermi2q %zmm9, %zmm10, %zmm14
@@ -1644,7 +1644,7 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
; AVX512-NEXT: vpermi2q %zmm0, %zmm8, %zmm9
; AVX512-NEXT: vinserti64x4 $0, %ymm15, %zmm9, %zmm8
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512-NEXT: vpalignr {{.*#+}} ymm9 = ymm13[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
@@ -1737,9 +1737,9 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm11
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm11 = mem[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm11, %xmm11
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %xmm12
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm12
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm14 = [0,0,2,5]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512-FCP-NEXT: vpermi2q %ymm12, %ymm13, %ymm14
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} xmm12 = [4,11]
; AVX512-FCP-NEXT: vpermi2q %zmm9, %zmm10, %zmm12
@@ -1754,7 +1754,7 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
; AVX512-FCP-NEXT: vpermi2q %zmm0, %zmm8, %zmm9
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm15, %zmm9, %zmm8
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm9 = ymm13[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rdi
@@ -1847,8 +1847,8 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm11
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm11 = mem[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm11, %xmm11
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm12
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm12
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm14 = [4,11]
; AVX512DQ-NEXT: vpermi2q %zmm9, %zmm10, %zmm14
@@ -1863,7 +1863,7 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-NEXT: vpermi2q %zmm0, %zmm8, %zmm9
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm15, %zmm9, %zmm8
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm9 = ymm13[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rdi
@@ -1956,9 +1956,9 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm11 = mem[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm11, %xmm11
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %xmm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm12
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm14 = [0,0,2,5]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vpermi2q %ymm12, %ymm13, %ymm14
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} xmm12 = [4,11]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm9, %zmm10, %zmm12
@@ -1973,7 +1973,7 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm0, %zmm8, %zmm9
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm15, %zmm9, %zmm8
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm9 = ymm13[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rdi
@@ -2063,8 +2063,8 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm11
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm11, %zmm7, %zmm7
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm12
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm11
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm12
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm11
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm12 = ymm11[0,1,2,3,4,5],ymm12[6,7]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm13 = [4,11]
; AVX512BW-NEXT: vpermi2q %zmm9, %zmm10, %zmm13
@@ -2098,7 +2098,7 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [7,0,9,0,7,0,9,0]
; AVX512BW-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]
; AVX512BW-NEXT: vpermi2q %zmm5, %zmm4, %zmm14
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm15
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm15
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm11 = ymm11[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3],ymm11[4,5,6,7]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm13 = [0,0,4,11]
@@ -2172,9 +2172,9 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm11
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm11, %zmm6, %zmm6
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm11
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm11
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm12 = [0,0,2,5]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512BW-FCP-NEXT: vpermi2q %ymm11, %ymm13, %ymm12
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm11 = [4,11]
; AVX512BW-FCP-NEXT: vpermi2q %zmm9, %zmm10, %zmm11
@@ -2189,7 +2189,7 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
; AVX512BW-FCP-NEXT: vpermi2q %zmm0, %zmm7, %zmm9
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm12, %zmm9, %zmm7
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm9 = ymm13[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm14[0,1,2,3],ymm9[4,5,6,7]
; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
@@ -2282,8 +2282,8 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm11
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm11, %zmm7, %zmm7
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm12
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm11
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm12
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm11
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm12 = ymm11[0,1,2,3,4,5],ymm12[6,7]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm13 = [4,11]
; AVX512DQ-BW-NEXT: vpermi2q %zmm9, %zmm10, %zmm13
@@ -2317,7 +2317,7 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [7,0,9,0,7,0,9,0]
; AVX512DQ-BW-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-BW-NEXT: vpermi2q %zmm5, %zmm4, %zmm14
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm15
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm15
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm11 = ymm11[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3],ymm11[4,5,6,7]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm13 = [0,0,4,11]
@@ -2391,9 +2391,9 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm11
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm11, %zmm6, %zmm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm11
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm12 = [0,0,2,5]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm13
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm13
; AVX512DQ-BW-FCP-NEXT: vpermi2q %ymm11, %ymm13, %ymm12
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm11 = [4,11]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm9, %zmm10, %zmm11
@@ -2408,7 +2408,7 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm0, %zmm7, %zmm9
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm12, %zmm9, %zmm7
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm9 = ymm13[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm14[0,1,2,3],ymm9[4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
@@ -3762,7 +3762,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm30, %zmm23, %zmm22
; AVX512-NEXT: vpermi2q %zmm26, %zmm3, %zmm25
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 464(%rdi), %xmm5
+; AVX512-NEXT: vmovdqa64 464(%rdi), %xmm5
; AVX512-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm5, %zmm25, %zmm21
; AVX512-NEXT: vmovdqa64 %zmm22, %zmm21 {%k1}
@@ -3783,7 +3783,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermi2q %zmm6, %zmm0, %zmm24
; AVX512-NEXT: vpermt2q %zmm30, %zmm23, %zmm24
; AVX512-NEXT: vpermi2q %zmm3, %zmm26, %zmm25
-; AVX512-NEXT: vmovdqa 512(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 512(%rdi), %ymm4
; AVX512-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512-NEXT: vinserti32x4 $0, %xmm4, %zmm25, %zmm23
@@ -3793,8 +3793,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [4,5,6,13,4,5,6,13]
; AVX512-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]
; AVX512-NEXT: vpermt2q %zmm13, %zmm5, %zmm4
-; AVX512-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm11
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm11
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm3 = [4,11]
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm10
@@ -3803,8 +3803,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti64x4 $0, %ymm10, %zmm4, %zmm24
; AVX512-NEXT: vpermi2q %zmm6, %zmm0, %zmm18
; AVX512-NEXT: vpermt2q %zmm30, %zmm5, %zmm18
-; AVX512-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm4
-; AVX512-NEXT: vmovdqa 576(%rdi), %ymm5
+; AVX512-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm4
+; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm5
; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
; AVX512-NEXT: vpermi2q %zmm27, %zmm2, %zmm3
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
@@ -3814,7 +3814,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [4,5,6,14,4,5,6,14]
; AVX512-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]
; AVX512-NEXT: vpermt2q %zmm13, %zmm10, %zmm4
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm11
; AVX512-NEXT: vpalignr {{.*#+}} ymm11 = ymm12[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm12 = [5,12]
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm7
@@ -3823,7 +3823,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm4
; AVX512-NEXT: vpermi2q %zmm0, %zmm6, %zmm20
; AVX512-NEXT: vpermt2q %zmm30, %zmm10, %zmm20
-; AVX512-NEXT: vmovdqa 640(%rdi), %ymm7
+; AVX512-NEXT: vmovdqa64 640(%rdi), %ymm7
; AVX512-NEXT: vpalignr {{.*#+}} ymm5 = ymm5[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],ymm5[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vpermi2q %zmm27, %zmm2, %zmm12
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm12[0,1,2,3],ymm5[4,5,6,7]
@@ -3949,7 +3949,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm29, %zmm22, %zmm5
; AVX512-FCP-NEXT: vpermi2q %zmm25, %zmm1, %zmm23
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 464(%rdi), %xmm4
+; AVX512-FCP-NEXT: vmovdqa64 464(%rdi), %xmm4
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm4, %zmm23, %zmm21
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm21 {%k1}
@@ -3970,7 +3970,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermi2q %zmm6, %zmm0, %zmm24
; AVX512-FCP-NEXT: vpermt2q %zmm29, %zmm5, %zmm24
; AVX512-FCP-NEXT: vpermi2q %zmm1, %zmm25, %zmm23
-; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %ymm4
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm4, %zmm23, %zmm23
@@ -3982,8 +3982,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm13, %zmm11, %zmm4
; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm24
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm5 = [0,1,2,5]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm12
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm12[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vpermt2q %ymm24, %ymm5, %ymm12
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} xmm3 = [4,11]
@@ -3993,8 +3993,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm10, %zmm4, %zmm24
; AVX512-FCP-NEXT: vpermi2q %zmm6, %zmm0, %zmm18
; AVX512-FCP-NEXT: vpermt2q %zmm29, %zmm11, %zmm18
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %xmm4
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %ymm10
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %xmm4
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm10
; AVX512-FCP-NEXT: vpermi2q %ymm4, %ymm10, %ymm5
; AVX512-FCP-NEXT: vpermi2q %zmm26, %zmm2, %zmm3
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7]
@@ -4011,7 +4011,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm4, %zmm1
; AVX512-FCP-NEXT: vpermi2q %zmm0, %zmm6, %zmm20
; AVX512-FCP-NEXT: vpermt2q %zmm29, %zmm5, %zmm20
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %ymm4
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm10[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],ymm10[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vpermi2q %zmm26, %zmm2, %zmm11
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]
@@ -4137,7 +4137,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm30, %zmm23, %zmm22
; AVX512DQ-NEXT: vpermi2q %zmm26, %zmm3, %zmm25
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 464(%rdi), %xmm5
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm5, %zmm25, %zmm21
; AVX512DQ-NEXT: vmovdqa64 %zmm22, %zmm21 {%k1}
@@ -4158,7 +4158,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermi2q %zmm6, %zmm0, %zmm24
; AVX512DQ-NEXT: vpermt2q %zmm30, %zmm23, %zmm24
; AVX512DQ-NEXT: vpermi2q %zmm3, %zmm26, %zmm25
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %ymm4
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm4, %zmm25, %zmm23
@@ -4168,8 +4168,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [4,5,6,13,4,5,6,13]
; AVX512DQ-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-NEXT: vpermt2q %zmm13, %zmm5, %zmm4
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm11
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm11
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm3 = [4,11]
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm10
@@ -4178,8 +4178,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm10, %zmm4, %zmm24
; AVX512DQ-NEXT: vpermi2q %zmm6, %zmm0, %zmm18
; AVX512DQ-NEXT: vpermt2q %zmm30, %zmm5, %zmm18
-; AVX512DQ-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm4
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %ymm5
+; AVX512DQ-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm4
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm5
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
; AVX512DQ-NEXT: vpermi2q %zmm27, %zmm2, %zmm3
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
@@ -4189,7 +4189,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [4,5,6,14,4,5,6,14]
; AVX512DQ-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-NEXT: vpermt2q %zmm13, %zmm10, %zmm4
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm11
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm11
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm11 = ymm12[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm12 = [5,12]
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm7
@@ -4198,7 +4198,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm4
; AVX512DQ-NEXT: vpermi2q %zmm0, %zmm6, %zmm20
; AVX512DQ-NEXT: vpermt2q %zmm30, %zmm10, %zmm20
-; AVX512DQ-NEXT: vmovdqa 640(%rdi), %ymm7
+; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %ymm7
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm5 = ymm5[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],ymm5[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vpermi2q %zmm27, %zmm2, %zmm12
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm12[0,1,2,3],ymm5[4,5,6,7]
@@ -4324,7 +4324,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm29, %zmm22, %zmm5
; AVX512DQ-FCP-NEXT: vpermi2q %zmm25, %zmm1, %zmm23
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 464(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 464(%rdi), %xmm4
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm4, %zmm23, %zmm21
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm21 {%k1}
@@ -4345,7 +4345,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermi2q %zmm6, %zmm0, %zmm24
; AVX512DQ-FCP-NEXT: vpermt2q %zmm29, %zmm5, %zmm24
; AVX512DQ-FCP-NEXT: vpermi2q %zmm1, %zmm25, %zmm23
-; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm4, %zmm23, %zmm23
@@ -4357,8 +4357,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm13, %zmm11, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm24
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm5 = [0,1,2,5]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm12
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm12[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vpermt2q %ymm24, %ymm5, %ymm12
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} xmm3 = [4,11]
@@ -4368,8 +4368,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm10, %zmm4, %zmm24
; AVX512DQ-FCP-NEXT: vpermi2q %zmm6, %zmm0, %zmm18
; AVX512DQ-FCP-NEXT: vpermt2q %zmm29, %zmm11, %zmm18
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %ymm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm10
; AVX512DQ-FCP-NEXT: vpermi2q %ymm4, %ymm10, %ymm5
; AVX512DQ-FCP-NEXT: vpermi2q %zmm26, %zmm2, %zmm3
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7]
@@ -4386,7 +4386,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm4, %zmm1
; AVX512DQ-FCP-NEXT: vpermi2q %zmm0, %zmm6, %zmm20
; AVX512DQ-FCP-NEXT: vpermt2q %zmm29, %zmm5, %zmm20
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm10[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],ymm10[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm26, %zmm2, %zmm11
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]
@@ -4512,7 +4512,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm3, %zmm23, %zmm22
; AVX512BW-NEXT: vpermi2q %zmm26, %zmm4, %zmm25
; AVX512BW-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 464(%rdi), %xmm5
+; AVX512BW-NEXT: vmovdqa64 464(%rdi), %xmm5
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm5, %zmm25, %zmm21
; AVX512BW-NEXT: vmovdqa64 %zmm22, %zmm21 {%k1}
@@ -4533,7 +4533,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermi2q %zmm6, %zmm0, %zmm24
; AVX512BW-NEXT: vpermt2q %zmm3, %zmm23, %zmm24
; AVX512BW-NEXT: vpermi2q %zmm4, %zmm26, %zmm25
-; AVX512BW-NEXT: vmovdqa 512(%rdi), %ymm5
+; AVX512BW-NEXT: vmovdqa64 512(%rdi), %ymm5
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm5, %xmm5
; AVX512BW-NEXT: vinserti32x4 $0, %xmm5, %zmm25, %zmm23
@@ -4543,8 +4543,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm25 = [4,5,6,13,4,5,6,13]
; AVX512BW-NEXT: # zmm25 = mem[0,1,2,3,0,1,2,3]
; AVX512BW-NEXT: vpermt2q %zmm13, %zmm25, %zmm5
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm4
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm11
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm4
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm11
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3,4,5],ymm4[6,7]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm12 = [4,11]
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm10
@@ -4553,8 +4553,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vinserti64x4 $0, %ymm4, %zmm5, %zmm24
; AVX512BW-NEXT: vpermi2q %zmm6, %zmm0, %zmm18
; AVX512BW-NEXT: vpermt2q %zmm3, %zmm25, %zmm18
-; AVX512BW-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm4
-; AVX512BW-NEXT: vmovdqa 576(%rdi), %ymm5
+; AVX512BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm4
+; AVX512BW-NEXT: vmovdqa64 576(%rdi), %ymm5
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
; AVX512BW-NEXT: vpermi2q %zmm28, %zmm2, %zmm12
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3],ymm4[4,5,6,7]
@@ -4573,7 +4573,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vinserti64x4 $0, %ymm7, %zmm10, %zmm7
; AVX512BW-NEXT: vpermi2q %zmm0, %zmm6, %zmm20
; AVX512BW-NEXT: vpermt2q %zmm3, %zmm12, %zmm20
-; AVX512BW-NEXT: vmovdqa 640(%rdi), %ymm10
+; AVX512BW-NEXT: vmovdqa64 640(%rdi), %ymm10
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm5 = ymm5[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],ymm5[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vpermi2q %zmm28, %zmm2, %zmm4
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
@@ -4636,13 +4636,13 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm12[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm12, %zmm11, %zmm4
; AVX512BW-FCP-NEXT: vpermi2q %zmm30, %zmm1, %zmm10
-; AVX512BW-FCP-NEXT: vmovdqa 464(%rdi), %xmm11
+; AVX512BW-FCP-NEXT: vmovdqa64 464(%rdi), %xmm11
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm11, %zmm10, %zmm25
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm10
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm10
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm14 = [0,1,2,5]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm11
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm12
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm12
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm8 = ymm11[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpermt2q %ymm10, %ymm14, %ymm11
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm10 = [4,11]
@@ -4761,7 +4761,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm14, %zmm7
; AVX512BW-FCP-NEXT: vpermi2q %zmm0, %zmm5, %zmm24
; AVX512BW-FCP-NEXT: vpermt2q %zmm2, %zmm23, %zmm24
-; AVX512BW-FCP-NEXT: vmovdqa 640(%rdi), %ymm14
+; AVX512BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm14
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm14 = ymm28[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],ymm28[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpermi2q %zmm31, %zmm12, %zmm4
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm14[4,5,6,7]
@@ -4887,7 +4887,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm3, %zmm23, %zmm22
; AVX512DQ-BW-NEXT: vpermi2q %zmm26, %zmm4, %zmm25
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 464(%rdi), %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 464(%rdi), %xmm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm5, %zmm25, %zmm21
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm22, %zmm21 {%k1}
@@ -4908,7 +4908,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermi2q %zmm6, %zmm0, %zmm24
; AVX512DQ-BW-NEXT: vpermt2q %zmm3, %zmm23, %zmm24
; AVX512DQ-BW-NEXT: vpermi2q %zmm4, %zmm26, %zmm25
-; AVX512DQ-BW-NEXT: vmovdqa 512(%rdi), %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %ymm5
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm5, %xmm5
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm5, %zmm25, %zmm23
@@ -4918,8 +4918,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm25 = [4,5,6,13,4,5,6,13]
; AVX512DQ-BW-NEXT: # zmm25 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-BW-NEXT: vpermt2q %zmm13, %zmm25, %zmm5
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm11
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm11
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3,4,5],ymm4[6,7]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm12 = [4,11]
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm10
@@ -4928,8 +4928,8 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm4, %zmm5, %zmm24
; AVX512DQ-BW-NEXT: vpermi2q %zmm6, %zmm0, %zmm18
; AVX512DQ-BW-NEXT: vpermt2q %zmm3, %zmm25, %zmm18
-; AVX512DQ-BW-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 576(%rdi), %ymm5
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %ymm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
; AVX512DQ-BW-NEXT: vpermi2q %zmm28, %zmm2, %zmm12
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3],ymm4[4,5,6,7]
@@ -4948,7 +4948,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm7, %zmm10, %zmm7
; AVX512DQ-BW-NEXT: vpermi2q %zmm0, %zmm6, %zmm20
; AVX512DQ-BW-NEXT: vpermt2q %zmm3, %zmm12, %zmm20
-; AVX512DQ-BW-NEXT: vmovdqa 640(%rdi), %ymm10
+; AVX512DQ-BW-NEXT: vmovdqa64 640(%rdi), %ymm10
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm5 = ymm5[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],ymm5[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vpermi2q %zmm28, %zmm2, %zmm4
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
@@ -5011,13 +5011,13 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm12 = xmm12[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm12, %zmm11, %zmm4
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm30, %zmm1, %zmm10
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 464(%rdi), %xmm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 464(%rdi), %xmm11
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm11, %zmm10, %zmm25
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm10
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm10
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm14 = [0,1,2,5]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm11
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm12
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm12
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm8 = ymm11[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm10, %ymm14, %ymm11
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm10 = [4,11]
@@ -5136,7 +5136,7 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm14, %zmm7
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm0, %zmm5, %zmm24
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm2, %zmm23, %zmm24
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 640(%rdi), %ymm14
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm14
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm14 = ymm28[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],ymm28[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm31, %zmm12, %zmm4
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm14[4,5,6,7]
@@ -7986,22 +7986,22 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm9 {%k1} = zmm1[4,5,4,5],zmm18[4,5,4,5]
; AVX512-NEXT: vpermt2q %zmm18, %zmm21, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 912(%rdi), %xmm11
+; AVX512-NEXT: vmovdqa64 912(%rdi), %xmm11
; AVX512-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm11, %zmm16, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 464(%rdi), %xmm11
+; AVX512-NEXT: vmovdqa64 464(%rdi), %xmm11
; AVX512-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm11, %zmm2, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vmovdqa 16(%rdi), %xmm2
; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm2, %zmm15, %zmm21
-; AVX512-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm2, %zmm0, %zmm16
-; AVX512-NEXT: vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
-; AVX512-NEXT: vmovdqa 1024(%rdi), %ymm7
+; AVX512-NEXT: vinserti32x4 $1, 1088(%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vmovdqa64 1024(%rdi), %ymm7
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm7[0,1,2,3,4,5],ymm0[6,7]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm0 = [4,11]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
@@ -8011,8 +8011,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3],ymm11[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm11, %zmm25, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm11
-; AVX512-NEXT: vmovdqa 576(%rdi), %ymm1
+; AVX512-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm11
+; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm1
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3,4,5],ymm11[6,7]
; AVX512-NEXT: vmovdqa64 %ymm1, %ymm25
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
@@ -8022,8 +8022,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm15[0,1,2,3],ymm11[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm11
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm11
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3,4,5],ymm11[6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -8033,15 +8033,15 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm11[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm14, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vinserti128 $1, 1536(%rdi), %ymm0, %ymm1
-; AVX512-NEXT: vmovdqa 1472(%rdi), %ymm11
+; AVX512-NEXT: vinserti32x4 $1, 1536(%rdi), %ymm0, %ymm1
+; AVX512-NEXT: vmovdqa64 1472(%rdi), %ymm11
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512-NEXT: vpermi2q %zmm28, %zmm8, %zmm0
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm30, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 1088(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 1088(%rdi), %ymm0
; AVX512-NEXT: vpalignr {{.*#+}} ymm14 = ymm7[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],ymm7[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm0 = [5,12]
; AVX512-NEXT: vmovdqa64 %zmm2, %zmm1
@@ -8054,10 +8054,10 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm15, %zmm0, %zmm6
; AVX512-NEXT: vpermt2q %zmm15, %zmm30, %zmm3
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 960(%rdi), %ymm15
+; AVX512-NEXT: vmovdqa64 960(%rdi), %ymm15
; AVX512-NEXT: vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm15, %xmm15
-; AVX512-NEXT: vmovdqa 512(%rdi), %ymm12
+; AVX512-NEXT: vmovdqa64 512(%rdi), %ymm12
; AVX512-NEXT: vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm12, %xmm12
; AVX512-NEXT: vmovdqa64 %zmm5, %zmm2
@@ -8120,10 +8120,10 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm2, %zmm9 {%k2}
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm24, %zmm1
-; AVX512-NEXT: vmovdqa 640(%rdi), %ymm14
+; AVX512-NEXT: vmovdqa64 640(%rdi), %ymm14
; AVX512-NEXT: vmovdqa64 %ymm25, %ymm2
; AVX512-NEXT: vpalignr {{.*#+}} ymm14 = ymm2[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],ymm2[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
-; AVX512-NEXT: vmovdqa 1408(%rdi), %ymm15
+; AVX512-NEXT: vmovdqa64 1408(%rdi), %ymm15
; AVX512-NEXT: vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm15, %xmm15
; AVX512-NEXT: vinserti32x4 $0, %xmm15, %zmm4, %zmm4
@@ -8132,13 +8132,13 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3],ymm14[4,5,6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm3, %zmm2, %zmm3
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm14
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm14
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
; AVX512-NEXT: # ymm14 = mem[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm14[4,5,6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm6, %zmm2
-; AVX512-NEXT: vmovdqa 1536(%rdi), %ymm14
+; AVX512-NEXT: vmovdqa64 1536(%rdi), %ymm14
; AVX512-NEXT: vpalignr {{.*#+}} ymm11 = ymm11[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
@@ -8465,24 +8465,24 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm28 {%k1} = zmm3[4,5,4,5],zmm24[4,5,4,5]
; AVX512-FCP-NEXT: vpermt2q %zmm24, %zmm5, %zmm3
; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 912(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 912(%rdi), %xmm5
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm5, %zmm14, %zmm3
; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 464(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 464(%rdi), %xmm5
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm5, %zmm2, %zmm30
; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm8, %zmm24
-; AVX512-FCP-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm23
-; AVX512-FCP-NEXT: vmovdqa 1024(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 1088(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm2
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm1[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1088(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm2
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,1,2,5]
; AVX512-FCP-NEXT: vpermt2q %ymm2, %ymm8, %ymm1
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} xmm2 = [4,11]
@@ -8493,11 +8493,11 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm25, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %ymm5
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %xmm5
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm8, %ymm1
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm3, %zmm5
@@ -8506,11 +8506,11 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm5
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm5
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm8, %ymm1
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm14, %zmm5
@@ -8519,7 +8519,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm16, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1536(%rdi), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm1
; AVX512-FCP-NEXT: vmovdqa64 1472(%rdi), %ymm25
; AVX512-FCP-NEXT: vpermi2q %ymm1, %ymm25, %ymm8
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -8541,10 +8541,10 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm13, %zmm4, %zmm5
; AVX512-FCP-NEXT: vpermt2q %zmm13, %zmm8, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 960(%rdi), %ymm14
+; AVX512-FCP-NEXT: vmovdqa64 960(%rdi), %ymm14
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm14 = mem[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm14, %xmm14
-; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %ymm13
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm13, %xmm13
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm12
@@ -8603,7 +8603,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm12 {%k2}
; AVX512-FCP-NEXT: vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm1 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm1 = ymm3[0,1,2,3],mem[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 1408(%rdi), %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 1408(%rdi), %ymm13
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm13, %xmm13
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm13, %zmm10, %zmm14
@@ -8619,7 +8619,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: # ymm2 = ymm5[0,1,2,3],mem[4,5,6,7]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm2
-; AVX512-FCP-NEXT: vmovdqa 1536(%rdi), %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm13
; AVX512-FCP-NEXT: vmovdqa64 %ymm25, %ymm0
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm0[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],ymm0[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm5[4,5,6,7]
@@ -8949,22 +8949,22 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm9 {%k1} = zmm1[4,5,4,5],zmm18[4,5,4,5]
; AVX512DQ-NEXT: vpermt2q %zmm18, %zmm21, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 912(%rdi), %xmm11
+; AVX512DQ-NEXT: vmovdqa64 912(%rdi), %xmm11
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm11, %zmm16, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm11
+; AVX512DQ-NEXT: vmovdqa64 464(%rdi), %xmm11
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm11, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm2, %zmm15, %zmm21
-; AVX512DQ-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm2, %zmm0, %zmm16
-; AVX512DQ-NEXT: vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
-; AVX512DQ-NEXT: vmovdqa 1024(%rdi), %ymm7
+; AVX512DQ-NEXT: vinserti32x4 $1, 1088(%rdi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vmovdqa64 1024(%rdi), %ymm7
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm7[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm0 = [4,11]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
@@ -8974,8 +8974,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3],ymm11[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm11, %zmm25, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm11
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %ymm1
+; AVX512DQ-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm11
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm1
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3,4,5],ymm11[6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm1, %ymm25
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
@@ -8985,8 +8985,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm15[0,1,2,3],ymm11[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm11
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm11
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3,4,5],ymm11[6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -8996,15 +8996,15 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm11[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm14, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 1536(%rdi), %ymm0, %ymm1
-; AVX512DQ-NEXT: vmovdqa 1472(%rdi), %ymm11
+; AVX512DQ-NEXT: vinserti32x4 $1, 1536(%rdi), %ymm0, %ymm1
+; AVX512DQ-NEXT: vmovdqa64 1472(%rdi), %ymm11
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-NEXT: vpermi2q %zmm28, %zmm8, %zmm0
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm30, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 1088(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 1088(%rdi), %ymm0
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm14 = ymm7[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],ymm7[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm0 = [5,12]
; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm1
@@ -9017,10 +9017,10 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm15, %zmm0, %zmm6
; AVX512DQ-NEXT: vpermt2q %zmm15, %zmm30, %zmm3
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 960(%rdi), %ymm15
+; AVX512DQ-NEXT: vmovdqa64 960(%rdi), %ymm15
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm15, %xmm15
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %ymm12
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %ymm12
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm12, %xmm12
; AVX512DQ-NEXT: vmovdqa64 %zmm5, %zmm2
@@ -9083,10 +9083,10 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm9 {%k2}
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm24, %zmm1
-; AVX512DQ-NEXT: vmovdqa 640(%rdi), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %ymm14
; AVX512DQ-NEXT: vmovdqa64 %ymm25, %ymm2
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm14 = ymm2[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],ymm2[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
-; AVX512DQ-NEXT: vmovdqa 1408(%rdi), %ymm15
+; AVX512DQ-NEXT: vmovdqa64 1408(%rdi), %ymm15
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm15, %xmm15
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm15, %zmm4, %zmm4
@@ -9095,13 +9095,13 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3],ymm14[4,5,6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm3, %zmm2, %zmm3
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm14
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm14 = mem[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm14[4,5,6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm6, %zmm2
-; AVX512DQ-NEXT: vmovdqa 1536(%rdi), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 1536(%rdi), %ymm14
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm11 = ymm11[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
@@ -9428,24 +9428,24 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm28 {%k1} = zmm3[4,5,4,5],zmm24[4,5,4,5]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm24, %zmm5, %zmm3
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 912(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 912(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm5, %zmm14, %zmm3
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 464(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 464(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm5, %zmm2, %zmm30
; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm8, %zmm24
-; AVX512DQ-FCP-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm23
-; AVX512DQ-FCP-NEXT: vmovdqa 1024(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 1088(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm1[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1088(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,1,2,5]
; AVX512DQ-FCP-NEXT: vpermt2q %ymm2, %ymm8, %ymm1
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} xmm2 = [4,11]
@@ -9456,11 +9456,11 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm25, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm8, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, %zmm5
@@ -9469,11 +9469,11 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm8, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm14, %zmm5
@@ -9482,7 +9482,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm16, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1536(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 1472(%rdi), %ymm25
; AVX512DQ-FCP-NEXT: vpermi2q %ymm1, %ymm25, %ymm8
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -9504,10 +9504,10 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm13, %zmm4, %zmm5
; AVX512DQ-FCP-NEXT: vpermt2q %zmm13, %zmm8, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 960(%rdi), %ymm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 960(%rdi), %ymm14
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm14 = mem[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm14, %xmm14
-; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm13, %xmm13
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm12
@@ -9566,7 +9566,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm12 {%k2}
; AVX512DQ-FCP-NEXT: vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm1 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm1 = ymm3[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 1408(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 1408(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm13, %xmm13
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm13, %zmm10, %zmm14
@@ -9582,7 +9582,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: # ymm2 = ymm5[0,1,2,3],mem[4,5,6,7]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm2
-; AVX512DQ-FCP-NEXT: vmovdqa 1536(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm25, %ymm0
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm0[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],ymm0[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm5[4,5,6,7]
@@ -9902,22 +9902,22 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm14 {%k1} = zmm1[4,5,4,5],zmm13[4,5,4,5]
; AVX512BW-NEXT: vpermt2q %zmm13, %zmm5, %zmm1
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 912(%rdi), %xmm5
+; AVX512BW-NEXT: vmovdqa64 912(%rdi), %xmm5
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm5, %zmm19, %zmm1
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 464(%rdi), %xmm5
+; AVX512BW-NEXT: vmovdqa64 464(%rdi), %xmm5
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm5, %zmm3, %zmm1
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm3
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm3, %zmm18, %zmm19
-; AVX512BW-NEXT: vmovdqa 1360(%rdi), %xmm3
+; AVX512BW-NEXT: vmovdqa64 1360(%rdi), %xmm3
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm3, %zmm0, %zmm18
-; AVX512BW-NEXT: vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 1024(%rdi), %ymm13
+; AVX512BW-NEXT: vinserti32x4 $1, 1088(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 1024(%rdi), %ymm13
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm3 = [4,11]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -9927,8 +9927,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm15, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 576(%rdi), %ymm1
+; AVX512BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 576(%rdi), %ymm1
; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
@@ -9938,8 +9938,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm11
@@ -9948,8 +9948,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vinserti128 $1, 1536(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 1472(%rdi), %ymm2
+; AVX512BW-NEXT: vinserti32x4 $1, 1536(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 1472(%rdi), %ymm2
; AVX512BW-NEXT: vmovdqu %ymm2, (%rsp) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
@@ -10021,12 +10021,12 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm19 {%k2}
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm18 {%k2}
-; AVX512BW-NEXT: vmovdqa 960(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 960(%rdi), %ymm1
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm1, %xmm1
; AVX512BW-NEXT: vinserti32x4 $0, %xmm1, %zmm20, %zmm1
; AVX512BW-NEXT: vmovdqa64 %zmm31, %zmm1 {%k2}
-; AVX512BW-NEXT: vmovdqa 512(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 512(%rdi), %ymm6
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512BW-NEXT: vinserti32x4 $0, %xmm6, %zmm27, %zmm6
@@ -10050,12 +10050,12 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm13[4,5,6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm4, %zmm0, %zmm4
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm13
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm13
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm12 = ymm12[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm12[4,5,6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm3, %zmm0, %zmm3
-; AVX512BW-NEXT: vmovdqa 1536(%rdi), %ymm12
+; AVX512BW-NEXT: vmovdqa64 1536(%rdi), %ymm12
; AVX512BW-NEXT: vpalignr $8, (%rsp), %ymm12, %ymm11 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm11 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm11[4,5,6,7]
@@ -10381,23 +10381,23 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm30 {%k1} = zmm4[4,5,4,5],zmm15[4,5,4,5]
; AVX512BW-FCP-NEXT: vpermt2q %zmm15, %zmm2, %zmm4
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm6, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm1, %zmm3, %zmm15
-; AVX512BW-FCP-NEXT: vmovdqa 1360(%rdi), %xmm1
+; AVX512BW-FCP-NEXT: vmovdqa64 1360(%rdi), %xmm1
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm17
-; AVX512BW-FCP-NEXT: vmovdqa 1088(%rdi), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm0
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [0,1,2,5]
-; AVX512BW-FCP-NEXT: vmovdqa 1024(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm7
; AVX512BW-FCP-NEXT: vmovdqa %ymm7, %ymm2
; AVX512BW-FCP-NEXT: vpermt2q %ymm0, %ymm4, %ymm2
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm1 = [4,11]
@@ -10408,7 +10408,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm14, %zmm0
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 640(%rdi), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 640(%rdi), %xmm2
; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm23
; AVX512BW-FCP-NEXT: vmovdqa64 %ymm23, %ymm3
; AVX512BW-FCP-NEXT: vpermt2q %ymm2, %ymm4, %ymm3
@@ -10444,23 +10444,23 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm10, %zmm0, %zmm0
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm6
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm6
; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm21
; AVX512BW-FCP-NEXT: vmovdqa64 %ymm21, %ymm8
; AVX512BW-FCP-NEXT: vpermt2q %ymm6, %ymm4, %ymm8
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-FCP-NEXT: vmovdqa 1088(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm6 = ymm7[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],ymm7[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm0
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 640(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm2
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm23[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],ymm23[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512BW-FCP-NEXT: vmovdqa 1536(%rdi), %xmm7
-; AVX512BW-FCP-NEXT: vmovdqa 1472(%rdi), %ymm8
+; AVX512BW-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm7
+; AVX512BW-FCP-NEXT: vmovdqa64 1472(%rdi), %ymm8
; AVX512BW-FCP-NEXT: vpermi2q %ymm7, %ymm8, %ymm4
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm21[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],ymm21[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm4, %zmm23
@@ -10504,12 +10504,12 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm15 {%k2}
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm17 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 960(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 960(%rdi), %ymm2
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm2, %xmm2
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm18, %zmm2
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm29, %zmm2 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 512(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %ymm3
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm3 = mem[8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm3, %xmm3
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm3, %zmm26, %zmm3
@@ -10521,7 +10521,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm5, %zmm24, %zmm5
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm5 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 1408(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 1408(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm6, %zmm11, %zmm6
@@ -10532,7 +10532,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm0[4,5,6,7]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
-; AVX512BW-FCP-NEXT: vmovdqa 1536(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm7
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm7 = ymm8[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],ymm8[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm7[4,5,6,7]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -10850,22 +10850,22 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm14 {%k1} = zmm1[4,5,4,5],zmm13[4,5,4,5]
; AVX512DQ-BW-NEXT: vpermt2q %zmm13, %zmm5, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 912(%rdi), %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 912(%rdi), %xmm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm5, %zmm19, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 464(%rdi), %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 464(%rdi), %xmm5
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm5, %zmm3, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm3
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm3, %zmm18, %zmm19
-; AVX512DQ-BW-NEXT: vmovdqa 1360(%rdi), %xmm3
+; AVX512DQ-BW-NEXT: vmovdqa64 1360(%rdi), %xmm3
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm3, %zmm0, %zmm18
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 1024(%rdi), %ymm13
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1088(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 1024(%rdi), %ymm13
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm3 = [4,11]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -10875,8 +10875,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm15, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 576(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
@@ -10886,8 +10886,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm2, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm11
@@ -10896,8 +10896,8 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1536(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 1472(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1536(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 1472(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vmovdqu %ymm2, (%rsp) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
@@ -10969,12 +10969,12 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm19 {%k2}
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm18 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 960(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 960(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm1, %xmm1
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm1, %zmm20, %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm31, %zmm1 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 512(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm6, %zmm27, %zmm6
@@ -10998,12 +10998,12 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm13[4,5,6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm4, %zmm0, %zmm4
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm13
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm13
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm12 = ymm12[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm12[4,5,6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm3, %zmm0, %zmm3
-; AVX512DQ-BW-NEXT: vmovdqa 1536(%rdi), %ymm12
+; AVX512DQ-BW-NEXT: vmovdqa64 1536(%rdi), %ymm12
; AVX512DQ-BW-NEXT: vpalignr $8, (%rsp), %ymm12, %ymm11 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm11 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm11[4,5,6,7]
@@ -11329,23 +11329,23 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm30 {%k1} = zmm4[4,5,4,5],zmm15[4,5,4,5]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm15, %zmm2, %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm6, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm1, %zmm3, %zmm15
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1360(%rdi), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1360(%rdi), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm17
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1088(%rdi), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [0,1,2,5]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1024(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm7
; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm7, %ymm2
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm0, %ymm4, %ymm2
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm1 = [4,11]
@@ -11356,7 +11356,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm14, %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 640(%rdi), %xmm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 640(%rdi), %xmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm23
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm23, %ymm3
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm2, %ymm4, %ymm3
@@ -11392,23 +11392,23 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm10, %zmm0, %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm6
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm21
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm21, %ymm8
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm6, %ymm4, %ymm8
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm8[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1088(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm6 = ymm7[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],ymm7[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 640(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm23[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],ymm23[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1536(%rdi), %xmm7
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1472(%rdi), %ymm8
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1472(%rdi), %ymm8
; AVX512DQ-BW-FCP-NEXT: vpermi2q %ymm7, %ymm8, %ymm4
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm21[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],ymm21[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm4, %zmm23
@@ -11452,12 +11452,12 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm15 {%k2}
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm17 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 960(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 960(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm2, %xmm2
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm18, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm29, %zmm2 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 512(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %ymm3
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm3 = mem[8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm3, %xmm3
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm3, %zmm26, %zmm3
@@ -11469,7 +11469,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm5, %zmm24, %zmm5
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm5 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1408(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1408(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm6, %zmm11, %zmm6
@@ -11480,7 +11480,7 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm4, %zmm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1536(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm7
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm7 = ymm8[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],ymm8[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm7[4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -17185,7 +17185,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm10, %zmm0, %zmm6
-; AVX512-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm2, %zmm6, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17197,13 +17197,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vmovdqa64 %zmm13, %zmm1
; AVX512-NEXT: vpermt2q %zmm12, %zmm0, %zmm1
-; AVX512-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vmovdqa64 %zmm4, %zmm1
; AVX512-NEXT: vpermt2q %zmm14, %zmm0, %zmm1
-; AVX512-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17215,7 +17215,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vmovdqa64 %zmm3, %zmm1
; AVX512-NEXT: vpermt2q %zmm11, %zmm0, %zmm1
-; AVX512-NEXT: vmovdqa 1808(%rdi), %xmm2
+; AVX512-NEXT: vmovdqa64 1808(%rdi), %xmm2
; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17225,8 +17225,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vinserti128 $1, 2880(%rdi), %ymm0, %ymm1
-; AVX512-NEXT: vmovdqa 2816(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 2816(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm9 = [4,11]
; AVX512-NEXT: vpermt2q %zmm18, %zmm9, %zmm7
@@ -17244,9 +17244,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm3, %zmm4, %zmm2
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm2
-; AVX512-NEXT: vmovdqa 576(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm2
+; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6,7]
; AVX512-NEXT: vmovdqa64 512(%rdi), %zmm16
; AVX512-NEXT: vmovdqa64 448(%rdi), %zmm3
@@ -17262,9 +17262,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm5, %zmm4, %zmm3
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm3, %zmm2
; AVX512-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm3
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm3
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm2
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1,2,3,4,5],ymm3[6,7]
; AVX512-NEXT: vmovdqa64 64(%rdi), %zmm23
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm2
@@ -17282,9 +17282,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm6, %zmm4, %zmm5
; AVX512-NEXT: vinserti64x4 $0, %ymm3, %zmm5, %zmm3
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vinserti128 $1, 1536(%rdi), %ymm0, %ymm5
-; AVX512-NEXT: vmovdqa 1472(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vinserti32x4 $1, 1536(%rdi), %ymm0, %ymm5
+; AVX512-NEXT: vmovdqa64 1472(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm3[0,1,2,3,4,5],ymm5[6,7]
; AVX512-NEXT: vmovdqa64 1408(%rdi), %zmm19
; AVX512-NEXT: vmovdqa64 1344(%rdi), %zmm7
@@ -17300,9 +17300,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm3, %zmm4, %zmm8
; AVX512-NEXT: vinserti64x4 $0, %ymm6, %zmm8, %zmm5
; AVX512-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vinserti128 $1, 1088(%rdi), %ymm0, %ymm6
-; AVX512-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vinserti32x4 $1, 1088(%rdi), %ymm0, %ymm6
+; AVX512-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm3[0,1,2,3,4,5],ymm6[6,7]
; AVX512-NEXT: vmovdqa64 960(%rdi), %zmm30
; AVX512-NEXT: vmovdqa64 896(%rdi), %zmm1
@@ -17320,8 +17320,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm10, %zmm5
; AVX512-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vinserti128 $1, 2432(%rdi), %ymm0, %ymm8
-; AVX512-NEXT: vmovdqa 2368(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 2368(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5],ymm8[6,7]
; AVX512-NEXT: vmovdqa64 2304(%rdi), %zmm18
; AVX512-NEXT: vmovdqa64 2240(%rdi), %zmm0
@@ -17338,9 +17338,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vinserti64x4 $0, %ymm10, %zmm14, %zmm5
; AVX512-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vinserti128 $1, 1984(%rdi), %ymm0, %ymm10
-; AVX512-NEXT: vmovdqa 1920(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vinserti32x4 $1, 1984(%rdi), %ymm0, %ymm10
+; AVX512-NEXT: vmovdqa64 1920(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4,5],ymm10[6,7]
; AVX512-NEXT: vmovdqa64 1856(%rdi), %zmm22
; AVX512-NEXT: vmovdqa64 1792(%rdi), %zmm5
@@ -17359,8 +17359,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti64x4 $0, %ymm10, %zmm15, %zmm5
; AVX512-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vinserti128 $1, 3328(%rdi), %ymm0, %ymm10
-; AVX512-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4,5],ymm10[6,7]
; AVX512-NEXT: vmovdqa64 3200(%rdi), %zmm25
; AVX512-NEXT: vmovdqa64 3136(%rdi), %zmm14
@@ -17402,7 +17402,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm6, %zmm4, %zmm11
; AVX512-NEXT: vinserti64x4 $0, %ymm10, %zmm11, %zmm4
; AVX512-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 2880(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 2880(%rdi), %ymm4
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm0 # 32-byte Folded Reload
; AVX512-NEXT: # ymm0 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm11 = [5,12]
@@ -17418,7 +17418,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm16 # 64-byte Folded Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm10, %zmm16, %zmm5
; AVX512-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 640(%rdi), %ymm10
+; AVX512-NEXT: vmovdqa64 640(%rdi), %ymm10
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vmovdqa64 %zmm8, %zmm10
@@ -17432,7 +17432,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm10 # 64-byte Folded Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm10, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vmovdqa64 %zmm2, %zmm5
@@ -17445,7 +17445,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vmovdqa64 %zmm3, %zmm8
@@ -17459,7 +17459,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 1088(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 1088(%rdi), %ymm1
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
@@ -17472,7 +17472,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 2432(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 2432(%rdi), %ymm1
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -17485,7 +17485,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm21, %zmm4, %zmm2
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 1984(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 1984(%rdi), %ymm1
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
@@ -17498,7 +17498,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm28, %zmm4, %zmm2
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 3328(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 3328(%rdi), %ymm1
; AVX512-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17971,7 +17971,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm25 {%k1}
; AVX512-NEXT: vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm0 # 32-byte Folded Reload
; AVX512-NEXT: # ymm0 = ymm11[0,1,2,3],mem[4,5,6,7]
-; AVX512-NEXT: vmovdqa 2752(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 2752(%rdi), %ymm3
; AVX512-NEXT: vpalignr {{.*#+}} ymm3 = mem[8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm3
; AVX512-NEXT: vinserti32x4 $0, %xmm3, %zmm26, %zmm3
@@ -17979,7 +17979,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; AVX512-NEXT: vmovdqa 512(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 512(%rdi), %ymm1
; AVX512-NEXT: vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
; AVX512-NEXT: vinserti32x4 $0, %xmm1, %zmm20, %zmm6
@@ -17995,7 +17995,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm4, %zmm14 {%k1}
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm4, %zmm1
-; AVX512-NEXT: vmovdqa 1408(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 1408(%rdi), %ymm4
; AVX512-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512-NEXT: vinserti32x4 $0, %xmm4, %zmm17, %zmm5
@@ -18004,7 +18004,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512-NEXT: vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
; AVX512-NEXT: # ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
-; AVX512-NEXT: vmovdqa 960(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 960(%rdi), %ymm8
; AVX512-NEXT: vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm8, %xmm8
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -18013,7 +18013,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512-NEXT: vinsertf64x4 $0, %ymm4, %zmm9, %zmm4
-; AVX512-NEXT: vmovdqa 2304(%rdi), %ymm10
+; AVX512-NEXT: vmovdqa64 2304(%rdi), %ymm10
; AVX512-NEXT: vpalignr {{.*#+}} ymm10 = mem[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm10
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -18023,7 +18023,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512-NEXT: vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm11 # 32-byte Folded Reload
; AVX512-NEXT: # ymm11 = mem[0,1,2,3],ymm9[4,5,6,7]
-; AVX512-NEXT: vmovdqa 1856(%rdi), %ymm12
+; AVX512-NEXT: vmovdqa64 1856(%rdi), %ymm12
; AVX512-NEXT: vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm12, %xmm12
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -18032,7 +18032,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm9, %zmm12 {%k1}
; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512-NEXT: vinsertf64x4 $0, %ymm11, %zmm9, %zmm11
-; AVX512-NEXT: vmovdqa 3200(%rdi), %ymm13
+; AVX512-NEXT: vmovdqa64 3200(%rdi), %ymm13
; AVX512-NEXT: vpalignr {{.*#+}} ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512-NEXT: vextracti128 $1, %ymm13, %xmm13
; AVX512-NEXT: vinserti32x4 $0, %xmm13, %zmm15, %zmm9
@@ -18176,7 +18176,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm13, %zmm0, %zmm8
-; AVX512-FCP-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm8, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18187,13 +18187,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 %zmm14, %zmm1
; AVX512-FCP-NEXT: vpermt2q %zmm6, %zmm0, %zmm1
-; AVX512-FCP-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512-FCP-NEXT: vpermt2q %zmm7, %zmm0, %zmm1
-; AVX512-FCP-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18205,7 +18205,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 %zmm4, %zmm1
; AVX512-FCP-NEXT: vpermt2q %zmm16, %zmm0, %zmm1
-; AVX512-FCP-NEXT: vmovdqa 1808(%rdi), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 1808(%rdi), %xmm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18216,10 +18216,10 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 2880(%rdi), %xmm2
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm10 = [0,1,2,5]
-; AVX512-FCP-NEXT: vmovdqa 2816(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqa 2880(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 2816(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 2880(%rdi), %ymm1
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm3[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],ymm3[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermt2q %ymm2, %ymm10, %ymm3
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} xmm6 = [4,11]
; AVX512-FCP-NEXT: vmovdqa64 %zmm15, %zmm1
@@ -18237,11 +18237,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm5, %zmm3, %zmm4
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm4, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %ymm4
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm1[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm31
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %xmm5
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %zmm22
; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %zmm7
@@ -18257,11 +18257,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm8, %zmm3, %zmm5
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm5
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm5
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %zmm28
; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -18277,11 +18277,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm8, %zmm3, %zmm5
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1472(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 1536(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 1472(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm5
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1536(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm5
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 1408(%rdi), %zmm24
; AVX512-FCP-NEXT: vmovdqa64 1344(%rdi), %zmm11
@@ -18297,11 +18297,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm8, %zmm3, %zmm5
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1024(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 1088(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm5
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1088(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm5
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 960(%rdi), %zmm17
; AVX512-FCP-NEXT: vmovdqa64 896(%rdi), %zmm13
@@ -18317,10 +18317,10 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm8, %zmm3, %zmm5
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 2368(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 2432(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 2368(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 2432(%rdi), %ymm5
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 2432(%rdi), %xmm5
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 2304(%rdi), %zmm18
@@ -18338,11 +18338,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm5, %zmm3, %zmm8
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm8, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1920(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 1984(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 1920(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 1984(%rdi), %ymm8
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1984(%rdi), %xmm8
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1984(%rdi), %xmm8
; AVX512-FCP-NEXT: vpermt2q %ymm8, %ymm10, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 1856(%rdi), %zmm16
; AVX512-FCP-NEXT: vmovdqa64 1792(%rdi), %zmm9
@@ -18359,8 +18359,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm8, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 3328(%rdi), %xmm8
-; AVX512-FCP-NEXT: vmovdqa 3264(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 3264(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermi2q %ymm8, %ymm1, %ymm10
; AVX512-FCP-NEXT: vmovdqa64 3200(%rdi), %zmm19
; AVX512-FCP-NEXT: vmovdqa64 3136(%rdi), %zmm0
@@ -18491,7 +18491,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm12, %zmm2, %zmm6
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm6, %zmm4
; AVX512-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 3328(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 3328(%rdi), %ymm4
; AVX512-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm1 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
@@ -18961,7 +18961,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm31, %zmm28 {%k1}
; AVX512-FCP-NEXT: vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm0 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm0 = ymm10[0,1,2,3],mem[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 2752(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 2752(%rdi), %ymm3
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm3 = mem[8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm3, %xmm3
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm3, %zmm16, %zmm3
@@ -18969,7 +18969,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm16
-; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %ymm0
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm0, %zmm18, %zmm5
@@ -18985,7 +18985,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm15 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm1
-; AVX512-FCP-NEXT: vmovdqa 1408(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 1408(%rdi), %ymm0
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
@@ -18995,7 +18995,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512-FCP-NEXT: vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 960(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 960(%rdi), %ymm8
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm8, %xmm8
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -19004,7 +19004,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512-FCP-NEXT: vinsertf64x4 $0, %ymm4, %zmm9, %zmm4
-; AVX512-FCP-NEXT: vmovdqa 2304(%rdi), %ymm9
+; AVX512-FCP-NEXT: vmovdqa64 2304(%rdi), %ymm9
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm9, %xmm9
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
@@ -19014,7 +19014,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512-FCP-NEXT: vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm10 = mem[0,1,2,3],ymm10[4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 1856(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 1856(%rdi), %ymm11
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm11 = mem[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm11, %xmm11
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
@@ -19023,7 +19023,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm12, %zmm11 {%k1}
; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
; AVX512-FCP-NEXT: vinsertf64x4 $0, %ymm10, %zmm12, %zmm10
-; AVX512-FCP-NEXT: vmovdqa 3200(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 3200(%rdi), %ymm12
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm12, %xmm12
; AVX512-FCP-NEXT: vinserti32x4 $0, %xmm12, %zmm22, %zmm12
@@ -19170,7 +19170,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm10, %zmm0, %zmm6
-; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm2, %zmm6, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -19182,13 +19182,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vmovdqa64 %zmm13, %zmm1
; AVX512DQ-NEXT: vpermt2q %zmm12, %zmm0, %zmm1
-; AVX512DQ-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vmovdqa64 %zmm4, %zmm1
; AVX512DQ-NEXT: vpermt2q %zmm14, %zmm0, %zmm1
-; AVX512DQ-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -19200,7 +19200,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vmovdqa64 %zmm3, %zmm1
; AVX512DQ-NEXT: vpermt2q %zmm11, %zmm0, %zmm1
-; AVX512DQ-NEXT: vmovdqa 1808(%rdi), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 1808(%rdi), %xmm2
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -19210,8 +19210,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vinserti128 $1, 2880(%rdi), %ymm0, %ymm1
-; AVX512DQ-NEXT: vmovdqa 2816(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 2816(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm9 = [4,11]
; AVX512DQ-NEXT: vpermt2q %zmm18, %zmm9, %zmm7
@@ -19229,9 +19229,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm3, %zmm4, %zmm2
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm2
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm2
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6,7]
; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %zmm16
; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %zmm3
@@ -19247,9 +19247,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm5, %zmm4, %zmm3
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm3, %zmm2
; AVX512DQ-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm3
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm3
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0,1,2,3,4,5],ymm3[6,7]
; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %zmm23
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm2
@@ -19267,9 +19267,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm6, %zmm4, %zmm5
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm3, %zmm5, %zmm3
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 1536(%rdi), %ymm0, %ymm5
-; AVX512DQ-NEXT: vmovdqa 1472(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vinserti32x4 $1, 1536(%rdi), %ymm0, %ymm5
+; AVX512DQ-NEXT: vmovdqa64 1472(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm3[0,1,2,3,4,5],ymm5[6,7]
; AVX512DQ-NEXT: vmovdqa64 1408(%rdi), %zmm19
; AVX512DQ-NEXT: vmovdqa64 1344(%rdi), %zmm7
@@ -19285,9 +19285,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm3, %zmm4, %zmm8
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm6, %zmm8, %zmm5
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 1088(%rdi), %ymm0, %ymm6
-; AVX512DQ-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vinserti32x4 $1, 1088(%rdi), %ymm0, %ymm6
+; AVX512DQ-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm3[0,1,2,3,4,5],ymm6[6,7]
; AVX512DQ-NEXT: vmovdqa64 960(%rdi), %zmm30
; AVX512DQ-NEXT: vmovdqa64 896(%rdi), %zmm1
@@ -19305,8 +19305,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm8, %zmm10, %zmm5
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vinserti128 $1, 2432(%rdi), %ymm0, %ymm8
-; AVX512DQ-NEXT: vmovdqa 2368(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 2368(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5],ymm8[6,7]
; AVX512DQ-NEXT: vmovdqa64 2304(%rdi), %zmm18
; AVX512DQ-NEXT: vmovdqa64 2240(%rdi), %zmm0
@@ -19323,9 +19323,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm10, %zmm14, %zmm5
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vinserti128 $1, 1984(%rdi), %ymm0, %ymm10
-; AVX512DQ-NEXT: vmovdqa 1920(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vinserti32x4 $1, 1984(%rdi), %ymm0, %ymm10
+; AVX512DQ-NEXT: vmovdqa64 1920(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4,5],ymm10[6,7]
; AVX512DQ-NEXT: vmovdqa64 1856(%rdi), %zmm22
; AVX512DQ-NEXT: vmovdqa64 1792(%rdi), %zmm5
@@ -19344,8 +19344,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm10, %zmm15, %zmm5
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vinserti128 $1, 3328(%rdi), %ymm0, %ymm10
-; AVX512DQ-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4,5],ymm10[6,7]
; AVX512DQ-NEXT: vmovdqa64 3200(%rdi), %zmm25
; AVX512DQ-NEXT: vmovdqa64 3136(%rdi), %zmm14
@@ -19387,7 +19387,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm6, %zmm4, %zmm11
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm10, %zmm11, %zmm4
; AVX512DQ-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 2880(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 2880(%rdi), %ymm4
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm0 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm0 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm11 = [5,12]
@@ -19403,7 +19403,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm16 # 64-byte Folded Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm10, %zmm16, %zmm5
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 640(%rdi), %ymm10
+; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %ymm10
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vmovdqa64 %zmm8, %zmm10
@@ -19417,7 +19417,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm10 # 64-byte Folded Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm10, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm5
@@ -19430,7 +19430,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vmovdqa64 %zmm3, %zmm8
@@ -19444,7 +19444,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 1088(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 1088(%rdi), %ymm1
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
@@ -19457,7 +19457,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 2432(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 2432(%rdi), %ymm1
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -19470,7 +19470,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm21, %zmm4, %zmm2
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 1984(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 1984(%rdi), %ymm1
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
@@ -19483,7 +19483,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm28, %zmm4, %zmm2
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 3328(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 3328(%rdi), %ymm1
; AVX512DQ-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -19956,7 +19956,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm25 {%k1}
; AVX512DQ-NEXT: vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm0 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm0 = ymm11[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 2752(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqa64 2752(%rdi), %ymm3
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm3 = mem[8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm3, %xmm3
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm3, %zmm26, %zmm3
@@ -19964,7 +19964,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %ymm1
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm1
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm1, %zmm20, %zmm6
@@ -19980,7 +19980,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm4, %zmm14 {%k1}
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm4, %zmm1
-; AVX512DQ-NEXT: vmovdqa 1408(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 1408(%rdi), %ymm4
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm4, %zmm17, %zmm5
@@ -19989,7 +19989,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-NEXT: vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 960(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 960(%rdi), %ymm8
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm8, %xmm8
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -19998,7 +19998,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512DQ-NEXT: vinsertf64x4 $0, %ymm4, %zmm9, %zmm4
-; AVX512DQ-NEXT: vmovdqa 2304(%rdi), %ymm10
+; AVX512DQ-NEXT: vmovdqa64 2304(%rdi), %ymm10
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm10 = mem[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm10, %xmm10
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -20008,7 +20008,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512DQ-NEXT: vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm11 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm11 = mem[0,1,2,3],ymm9[4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 1856(%rdi), %ymm12
+; AVX512DQ-NEXT: vmovdqa64 1856(%rdi), %ymm12
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm12, %xmm12
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -20017,7 +20017,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm9, %zmm12 {%k1}
; AVX512DQ-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512DQ-NEXT: vinsertf64x4 $0, %ymm11, %zmm9, %zmm11
-; AVX512DQ-NEXT: vmovdqa 3200(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 3200(%rdi), %ymm13
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
; AVX512DQ-NEXT: vextracti128 $1, %ymm13, %xmm13
; AVX512DQ-NEXT: vinserti32x4 $0, %xmm13, %zmm15, %zmm9
@@ -20161,7 +20161,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm13, %zmm0, %zmm8
-; AVX512DQ-FCP-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm8, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -20172,13 +20172,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm14, %zmm1
; AVX512DQ-FCP-NEXT: vpermt2q %zmm6, %zmm0, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512DQ-FCP-NEXT: vpermt2q %zmm7, %zmm0, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -20190,7 +20190,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, %zmm1
; AVX512DQ-FCP-NEXT: vpermt2q %zmm16, %zmm0, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 1808(%rdi), %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 1808(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -20201,10 +20201,10 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 2880(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm10 = [0,1,2,5]
-; AVX512DQ-FCP-NEXT: vmovdqa 2816(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqa 2880(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 2816(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 2880(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm3[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],ymm3[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %ymm2, %ymm10, %ymm3
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} xmm6 = [4,11]
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm15, %zmm1
@@ -20222,11 +20222,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm5, %zmm3, %zmm4
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm4, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm1[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm31
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %zmm22
; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %zmm7
@@ -20242,11 +20242,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm8, %zmm3, %zmm5
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %zmm28
; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm4
@@ -20262,11 +20262,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm8, %zmm3, %zmm5
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1472(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 1536(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 1472(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1536(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 1408(%rdi), %zmm24
; AVX512DQ-FCP-NEXT: vmovdqa64 1344(%rdi), %zmm11
@@ -20282,11 +20282,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm8, %zmm3, %zmm5
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1024(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 1088(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1088(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 960(%rdi), %zmm17
; AVX512DQ-FCP-NEXT: vmovdqa64 896(%rdi), %zmm13
@@ -20302,10 +20302,10 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm8, %zmm3, %zmm5
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm5, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 2368(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 2432(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 2368(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 2432(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 2432(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm10, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 2304(%rdi), %zmm18
@@ -20323,11 +20323,11 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm5, %zmm3, %zmm8
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm8, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1920(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 1984(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 1920(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 1984(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm5 = ymm1[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1984(%rdi), %xmm8
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1984(%rdi), %xmm8
; AVX512DQ-FCP-NEXT: vpermt2q %ymm8, %ymm10, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 1856(%rdi), %zmm16
; AVX512DQ-FCP-NEXT: vmovdqa64 1792(%rdi), %zmm9
@@ -20344,8 +20344,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm8, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 3328(%rdi), %xmm8
-; AVX512DQ-FCP-NEXT: vmovdqa 3264(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 3264(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermi2q %ymm8, %ymm1, %ymm10
; AVX512DQ-FCP-NEXT: vmovdqa64 3200(%rdi), %zmm19
; AVX512DQ-FCP-NEXT: vmovdqa64 3136(%rdi), %zmm0
@@ -20476,7 +20476,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm12, %zmm2, %zmm6
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm6, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 3328(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 3328(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm1 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
@@ -20946,7 +20946,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm31, %zmm28 {%k1}
; AVX512DQ-FCP-NEXT: vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm0 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm0 = ymm10[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 2752(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 2752(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm3 = mem[8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm3, %xmm3
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm3, %zmm16, %zmm3
@@ -20954,7 +20954,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm16
-; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm0, %zmm18, %zmm5
@@ -20970,7 +20970,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm15 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 1408(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 1408(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
@@ -20980,7 +20980,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 960(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 960(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm8, %xmm8
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -20989,7 +20989,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinsertf64x4 $0, %ymm4, %zmm9, %zmm4
-; AVX512DQ-FCP-NEXT: vmovdqa 2304(%rdi), %ymm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 2304(%rdi), %ymm9
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm9, %xmm9
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
@@ -20999,7 +20999,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm10 = mem[0,1,2,3],ymm10[4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 1856(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 1856(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm11 = mem[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm11, %xmm11
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
@@ -21008,7 +21008,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm12, %zmm11 {%k1}
; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinsertf64x4 $0, %ymm10, %zmm12, %zmm10
-; AVX512DQ-FCP-NEXT: vmovdqa 3200(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 3200(%rdi), %ymm12
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm12, %xmm12
; AVX512DQ-FCP-NEXT: vinserti32x4 $0, %xmm12, %zmm22, %zmm12
@@ -21154,7 +21154,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vmovdqa64 %zmm14, %zmm1
; AVX512BW-NEXT: vpermt2q %zmm16, %zmm0, %zmm1
-; AVX512BW-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512BW-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21166,13 +21166,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vmovdqa64 %zmm12, %zmm1
; AVX512BW-NEXT: vpermt2q %zmm10, %zmm0, %zmm1
-; AVX512BW-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512BW-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vmovdqa64 %zmm13, %zmm1
; AVX512BW-NEXT: vpermt2q %zmm11, %zmm0, %zmm1
-; AVX512BW-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512BW-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21184,7 +21184,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vmovdqa64 %zmm3, %zmm1
; AVX512BW-NEXT: vpermt2q %zmm9, %zmm0, %zmm1
-; AVX512BW-NEXT: vmovdqa 1808(%rdi), %xmm2
+; AVX512BW-NEXT: vmovdqa64 1808(%rdi), %xmm2
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21194,8 +21194,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vinserti128 $1, 2880(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 2816(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 2816(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm9 = [4,11]
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm1
@@ -21215,9 +21215,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm2, %zmm4, %zmm1
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqa64 512(%rdi), %zmm27
; AVX512BW-NEXT: vmovdqa64 448(%rdi), %zmm2
@@ -21234,9 +21234,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm2, %zmm4, %zmm1
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm31
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm8
@@ -21253,9 +21253,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm2, %zmm4, %zmm1
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vinserti128 $1, 1536(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 1472(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vinserti32x4 $1, 1536(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 1472(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqa64 1408(%rdi), %zmm26
; AVX512BW-NEXT: vmovdqa64 1344(%rdi), %zmm7
@@ -21271,9 +21271,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm2, %zmm4, %zmm1
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 1024(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vinserti32x4 $1, 1088(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 1024(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqa64 960(%rdi), %zmm19
; AVX512BW-NEXT: vmovdqa64 896(%rdi), %zmm6
@@ -21291,8 +21291,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vinserti128 $1, 2432(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 2368(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 2368(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqa64 2304(%rdi), %zmm20
; AVX512BW-NEXT: vmovdqa64 2240(%rdi), %zmm5
@@ -21309,9 +21309,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm3, %zmm4, %zmm1
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vinserti128 $1, 1984(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 1920(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vinserti32x4 $1, 1984(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vmovdqa64 1920(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqa64 1856(%rdi), %zmm21
; AVX512BW-NEXT: vmovdqa64 1792(%rdi), %zmm2
@@ -21329,8 +21329,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vinserti128 $1, 3328(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT: vmovdqa 3264(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 3264(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512BW-NEXT: vmovdqa64 3200(%rdi), %zmm17
; AVX512BW-NEXT: vmovdqa64 3136(%rdi), %zmm11
@@ -21369,7 +21369,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm30, %zmm4, %zmm10
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm10, %zmm0
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 2880(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa64 2880(%rdi), %ymm0
; AVX512BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm4 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm0 = [5,12]
@@ -21386,7 +21386,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm5
; AVX512BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 640(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 640(%rdi), %ymm9
; AVX512BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vpermt2q %zmm27, %zmm0, %zmm1
@@ -21398,7 +21398,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm5
; AVX512BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21411,7 +21411,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm5
; AVX512BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 1536(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 1536(%rdi), %ymm9
; AVX512BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm5
@@ -21427,7 +21427,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm7
; AVX512BW-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 1088(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 1088(%rdi), %ymm9
; AVX512BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -21441,7 +21441,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm9
; AVX512BW-NEXT: vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 2432(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 2432(%rdi), %ymm9
; AVX512BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm2 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
@@ -21454,7 +21454,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm9 # 64-byte Folded Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm9, %zmm2
; AVX512BW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 1984(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 1984(%rdi), %ymm2
; AVX512BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -21467,7 +21467,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm23, %zmm10, %zmm3
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm3, %zmm2
; AVX512BW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 3328(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 3328(%rdi), %ymm2
; AVX512BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm1 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
@@ -21941,13 +21941,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm26 {%k2}
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm31, %zmm27 {%k2}
-; AVX512BW-NEXT: vmovdqa 2752(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa64 2752(%rdi), %ymm0
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512BW-NEXT: vinserti32x4 $0, %xmm0, %zmm28, %zmm1
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k2}
-; AVX512BW-NEXT: vmovdqa 512(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa64 512(%rdi), %ymm0
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -21961,35 +21961,35 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vinserti32x4 $0, %xmm3, %zmm4, %zmm3
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm4, %zmm3 {%k2}
-; AVX512BW-NEXT: vmovdqa 1408(%rdi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 1408(%rdi), %ymm4
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-NEXT: vinserti32x4 $0, %xmm4, %zmm5, %zmm4
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm4 {%k2}
-; AVX512BW-NEXT: vmovdqa 960(%rdi), %ymm5
+; AVX512BW-NEXT: vmovdqa64 960(%rdi), %ymm5
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm5, %xmm5
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512BW-NEXT: vinserti32x4 $0, %xmm5, %zmm6, %zmm5
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm6, %zmm5 {%k2}
-; AVX512BW-NEXT: vmovdqa 2304(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 2304(%rdi), %ymm6
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-NEXT: vinserti32x4 $0, %xmm6, %zmm7, %zmm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512BW-NEXT: vmovdqa 1856(%rdi), %ymm7
+; AVX512BW-NEXT: vmovdqa64 1856(%rdi), %ymm7
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm7, %xmm7
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512BW-NEXT: vinserti32x4 $0, %xmm7, %zmm8, %zmm7
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm8, %zmm7 {%k2}
-; AVX512BW-NEXT: vmovdqa 3200(%rdi), %ymm8
+; AVX512BW-NEXT: vmovdqa64 3200(%rdi), %ymm8
; AVX512BW-NEXT: vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512BW-NEXT: vextracti128 $1, %ymm8, %xmm8
; AVX512BW-NEXT: vinserti32x4 $0, %xmm8, %zmm30, %zmm8
@@ -22132,7 +22132,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm14, %zmm1
; AVX512BW-FCP-NEXT: vpermt2q %zmm10, %zmm0, %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -22143,13 +22143,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm12, %zmm1
; AVX512BW-FCP-NEXT: vpermt2q %zmm11, %zmm0, %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm1
; AVX512BW-FCP-NEXT: vpermt2q %zmm13, %zmm0, %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -22161,7 +22161,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm1
; AVX512BW-FCP-NEXT: vpermt2q %zmm17, %zmm0, %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 1808(%rdi), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 1808(%rdi), %xmm2
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -22192,8 +22192,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermt2q %zmm3, %zmm13, %zmm2
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 640(%rdi), %xmm2
-; AVX512BW-FCP-NEXT: vmovdqa 576(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 640(%rdi), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vmovdqa %ymm1, %ymm3
; AVX512BW-FCP-NEXT: vpermt2q %ymm2, %ymm9, %ymm3
; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %zmm19
@@ -22210,9 +22210,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermt2q %zmm4, %zmm13, %zmm3
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm3, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm3
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm3
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm3, %ymm9, %ymm4
; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm31
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm2
@@ -22229,9 +22229,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermt2q %zmm5, %zmm13, %zmm4
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm4, %zmm3
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 1536(%rdi), %xmm4
-; AVX512BW-FCP-NEXT: vmovdqa 1472(%rdi), %ymm5
-; AVX512BW-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm4
+; AVX512BW-FCP-NEXT: vmovdqa64 1472(%rdi), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm4, %ymm9, %ymm5
; AVX512BW-FCP-NEXT: vmovdqa64 1408(%rdi), %zmm10
; AVX512BW-FCP-NEXT: vmovdqa64 1344(%rdi), %zmm8
@@ -22247,9 +22247,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermt2q %zmm3, %zmm13, %zmm6
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm6, %zmm3
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512BW-FCP-NEXT: vmovdqa 1024(%rdi), %ymm6
-; AVX512BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512BW-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm4, %ymm9, %ymm6
; AVX512BW-FCP-NEXT: vmovdqa64 960(%rdi), %zmm17
; AVX512BW-FCP-NEXT: vmovdqa64 896(%rdi), %zmm15
@@ -22266,8 +22266,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm6, %zmm3
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa 2432(%rdi), %xmm4
-; AVX512BW-FCP-NEXT: vmovdqa 2368(%rdi), %ymm6
-; AVX512BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 2368(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm4, %ymm9, %ymm6
; AVX512BW-FCP-NEXT: vmovdqa64 2304(%rdi), %zmm16
; AVX512BW-FCP-NEXT: vmovdqa64 2240(%rdi), %zmm0
@@ -22284,9 +22284,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermt2q %zmm3, %zmm13, %zmm6
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm6, %zmm3
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 1984(%rdi), %xmm4
-; AVX512BW-FCP-NEXT: vmovdqa 1920(%rdi), %ymm6
-; AVX512BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1984(%rdi), %xmm4
+; AVX512BW-FCP-NEXT: vmovdqa64 1920(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm4, %ymm9, %ymm6
; AVX512BW-FCP-NEXT: vmovdqa64 1856(%rdi), %zmm23
; AVX512BW-FCP-NEXT: vmovdqa64 1792(%rdi), %zmm3
@@ -22304,8 +22304,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm18, %zmm4
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 3328(%rdi), %xmm18
-; AVX512BW-FCP-NEXT: vmovdqa 3264(%rdi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 3264(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermi2q %ymm18, %ymm4, %ymm9
; AVX512BW-FCP-NEXT: vmovdqa64 3200(%rdi), %zmm18
; AVX512BW-FCP-NEXT: vmovdqa64 3136(%rdi), %zmm4
@@ -22350,7 +22350,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermt2q %zmm19, %zmm13, %zmm9
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm19, %zmm20
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 2880(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 2880(%rdi), %ymm11
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm26[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],ymm26[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm0[4,5,6,7]
; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm0 = [9,0,7,0,9,0,7,0]
@@ -22380,7 +22380,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm1
; AVX512BW-FCP-NEXT: vpermt2q %zmm31, %zmm13, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm2 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -22393,7 +22393,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm10, %zmm13, %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 1536(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm2
; AVX512BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -22407,7 +22407,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm17, %zmm13, %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 1088(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm2
; AVX512BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -22421,7 +22421,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm16, %zmm13, %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 2432(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 2432(%rdi), %ymm2
; AVX512BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -22434,7 +22434,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm23, %zmm13, %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 1984(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 1984(%rdi), %ymm2
; AVX512BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -22446,7 +22446,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermi2q %zmm18, %zmm4, %zmm13
-; AVX512BW-FCP-NEXT: vmovdqa 3328(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 3328(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3],ymm1[4,5,6,7]
@@ -22915,13 +22915,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm27 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 2752(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 2752(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm0, %zmm24, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 512(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm0, %zmm29, %zmm0
@@ -22934,35 +22934,35 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm3, %zmm4, %zmm3
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm3 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 1408(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 1408(%rdi), %ymm4
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm4, %zmm5, %zmm4
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm4 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 960(%rdi), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 960(%rdi), %ymm5
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm5, %xmm5
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm5, %zmm6, %zmm5
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, %zmm5 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 2304(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 2304(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm6, %zmm7, %zmm6
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 1856(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 1856(%rdi), %ymm7
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm7, %xmm7
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm7, %zmm8, %zmm7
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm7 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 3200(%rdi), %ymm8
+; AVX512BW-FCP-NEXT: vmovdqa64 3200(%rdi), %ymm8
; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm8, %xmm8
; AVX512BW-FCP-NEXT: vinserti32x4 $0, %xmm8, %zmm16, %zmm8
@@ -23107,7 +23107,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm14, %zmm1
; AVX512DQ-BW-NEXT: vpermt2q %zmm16, %zmm0, %zmm1
-; AVX512DQ-BW-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -23119,13 +23119,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm12, %zmm1
; AVX512DQ-BW-NEXT: vpermt2q %zmm10, %zmm0, %zmm1
-; AVX512DQ-BW-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm13, %zmm1
; AVX512DQ-BW-NEXT: vpermt2q %zmm11, %zmm0, %zmm1
-; AVX512DQ-BW-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -23137,7 +23137,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, %zmm1
; AVX512DQ-BW-NEXT: vpermt2q %zmm9, %zmm0, %zmm1
-; AVX512DQ-BW-NEXT: vmovdqa 1808(%rdi), %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 1808(%rdi), %xmm2
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -23147,8 +23147,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vinserti128 $1, 2880(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 2816(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 2816(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm9 = [4,11]
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm1
@@ -23168,9 +23168,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm2, %zmm4, %zmm1
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vinserti128 $1, 640(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 576(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %zmm27
; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdi), %zmm2
@@ -23187,9 +23187,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm2, %zmm4, %zmm1
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm31
; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm8
@@ -23206,9 +23206,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm2, %zmm4, %zmm1
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1536(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 1472(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1536(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 1472(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqa64 1408(%rdi), %zmm26
; AVX512DQ-BW-NEXT: vmovdqa64 1344(%rdi), %zmm7
@@ -23224,9 +23224,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm2, %zmm4, %zmm1
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 1024(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1088(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 1024(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqa64 960(%rdi), %zmm19
; AVX512DQ-BW-NEXT: vmovdqa64 896(%rdi), %zmm6
@@ -23244,8 +23244,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vinserti128 $1, 2432(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 2368(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 2368(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqa64 2304(%rdi), %zmm20
; AVX512DQ-BW-NEXT: vmovdqa64 2240(%rdi), %zmm5
@@ -23262,9 +23262,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm3, %zmm4, %zmm1
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1984(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 1920(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1984(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 1920(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqa64 1856(%rdi), %zmm21
; AVX512DQ-BW-NEXT: vmovdqa64 1792(%rdi), %zmm2
@@ -23282,8 +23282,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vinserti128 $1, 3328(%rdi), %ymm0, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 3264(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 3264(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
; AVX512DQ-BW-NEXT: vmovdqa64 3200(%rdi), %zmm17
; AVX512DQ-BW-NEXT: vmovdqa64 3136(%rdi), %zmm11
@@ -23322,7 +23322,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm30, %zmm4, %zmm10
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm10, %zmm0
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 2880(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 2880(%rdi), %ymm0
; AVX512DQ-BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm4 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm0 = [5,12]
@@ -23339,7 +23339,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm5
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 640(%rdi), %ymm9
+; AVX512DQ-BW-NEXT: vmovdqa64 640(%rdi), %ymm9
; AVX512DQ-BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vpermt2q %zmm27, %zmm0, %zmm1
@@ -23351,7 +23351,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm5
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512DQ-BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -23364,7 +23364,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm5
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 1536(%rdi), %ymm9
+; AVX512DQ-BW-NEXT: vmovdqa64 1536(%rdi), %ymm9
; AVX512DQ-BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm5
@@ -23380,7 +23380,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm7
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 1088(%rdi), %ymm9
+; AVX512DQ-BW-NEXT: vmovdqa64 1088(%rdi), %ymm9
; AVX512DQ-BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -23394,7 +23394,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm9, %zmm12, %zmm9
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 2432(%rdi), %ymm9
+; AVX512DQ-BW-NEXT: vmovdqa64 2432(%rdi), %ymm9
; AVX512DQ-BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm2 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
@@ -23407,7 +23407,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm9 # 64-byte Folded Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm9, %zmm2
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 1984(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 1984(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
@@ -23420,7 +23420,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm23, %zmm10, %zmm3
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm3, %zmm2
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 3328(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 3328(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm1 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
@@ -23894,13 +23894,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm26 {%k2}
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm31, %zmm27 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 2752(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 2752(%rdi), %ymm0
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm0, %zmm28, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 512(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %ymm0
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -23914,35 +23914,35 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm3, %zmm4, %zmm3
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, %zmm3 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 1408(%rdi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 1408(%rdi), %ymm4
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm4, %zmm5, %zmm4
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm4 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 960(%rdi), %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 960(%rdi), %ymm5
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm5, %xmm5
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm5, %zmm6, %zmm5
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, %zmm5 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 2304(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 2304(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm6, %zmm7, %zmm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 1856(%rdi), %ymm7
+; AVX512DQ-BW-NEXT: vmovdqa64 1856(%rdi), %ymm7
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm7, %xmm7
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm7, %zmm8, %zmm7
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, %zmm7 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 3200(%rdi), %ymm8
+; AVX512DQ-BW-NEXT: vmovdqa64 3200(%rdi), %ymm8
; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm8, %xmm8
; AVX512DQ-BW-NEXT: vinserti32x4 $0, %xmm8, %zmm30, %zmm8
@@ -24085,7 +24085,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm14, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm10, %zmm0, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 464(%rdi), %xmm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 464(%rdi), %xmm2
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -24096,13 +24096,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm12, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm11, %zmm0, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1360(%rdi), %xmm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1360(%rdi), %xmm2
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm13, %zmm0, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 912(%rdi), %xmm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 912(%rdi), %xmm2
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -24114,7 +24114,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm17, %zmm0, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1808(%rdi), %xmm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1808(%rdi), %xmm2
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm2, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -24145,8 +24145,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm3, %zmm13, %zmm2
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm2, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 640(%rdi), %xmm2
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 576(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 640(%rdi), %xmm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, %ymm3
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm2, %ymm9, %ymm3
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %zmm19
@@ -24163,9 +24163,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm4, %zmm13, %zmm3
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm3, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm3, %ymm9, %ymm4
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm31
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm2
@@ -24182,9 +24182,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm5, %zmm13, %zmm4
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm4, %zmm3
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1536(%rdi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1472(%rdi), %ymm5
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1472(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm4, %ymm9, %ymm5
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1408(%rdi), %zmm10
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1344(%rdi), %zmm8
@@ -24200,9 +24200,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm3, %zmm13, %zmm6
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm6, %zmm3
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1024(%rdi), %ymm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm4, %ymm9, %ymm6
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 960(%rdi), %zmm17
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 896(%rdi), %zmm15
@@ -24219,8 +24219,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm6, %zmm3
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa 2432(%rdi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2368(%rdi), %ymm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2368(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm4, %ymm9, %ymm6
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2304(%rdi), %zmm16
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2240(%rdi), %zmm0
@@ -24237,9 +24237,9 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm3, %zmm13, %zmm6
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm6, %zmm3
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1984(%rdi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1920(%rdi), %ymm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1984(%rdi), %xmm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1920(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm4, %ymm9, %ymm6
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1856(%rdi), %zmm23
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1792(%rdi), %zmm3
@@ -24257,8 +24257,8 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm18, %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3328(%rdi), %xmm18
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3264(%rdi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3264(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermi2q %ymm18, %ymm4, %ymm9
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3200(%rdi), %zmm18
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3136(%rdi), %zmm4
@@ -24303,7 +24303,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm19, %zmm13, %zmm9
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm19, %zmm20
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2880(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2880(%rdi), %ymm11
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm26[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],ymm26[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm0[4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm0 = [9,0,7,0,9,0,7,0]
@@ -24333,7 +24333,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm31, %zmm13, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm9
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm9
; AVX512DQ-BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm2 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -24346,7 +24346,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm10, %zmm13, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1536(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -24360,7 +24360,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm17, %zmm13, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1088(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -24374,7 +24374,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm16, %zmm13, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2432(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2432(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -24387,7 +24387,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm23, %zmm13, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1984(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1984(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
@@ -24399,7 +24399,7 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm18, %zmm4, %zmm13
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3328(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3328(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3],ymm1[4,5,6,7]
@@ -24868,13 +24868,13 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm27 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2752(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2752(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm0, %zmm24, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 512(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm0, %zmm29, %zmm0
@@ -24887,35 +24887,35 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm3, %zmm4, %zmm3
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm3 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1408(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1408(%rdi), %ymm4
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm4, %xmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm4, %zmm5, %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm4 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 960(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 960(%rdi), %ymm5
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm5, %xmm5
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm5, %zmm6, %zmm5
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, %zmm5 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2304(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2304(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm6, %zmm7, %zmm6
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1856(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1856(%rdi), %ymm7
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm7, %xmm7
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm7, %zmm8, %zmm7
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm7 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3200(%rdi), %ymm8
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3200(%rdi), %ymm8
; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm8, %xmm8
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $0, %xmm8, %zmm16, %zmm8
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-8.ll
index 7c428e8c55011..71d045b273edd 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-8.ll
@@ -724,8 +724,8 @@ define void @load_i64_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm6
; AVX512-NEXT: vmovaps (%rdi), %xmm0
; AVX512-NEXT: vmovaps 64(%rdi), %xmm1
-; AVX512-NEXT: vinsertf128 $1, 192(%rdi), %ymm1, %ymm1
-; AVX512-NEXT: vinsertf128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vinsertf32x4 $1, 192(%rdi), %ymm1, %ymm1
+; AVX512-NEXT: vinsertf32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
; AVX512-NEXT: vmovaps 192(%rdi), %ymm1
@@ -780,8 +780,8 @@ define void @load_i64_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %zmm6
; AVX512-FCP-NEXT: vmovaps (%rdi), %xmm0
; AVX512-FCP-NEXT: vmovaps 64(%rdi), %xmm1
-; AVX512-FCP-NEXT: vinsertf128 $1, 192(%rdi), %ymm1, %ymm1
-; AVX512-FCP-NEXT: vinsertf128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512-FCP-NEXT: vinsertf32x4 $1, 192(%rdi), %ymm1, %ymm1
+; AVX512-FCP-NEXT: vinsertf32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512-FCP-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512-FCP-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
; AVX512-FCP-NEXT: vmovaps 192(%rdi), %ymm1
@@ -836,8 +836,8 @@ define void @load_i64_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm6
; AVX512DQ-NEXT: vmovaps (%rdi), %xmm0
; AVX512DQ-NEXT: vmovaps 64(%rdi), %xmm1
-; AVX512DQ-NEXT: vinsertf128 $1, 192(%rdi), %ymm1, %ymm1
-; AVX512DQ-NEXT: vinsertf128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vinsertf32x4 $1, 192(%rdi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vinsertf32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512DQ-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
; AVX512DQ-NEXT: vmovaps 192(%rdi), %ymm1
@@ -892,8 +892,8 @@ define void @load_i64_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %zmm6
; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %xmm0
; AVX512DQ-FCP-NEXT: vmovaps 64(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vinsertf128 $1, 192(%rdi), %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vinsertf128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT: vinsertf32x4 $1, 192(%rdi), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vinsertf32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512DQ-FCP-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
; AVX512DQ-FCP-NEXT: vmovaps 192(%rdi), %ymm1
@@ -948,8 +948,8 @@ define void @load_i64_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm6
; AVX512BW-NEXT: vmovaps (%rdi), %xmm0
; AVX512BW-NEXT: vmovaps 64(%rdi), %xmm1
-; AVX512BW-NEXT: vinsertf128 $1, 192(%rdi), %ymm1, %ymm1
-; AVX512BW-NEXT: vinsertf128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vinsertf32x4 $1, 192(%rdi), %ymm1, %ymm1
+; AVX512BW-NEXT: vinsertf32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512BW-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512BW-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
; AVX512BW-NEXT: vmovaps 192(%rdi), %ymm1
@@ -1004,8 +1004,8 @@ define void @load_i64_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm6
; AVX512BW-FCP-NEXT: vmovaps (%rdi), %xmm0
; AVX512BW-FCP-NEXT: vmovaps 64(%rdi), %xmm1
-; AVX512BW-FCP-NEXT: vinsertf128 $1, 192(%rdi), %ymm1, %ymm1
-; AVX512BW-FCP-NEXT: vinsertf128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512BW-FCP-NEXT: vinsertf32x4 $1, 192(%rdi), %ymm1, %ymm1
+; AVX512BW-FCP-NEXT: vinsertf32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512BW-FCP-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512BW-FCP-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
; AVX512BW-FCP-NEXT: vmovaps 192(%rdi), %ymm1
@@ -1060,8 +1060,8 @@ define void @load_i64_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm6
; AVX512DQ-BW-NEXT: vmovaps (%rdi), %xmm0
; AVX512DQ-BW-NEXT: vmovaps 64(%rdi), %xmm1
-; AVX512DQ-BW-NEXT: vinsertf128 $1, 192(%rdi), %ymm1, %ymm1
-; AVX512DQ-BW-NEXT: vinsertf128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vinsertf32x4 $1, 192(%rdi), %ymm1, %ymm1
+; AVX512DQ-BW-NEXT: vinsertf32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512DQ-BW-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
; AVX512DQ-BW-NEXT: vmovaps 192(%rdi), %ymm1
@@ -1116,8 +1116,8 @@ define void @load_i64_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm6
; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %xmm0
; AVX512DQ-BW-FCP-NEXT: vmovaps 64(%rdi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vinsertf128 $1, 192(%rdi), %ymm1, %ymm1
-; AVX512DQ-BW-FCP-NEXT: vinsertf128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT: vinsertf32x4 $1, 192(%rdi), %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT: vinsertf32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-FCP-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512DQ-BW-FCP-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
; AVX512DQ-BW-FCP-NEXT: vmovaps 192(%rdi), %ymm1
@@ -1814,8 +1814,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512-NEXT: vmovdqa (%rdi), %xmm9
; AVX512-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512-NEXT: vinserti128 $1, 192(%rdi), %ymm10, %ymm10
-; AVX512-NEXT: vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512-NEXT: vinserti32x4 $1, 128(%rdi), %ymm9, %ymm9
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm16
; AVX512-NEXT: vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
@@ -1832,8 +1832,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpermt2q %zmm4, %zmm10, %zmm11
; AVX512-NEXT: vpermi2q %zmm3, %zmm0, %zmm10
; AVX512-NEXT: vmovdqa64 %zmm11, %zmm10 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm11
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm14
; AVX512-NEXT: vmovdqa (%rdi), %ymm15
@@ -1926,8 +1926,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm9
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm10, %ymm10
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm9, %ymm9
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm16
; AVX512-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
@@ -1944,8 +1944,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpermt2q %zmm4, %zmm10, %zmm11
; AVX512-FCP-NEXT: vpermi2q %zmm3, %zmm0, %zmm10
; AVX512-FCP-NEXT: vmovdqa64 %zmm11, %zmm10 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm11
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm14
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm15
@@ -2038,8 +2038,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm9
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdi), %ymm10, %ymm10
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rdi), %ymm9, %ymm9
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm16
; AVX512DQ-NEXT: vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
@@ -2056,8 +2056,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpermt2q %zmm4, %zmm10, %zmm11
; AVX512DQ-NEXT: vpermi2q %zmm3, %zmm0, %zmm10
; AVX512DQ-NEXT: vmovdqa64 %zmm11, %zmm10 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm11
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm11
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm14
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm15
@@ -2150,8 +2150,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm9
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm10, %ymm10
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm9, %ymm9
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm16
; AVX512DQ-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
@@ -2168,8 +2168,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpermt2q %zmm4, %zmm10, %zmm11
; AVX512DQ-FCP-NEXT: vpermi2q %zmm3, %zmm0, %zmm10
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm11, %zmm10 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm14
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm15
@@ -2262,8 +2262,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512BW-NEXT: vmovdqa (%rdi), %xmm9
; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdi), %ymm10, %ymm10
-; AVX512BW-NEXT: vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm9, %ymm9
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm16
; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
@@ -2280,8 +2280,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpermt2q %zmm4, %zmm10, %zmm11
; AVX512BW-NEXT: vpermi2q %zmm3, %zmm0, %zmm10
; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm10 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm11
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm11
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512BW-NEXT: vmovdqa 64(%rdi), %ymm14
; AVX512BW-NEXT: vmovdqa (%rdi), %ymm15
@@ -2374,8 +2374,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm9
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512BW-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm10, %ymm10
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm9, %ymm9
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm16
; AVX512BW-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
@@ -2392,8 +2392,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpermt2q %zmm4, %zmm10, %zmm11
; AVX512BW-FCP-NEXT: vpermi2q %zmm3, %zmm0, %zmm10
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm10 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm11
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %ymm14
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm15
@@ -2486,8 +2486,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm9
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdi), %ymm10, %ymm10
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm9, %ymm9
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm16
; AVX512DQ-BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
@@ -2504,8 +2504,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpermt2q %zmm4, %zmm10, %zmm11
; AVX512DQ-BW-NEXT: vpermi2q %zmm3, %zmm0, %zmm10
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm11, %zmm10 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm11
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm11
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %ymm14
; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm15
@@ -2598,8 +2598,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm8 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm9
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm10
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm10, %ymm10
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm9, %ymm9
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm11, %zmm8, %zmm16
; AVX512DQ-BW-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
@@ -2616,8 +2616,8 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm4, %zmm10, %zmm11
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm3, %zmm0, %zmm10
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm10 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm11
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm12
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm12
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %ymm14
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm15
@@ -4092,8 +4092,8 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm30, %zmm19, %zmm11
; AVX512-NEXT: vpermi2q %zmm10, %zmm14, %zmm19
; AVX512-NEXT: vmovdqa64 %zmm11, %zmm19 {%k1}
-; AVX512-NEXT: vmovdqa 704(%rdi), %ymm11
-; AVX512-NEXT: vmovdqa 640(%rdi), %ymm12
+; AVX512-NEXT: vmovdqa64 704(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 640(%rdi), %ymm12
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm26
; AVX512-NEXT: vmovdqa64 512(%rdi), %ymm27
@@ -4211,17 +4211,17 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermi2q %zmm10, %zmm14, %zmm26
; AVX512-NEXT: vpermt2q %zmm10, %zmm2, %zmm14
; AVX512-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512-NEXT: vinserti128 $1, 192(%rdi), %ymm2, %ymm2
-; AVX512-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm10, %zmm18, %zmm10
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512-NEXT: vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512-NEXT: vinserti32x4 $1, 704(%rdi), %ymm1, %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
; AVX512-NEXT: vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
-; AVX512-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm7, %zmm29, %zmm7
; AVX512-NEXT: vmovdqa64 %zmm12, %zmm23 {%k1}
@@ -4315,8 +4315,8 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm30, %zmm19, %zmm11
; AVX512-FCP-NEXT: vpermi2q %zmm10, %zmm14, %zmm19
; AVX512-FCP-NEXT: vmovdqa64 %zmm11, %zmm19 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 704(%rdi), %ymm11
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 704(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %ymm12
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm26
; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %ymm27
@@ -4434,17 +4434,17 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermi2q %zmm10, %zmm14, %zmm26
; AVX512-FCP-NEXT: vpermt2q %zmm10, %zmm2, %zmm14
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm2, %ymm2
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm10, %zmm18, %zmm10
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm1, %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
-; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512-FCP-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm29, %zmm7
; AVX512-FCP-NEXT: vmovdqa64 %zmm12, %zmm23 {%k1}
@@ -4538,8 +4538,8 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm30, %zmm19, %zmm11
; AVX512DQ-NEXT: vpermi2q %zmm10, %zmm14, %zmm19
; AVX512DQ-NEXT: vmovdqa64 %zmm11, %zmm19 {%k1}
-; AVX512DQ-NEXT: vmovdqa 704(%rdi), %ymm11
-; AVX512DQ-NEXT: vmovdqa 640(%rdi), %ymm12
+; AVX512DQ-NEXT: vmovdqa64 704(%rdi), %ymm11
+; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %ymm12
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm26
; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %ymm27
@@ -4657,17 +4657,17 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermi2q %zmm10, %zmm14, %zmm26
; AVX512DQ-NEXT: vpermt2q %zmm10, %zmm2, %zmm14
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdi), %ymm2, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm10, %zmm18, %zmm10
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512DQ-NEXT: vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512DQ-NEXT: vinserti32x4 $1, 704(%rdi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512DQ-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512DQ-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm7, %zmm29, %zmm7
; AVX512DQ-NEXT: vmovdqa64 %zmm12, %zmm23 {%k1}
@@ -4761,8 +4761,8 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm30, %zmm19, %zmm11
; AVX512DQ-FCP-NEXT: vpermi2q %zmm10, %zmm14, %zmm19
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm11, %zmm19 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 704(%rdi), %ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 704(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %ymm12
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm26
; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %ymm27
@@ -4880,17 +4880,17 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermi2q %zmm10, %zmm14, %zmm26
; AVX512DQ-FCP-NEXT: vpermt2q %zmm10, %zmm2, %zmm14
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm10, %zmm18, %zmm10
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
-; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm29, %zmm7
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm12, %zmm23 {%k1}
@@ -4984,8 +4984,8 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm30, %zmm19, %zmm11
; AVX512BW-NEXT: vpermi2q %zmm10, %zmm14, %zmm19
; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm19 {%k1}
-; AVX512BW-NEXT: vmovdqa 704(%rdi), %ymm11
-; AVX512BW-NEXT: vmovdqa 640(%rdi), %ymm12
+; AVX512BW-NEXT: vmovdqa64 704(%rdi), %ymm11
+; AVX512BW-NEXT: vmovdqa64 640(%rdi), %ymm12
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512BW-NEXT: vmovdqa64 576(%rdi), %ymm26
; AVX512BW-NEXT: vmovdqa64 512(%rdi), %ymm27
@@ -5103,17 +5103,17 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermi2q %zmm10, %zmm14, %zmm26
; AVX512BW-NEXT: vpermt2q %zmm10, %zmm2, %zmm14
; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdi), %ymm2, %ymm2
-; AVX512BW-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm10, %zmm18, %zmm10
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512BW-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512BW-NEXT: vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512BW-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512BW-NEXT: vinserti32x4 $1, 704(%rdi), %ymm1, %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
-; AVX512BW-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512BW-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512BW-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm7, %zmm29, %zmm7
; AVX512BW-NEXT: vmovdqa64 %zmm12, %zmm23 {%k1}
@@ -5207,8 +5207,8 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermt2q %zmm30, %zmm19, %zmm11
; AVX512BW-FCP-NEXT: vpermi2q %zmm10, %zmm14, %zmm19
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm19 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 704(%rdi), %ymm11
-; AVX512BW-FCP-NEXT: vmovdqa 640(%rdi), %ymm12
+; AVX512BW-FCP-NEXT: vmovdqa64 704(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm12
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm26
; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %ymm27
@@ -5326,17 +5326,17 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermi2q %zmm10, %zmm14, %zmm26
; AVX512BW-FCP-NEXT: vpermt2q %zmm10, %zmm2, %zmm14
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512BW-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm2, %ymm2
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm10, %zmm18, %zmm10
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512BW-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm1, %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
-; AVX512BW-FCP-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512BW-FCP-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm29, %zmm7
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm12, %zmm23 {%k1}
@@ -5430,8 +5430,8 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm30, %zmm19, %zmm11
; AVX512DQ-BW-NEXT: vpermi2q %zmm10, %zmm14, %zmm19
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm11, %zmm19 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 704(%rdi), %ymm11
-; AVX512DQ-BW-NEXT: vmovdqa 640(%rdi), %ymm12
+; AVX512DQ-BW-NEXT: vmovdqa64 704(%rdi), %ymm11
+; AVX512DQ-BW-NEXT: vmovdqa64 640(%rdi), %ymm12
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %ymm26
; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %ymm27
@@ -5549,17 +5549,17 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermi2q %zmm10, %zmm14, %zmm26
; AVX512DQ-BW-NEXT: vpermt2q %zmm10, %zmm2, %zmm14
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdi), %ymm2, %ymm2
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm10, %zmm18, %zmm10
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 704(%rdi), %ymm1, %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
-; AVX512DQ-BW-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512DQ-BW-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm7, %zmm29, %zmm7
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm12, %zmm23 {%k1}
@@ -5653,8 +5653,8 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm30, %zmm19, %zmm11
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm10, %zmm14, %zmm19
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm19 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 704(%rdi), %ymm11
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 640(%rdi), %ymm12
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 704(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm12
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm26
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %ymm27
@@ -5772,17 +5772,17 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm10, %zmm14, %zmm26
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm10, %zmm2, %zmm14
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm2, %ymm2
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm10, %zmm18, %zmm10
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 576(%rdi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm1, %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm29, %zmm7
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm12, %zmm23 {%k1}
@@ -8765,10 +8765,10 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vpermt2q %zmm19, %zmm0, %zmm2
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512-NEXT: vmovdqa64 1216(%rdi), %ymm22
-; AVX512-NEXT: vmovdqa 1152(%rdi), %ymm10
+; AVX512-NEXT: vmovdqa64 1152(%rdi), %ymm10
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
-; AVX512-NEXT: vmovdqa 1088(%rdi), %ymm9
-; AVX512-NEXT: vmovdqa 1024(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 1088(%rdi), %ymm9
+; AVX512-NEXT: vmovdqa64 1024(%rdi), %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm4, %zmm2, %zmm1
@@ -8778,8 +8778,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm11, %zmm4
; AVX512-NEXT: vpermt2q %zmm7, %zmm0, %zmm4
; AVX512-NEXT: vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512-NEXT: vmovdqa 640(%rdi), %ymm13
+; AVX512-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512-NEXT: vmovdqa64 640(%rdi), %ymm13
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm17
; AVX512-NEXT: vmovdqa64 512(%rdi), %ymm21
@@ -8792,8 +8792,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm8, %zmm11
; AVX512-NEXT: vpermt2q %zmm20, %zmm0, %zmm11
; AVX512-NEXT: vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm4
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm14
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm14
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
; AVX512-NEXT: vmovdqa64 64(%rdi), %ymm25
; AVX512-NEXT: vmovdqa64 (%rdi), %ymm28
@@ -8807,11 +8807,11 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 1792(%rdi), %zmm26
; AVX512-NEXT: vpermi2q %zmm12, %zmm26, %zmm0
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512-NEXT: vmovdqa 1728(%rdi), %ymm2
-; AVX512-NEXT: vmovdqa 1664(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 1728(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 1664(%rdi), %ymm8
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
; AVX512-NEXT: vmovdqa64 1600(%rdi), %ymm30
-; AVX512-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm11, %zmm0, %zmm0
@@ -9124,25 +9124,25 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm27, %zmm10
; AVX512-NEXT: vmovdqa64 %zmm31, %zmm25 {%k1}
; AVX512-NEXT: vmovdqa 64(%rdi), %xmm0
-; AVX512-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
; AVX512-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-NEXT: vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512-NEXT: vinserti32x4 $1, 128(%rdi), %ymm1, %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm25, %zmm2
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512-NEXT: vmovdqa 576(%rdi), %xmm5
-; AVX512-NEXT: vinserti128 $1, 704(%rdi), %ymm5, %ymm5
-; AVX512-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512-NEXT: vmovdqa64 576(%rdi), %xmm5
+; AVX512-NEXT: vinserti32x4 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm18, %zmm7, %zmm27
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512-NEXT: vmovdqa 1088(%rdi), %xmm13
-; AVX512-NEXT: vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512-NEXT: vmovdqa64 1088(%rdi), %xmm13
+; AVX512-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm13, %ymm13
; AVX512-NEXT: vmovdqa64 1024(%rdi), %xmm18
; AVX512-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
@@ -9283,10 +9283,10 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vpermt2q %zmm19, %zmm0, %zmm2
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512-FCP-NEXT: vmovdqa64 1216(%rdi), %ymm22
-; AVX512-FCP-NEXT: vmovdqa 1152(%rdi), %ymm10
+; AVX512-FCP-NEXT: vmovdqa64 1152(%rdi), %ymm10
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
-; AVX512-FCP-NEXT: vmovdqa 1088(%rdi), %ymm9
-; AVX512-FCP-NEXT: vmovdqa 1024(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm9
+; AVX512-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm2, %zmm1
@@ -9296,8 +9296,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm11, %zmm4
; AVX512-FCP-NEXT: vpermt2q %zmm7, %zmm0, %zmm4
; AVX512-FCP-NEXT: vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512-FCP-NEXT: vmovdqa 640(%rdi), %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %ymm13
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm17
; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %ymm21
@@ -9310,8 +9310,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm8, %zmm11
; AVX512-FCP-NEXT: vpermt2q %zmm20, %zmm0, %zmm11
; AVX512-FCP-NEXT: vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm14
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm14
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %ymm25
; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %ymm28
@@ -9325,11 +9325,11 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 1792(%rdi), %zmm26
; AVX512-FCP-NEXT: vpermi2q %zmm12, %zmm26, %zmm0
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 1728(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqa 1664(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 1728(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 1664(%rdi), %ymm8
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
; AVX512-FCP-NEXT: vmovdqa64 1600(%rdi), %ymm30
-; AVX512-FCP-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm11, %zmm0, %zmm0
@@ -9642,25 +9642,25 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm27, %zmm10
; AVX512-FCP-NEXT: vmovdqa64 %zmm31, %zmm25 {%k1}
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm0
-; AVX512-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm1, %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm25, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %xmm5
-; AVX512-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm5, %ymm5
-; AVX512-FCP-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512-FCP-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %xmm5
+; AVX512-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm18, %zmm7, %zmm27
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 1088(%rdi), %xmm13
-; AVX512-FCP-NEXT: vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm13
+; AVX512-FCP-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm13, %ymm13
; AVX512-FCP-NEXT: vmovdqa64 1024(%rdi), %xmm18
; AVX512-FCP-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
@@ -9801,10 +9801,10 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vpermt2q %zmm19, %zmm0, %zmm2
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512DQ-NEXT: vmovdqa64 1216(%rdi), %ymm22
-; AVX512DQ-NEXT: vmovdqa 1152(%rdi), %ymm10
+; AVX512DQ-NEXT: vmovdqa64 1152(%rdi), %ymm10
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
-; AVX512DQ-NEXT: vmovdqa 1088(%rdi), %ymm9
-; AVX512DQ-NEXT: vmovdqa 1024(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 1088(%rdi), %ymm9
+; AVX512DQ-NEXT: vmovdqa64 1024(%rdi), %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm4, %zmm2, %zmm1
@@ -9814,8 +9814,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm11, %zmm4
; AVX512DQ-NEXT: vpermt2q %zmm7, %zmm0, %zmm4
; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512DQ-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512DQ-NEXT: vmovdqa 640(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %ymm13
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm17
; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %ymm21
@@ -9828,8 +9828,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm8, %zmm11
; AVX512DQ-NEXT: vpermt2q %zmm20, %zmm0, %zmm11
; AVX512DQ-NEXT: vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm4
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm14
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %ymm25
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %ymm28
@@ -9843,11 +9843,11 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 1792(%rdi), %zmm26
; AVX512DQ-NEXT: vpermi2q %zmm12, %zmm26, %zmm0
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512DQ-NEXT: vmovdqa 1728(%rdi), %ymm2
-; AVX512DQ-NEXT: vmovdqa 1664(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 1728(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 1664(%rdi), %ymm8
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
; AVX512DQ-NEXT: vmovdqa64 1600(%rdi), %ymm30
-; AVX512DQ-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm11, %zmm0, %zmm0
@@ -10160,25 +10160,25 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm27, %zmm10
; AVX512DQ-NEXT: vmovdqa64 %zmm31, %zmm25 {%k1}
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm0
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rdi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm25, %zmm2
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %xmm5
-; AVX512DQ-NEXT: vinserti128 $1, 704(%rdi), %ymm5, %ymm5
-; AVX512DQ-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512DQ-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %xmm5
+; AVX512DQ-NEXT: vinserti32x4 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512DQ-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm18, %zmm7, %zmm27
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512DQ-NEXT: vmovdqa 1088(%rdi), %xmm13
-; AVX512DQ-NEXT: vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512DQ-NEXT: vmovdqa64 1088(%rdi), %xmm13
+; AVX512DQ-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm13, %ymm13
; AVX512DQ-NEXT: vmovdqa64 1024(%rdi), %xmm18
; AVX512DQ-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
@@ -10319,10 +10319,10 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vpermt2q %zmm19, %zmm0, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa64 1216(%rdi), %ymm22
-; AVX512DQ-FCP-NEXT: vmovdqa 1152(%rdi), %ymm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 1152(%rdi), %ymm10
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 1088(%rdi), %ymm9
-; AVX512DQ-FCP-NEXT: vmovdqa 1024(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm2, %zmm1
@@ -10332,8 +10332,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm11, %zmm4
; AVX512DQ-FCP-NEXT: vpermt2q %zmm7, %zmm0, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512DQ-FCP-NEXT: vmovdqa 640(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm17
; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %ymm21
@@ -10346,8 +10346,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, %zmm11
; AVX512DQ-FCP-NEXT: vpermt2q %zmm20, %zmm0, %zmm11
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm14
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %ymm25
; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %ymm28
@@ -10361,11 +10361,11 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 1792(%rdi), %zmm26
; AVX512DQ-FCP-NEXT: vpermi2q %zmm12, %zmm26, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 1728(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa 1664(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 1728(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 1664(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
; AVX512DQ-FCP-NEXT: vmovdqa64 1600(%rdi), %ymm30
-; AVX512DQ-FCP-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm11, %zmm0, %zmm0
@@ -10678,25 +10678,25 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm27, %zmm10
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm31, %zmm25 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm25, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %xmm5
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm5, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm18, %zmm7, %zmm27
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 1088(%rdi), %xmm13
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm13
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm13, %ymm13
; AVX512DQ-FCP-NEXT: vmovdqa64 1024(%rdi), %xmm18
; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
@@ -10837,10 +10837,10 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vpermt2q %zmm19, %zmm0, %zmm2
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512BW-NEXT: vmovdqa64 1216(%rdi), %ymm22
-; AVX512BW-NEXT: vmovdqa 1152(%rdi), %ymm10
+; AVX512BW-NEXT: vmovdqa64 1152(%rdi), %ymm10
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
-; AVX512BW-NEXT: vmovdqa 1088(%rdi), %ymm9
-; AVX512BW-NEXT: vmovdqa 1024(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 1088(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 1024(%rdi), %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm4, %zmm2, %zmm1
@@ -10850,8 +10850,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 %zmm11, %zmm4
; AVX512BW-NEXT: vpermt2q %zmm7, %zmm0, %zmm4
; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512BW-NEXT: vmovdqa 640(%rdi), %ymm13
+; AVX512BW-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512BW-NEXT: vmovdqa64 640(%rdi), %ymm13
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
; AVX512BW-NEXT: vmovdqa64 576(%rdi), %ymm17
; AVX512BW-NEXT: vmovdqa64 512(%rdi), %ymm21
@@ -10864,8 +10864,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 %zmm8, %zmm11
; AVX512BW-NEXT: vpermt2q %zmm20, %zmm0, %zmm11
; AVX512BW-NEXT: vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm4
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm14
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm14
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %ymm25
; AVX512BW-NEXT: vmovdqa64 (%rdi), %ymm28
@@ -10879,11 +10879,11 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 1792(%rdi), %zmm26
; AVX512BW-NEXT: vpermi2q %zmm12, %zmm26, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512BW-NEXT: vmovdqa 1728(%rdi), %ymm2
-; AVX512BW-NEXT: vmovdqa 1664(%rdi), %ymm8
+; AVX512BW-NEXT: vmovdqa64 1728(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 1664(%rdi), %ymm8
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
; AVX512BW-NEXT: vmovdqa64 1600(%rdi), %ymm30
-; AVX512BW-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm11, %zmm0, %zmm0
@@ -11196,25 +11196,25 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm27, %zmm10
; AVX512BW-NEXT: vmovdqa64 %zmm31, %zmm25 {%k1}
; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm0
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
; AVX512BW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512BW-NEXT: vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm1, %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm25, %zmm2
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512BW-NEXT: vmovdqa 576(%rdi), %xmm5
-; AVX512BW-NEXT: vinserti128 $1, 704(%rdi), %ymm5, %ymm5
-; AVX512BW-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512BW-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512BW-NEXT: vmovdqa64 576(%rdi), %xmm5
+; AVX512BW-NEXT: vinserti32x4 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512BW-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm18, %zmm7, %zmm27
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512BW-NEXT: vmovdqa 1088(%rdi), %xmm13
-; AVX512BW-NEXT: vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512BW-NEXT: vmovdqa64 1088(%rdi), %xmm13
+; AVX512BW-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm13, %ymm13
; AVX512BW-NEXT: vmovdqa64 1024(%rdi), %xmm18
; AVX512BW-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
@@ -11355,10 +11355,10 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vpermt2q %zmm19, %zmm0, %zmm2
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa64 1216(%rdi), %ymm22
-; AVX512BW-FCP-NEXT: vmovdqa 1152(%rdi), %ymm10
+; AVX512BW-FCP-NEXT: vmovdqa64 1152(%rdi), %ymm10
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
-; AVX512BW-FCP-NEXT: vmovdqa 1088(%rdi), %ymm9
-; AVX512BW-FCP-NEXT: vmovdqa 1024(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm9
+; AVX512BW-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm2, %zmm1
@@ -11368,8 +11368,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm4
; AVX512BW-FCP-NEXT: vpermt2q %zmm7, %zmm0, %zmm4
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512BW-FCP-NEXT: vmovdqa 640(%rdi), %ymm13
+; AVX512BW-FCP-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm13
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm17
; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %ymm21
@@ -11382,8 +11382,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm11
; AVX512BW-FCP-NEXT: vpermt2q %zmm20, %zmm0, %zmm11
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm14
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm14
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %ymm25
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %ymm28
@@ -11397,11 +11397,11 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 1792(%rdi), %zmm26
; AVX512BW-FCP-NEXT: vpermi2q %zmm12, %zmm26, %zmm0
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 1728(%rdi), %ymm2
-; AVX512BW-FCP-NEXT: vmovdqa 1664(%rdi), %ymm8
+; AVX512BW-FCP-NEXT: vmovdqa64 1728(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 1664(%rdi), %ymm8
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
; AVX512BW-FCP-NEXT: vmovdqa64 1600(%rdi), %ymm30
-; AVX512BW-FCP-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm11, %zmm0, %zmm0
@@ -11714,25 +11714,25 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm27, %zmm10
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm31, %zmm25 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm0
-; AVX512BW-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm1, %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm25, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 576(%rdi), %xmm5
-; AVX512BW-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm5, %ymm5
-; AVX512BW-FCP-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512BW-FCP-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %xmm5
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm18, %zmm7, %zmm27
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 1088(%rdi), %xmm13
-; AVX512BW-FCP-NEXT: vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512BW-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm13
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm13, %ymm13
; AVX512BW-FCP-NEXT: vmovdqa64 1024(%rdi), %xmm18
; AVX512BW-FCP-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
@@ -11873,10 +11873,10 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vpermt2q %zmm19, %zmm0, %zmm2
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa64 1216(%rdi), %ymm22
-; AVX512DQ-BW-NEXT: vmovdqa 1152(%rdi), %ymm10
+; AVX512DQ-BW-NEXT: vmovdqa64 1152(%rdi), %ymm10
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
-; AVX512DQ-BW-NEXT: vmovdqa 1088(%rdi), %ymm9
-; AVX512DQ-BW-NEXT: vmovdqa 1024(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 1088(%rdi), %ymm9
+; AVX512DQ-BW-NEXT: vmovdqa64 1024(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm4, %zmm2, %zmm1
@@ -11886,8 +11886,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm11, %zmm4
; AVX512DQ-BW-NEXT: vpermt2q %zmm7, %zmm0, %zmm4
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512DQ-BW-NEXT: vmovdqa 640(%rdi), %ymm13
+; AVX512DQ-BW-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512DQ-BW-NEXT: vmovdqa64 640(%rdi), %ymm13
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %ymm17
; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %ymm21
@@ -11900,8 +11900,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, %zmm11
; AVX512DQ-BW-NEXT: vpermt2q %zmm20, %zmm0, %zmm11
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm14
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm14
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %ymm25
; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %ymm28
@@ -11915,11 +11915,11 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 1792(%rdi), %zmm26
; AVX512DQ-BW-NEXT: vpermi2q %zmm12, %zmm26, %zmm0
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 1728(%rdi), %ymm2
-; AVX512DQ-BW-NEXT: vmovdqa 1664(%rdi), %ymm8
+; AVX512DQ-BW-NEXT: vmovdqa64 1728(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 1664(%rdi), %ymm8
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
; AVX512DQ-BW-NEXT: vmovdqa64 1600(%rdi), %ymm30
-; AVX512DQ-BW-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm11, %zmm0, %zmm0
@@ -12232,25 +12232,25 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm27, %zmm10
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm31, %zmm25 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm0
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm1, %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm25, %zmm2
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 576(%rdi), %xmm5
-; AVX512DQ-BW-NEXT: vinserti128 $1, 704(%rdi), %ymm5, %ymm5
-; AVX512DQ-BW-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512DQ-BW-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %xmm5
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm18, %zmm7, %zmm27
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 1088(%rdi), %xmm13
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512DQ-BW-NEXT: vmovdqa64 1088(%rdi), %xmm13
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm13, %ymm13
; AVX512DQ-BW-NEXT: vmovdqa64 1024(%rdi), %xmm18
; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
@@ -12391,10 +12391,10 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm19, %zmm0, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1216(%rdi), %ymm22
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1152(%rdi), %ymm10
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1152(%rdi), %ymm10
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1088(%rdi), %ymm9
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1024(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm9
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm2, %zmm1
@@ -12404,8 +12404,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm11, %zmm4
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm7, %zmm0, %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 704(%rdi), %ymm7
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 640(%rdi), %ymm13
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 704(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm13
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm17
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %ymm21
@@ -12418,8 +12418,8 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm11
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm20, %zmm0, %zmm11
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm14
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm14
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %ymm25
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %ymm28
@@ -12433,11 +12433,11 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1792(%rdi), %zmm26
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm12, %zmm26, %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1728(%rdi), %ymm2
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1664(%rdi), %ymm8
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1728(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1664(%rdi), %ymm8
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1600(%rdi), %ymm30
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1536(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm11, %zmm0, %zmm0
@@ -12750,25 +12750,25 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm27, %zmm10
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm31, %zmm25 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm0
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm1, %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm25, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 576(%rdi), %xmm5
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm5, %ymm5
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 512(%rdi), %xmm6
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %xmm5
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %xmm6
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm6, %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm18, %zmm7, %zmm27
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1088(%rdi), %xmm13
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm13
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm13, %ymm13
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1024(%rdi), %xmm18
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
@@ -18974,14 +18974,14 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm16, %zmm1
; AVX512-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 3200(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 3200(%rdi), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512-NEXT: vmovdqa 3136(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 3072(%rdi), %ymm7
+; AVX512-NEXT: vmovdqa64 3136(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 3072(%rdi), %ymm7
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -18993,8 +18993,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512-NEXT: vpermt2q %zmm4, %zmm2, %zmm1
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa64 640(%rdi), %ymm20
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
; AVX512-NEXT: vmovdqa64 576(%rdi), %ymm22
@@ -19008,8 +19008,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512-NEXT: vpermt2q %zmm12, %zmm2, %zmm1
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm14
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm14
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512-NEXT: vmovdqa64 64(%rdi), %ymm21
; AVX512-NEXT: vmovdqa (%rdi), %ymm13
@@ -19025,15 +19025,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm10, %zmm2, %zmm1
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa 1728(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 1664(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 1728(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 1664(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512-NEXT: vmovdqa 1600(%rdi), %ymm4
-; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 1536(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 1600(%rdi), %ymm4
+; AVX512-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 1536(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -19049,15 +19049,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa 1216(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 1152(%rdi), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 1216(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 1152(%rdi), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512-NEXT: vmovdqa 1088(%rdi), %ymm4
-; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 1088(%rdi), %ymm4
+; AVX512-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -19096,11 +19096,11 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa 2240(%rdi), %ymm12
-; AVX512-NEXT: vmovdqa 2176(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 2240(%rdi), %ymm12
+; AVX512-NEXT: vmovdqa64 2176(%rdi), %ymm11
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
-; AVX512-NEXT: vmovdqa 2112(%rdi), %ymm10
-; AVX512-NEXT: vmovdqa 2048(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 2112(%rdi), %ymm10
+; AVX512-NEXT: vmovdqa64 2048(%rdi), %ymm3
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -19119,8 +19119,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqa64 3776(%rdi), %ymm17
; AVX512-NEXT: vmovdqa64 3712(%rdi), %ymm23
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
-; AVX512-NEXT: vmovdqa 3648(%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 3584(%rdi), %ymm0
+; AVX512-NEXT: vmovdqa64 3648(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 3584(%rdi), %ymm0
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm5, %zmm2, %zmm2
@@ -19178,10 +19178,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512-NEXT: vpermt2q %zmm13, %zmm2, %zmm6
; AVX512-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -19192,10 +19192,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
; AVX512-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
; AVX512-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -19792,32 +19792,32 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vmovdqa64 %zmm26, %zmm17 {%k1}
; AVX512-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512-NEXT: vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdi), %ymm8, %ymm8
; AVX512-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm28, %zmm17, %zmm26
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
-; AVX512-NEXT: vmovdqa 576(%rdi), %xmm13
-; AVX512-NEXT: vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512-NEXT: vmovdqa64 576(%rdi), %xmm13
+; AVX512-NEXT: vinserti32x4 $1, 704(%rdi), %ymm13, %ymm13
; AVX512-NEXT: vmovdqa64 512(%rdi), %xmm28
; AVX512-NEXT: vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm1
; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vmovdqa64 %zmm24, %zmm16 {%k1}
-; AVX512-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512-NEXT: vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
-; AVX512-NEXT: vmovdqa 1024(%rdi), %xmm7
-; AVX512-NEXT: vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512-NEXT: vmovdqa64 1024(%rdi), %xmm7
+; AVX512-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm7, %ymm7
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm5, %zmm16, %zmm24
; AVX512-NEXT: vmovdqa64 %zmm29, %zmm19 {%k1}
-; AVX512-NEXT: vmovdqa 1600(%rdi), %xmm5
-; AVX512-NEXT: vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
-; AVX512-NEXT: vmovdqa 1536(%rdi), %xmm11
-; AVX512-NEXT: vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512-NEXT: vmovdqa64 1600(%rdi), %xmm5
+; AVX512-NEXT: vinserti32x4 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512-NEXT: vmovdqa64 1536(%rdi), %xmm11
+; AVX512-NEXT: vinserti32x4 $1, 1664(%rdi), %ymm11, %ymm11
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm29, %zmm19, %zmm19
; AVX512-NEXT: vmovdqa64 %zmm21, %zmm23 {%k1}
@@ -20044,14 +20044,14 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm16, %zmm1
; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 3200(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 3200(%rdi), %ymm0
; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512-FCP-NEXT: vmovdqa 3136(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 3072(%rdi), %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 3136(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 3072(%rdi), %ymm7
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -20063,8 +20063,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512-FCP-NEXT: vpermt2q %zmm4, %zmm2, %zmm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 640(%rdi), %ymm20
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %ymm22
@@ -20078,8 +20078,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512-FCP-NEXT: vpermt2q %zmm12, %zmm2, %zmm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm14
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm14
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %ymm21
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm13
@@ -20095,15 +20095,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm10, %zmm2, %zmm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 1728(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1664(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1728(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1664(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512-FCP-NEXT: vmovdqa 1600(%rdi), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1536(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1600(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -20119,15 +20119,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 1216(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1152(%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1216(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1152(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512-FCP-NEXT: vmovdqa 1088(%rdi), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -20166,11 +20166,11 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 2240(%rdi), %ymm12
-; AVX512-FCP-NEXT: vmovdqa 2176(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 2240(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 2176(%rdi), %ymm11
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
-; AVX512-FCP-NEXT: vmovdqa 2112(%rdi), %ymm10
-; AVX512-FCP-NEXT: vmovdqa 2048(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 2112(%rdi), %ymm10
+; AVX512-FCP-NEXT: vmovdqa64 2048(%rdi), %ymm3
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -20189,8 +20189,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqa64 3776(%rdi), %ymm17
; AVX512-FCP-NEXT: vmovdqa64 3712(%rdi), %ymm23
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
-; AVX512-FCP-NEXT: vmovdqa 3648(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 3584(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 3648(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 3584(%rdi), %ymm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm2, %zmm2
@@ -20248,10 +20248,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512-FCP-NEXT: vpermt2q %zmm13, %zmm2, %zmm6
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -20262,10 +20262,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
; AVX512-FCP-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -20862,32 +20862,32 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 %zmm26, %zmm17 {%k1}
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm8, %ymm8
; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm28, %zmm17, %zmm26
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 576(%rdi), %xmm13
-; AVX512-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 576(%rdi), %xmm13
+; AVX512-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm13, %ymm13
; AVX512-FCP-NEXT: vmovdqa64 512(%rdi), %xmm28
; AVX512-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 %zmm24, %zmm16 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512-FCP-NEXT: vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
-; AVX512-FCP-NEXT: vmovdqa 1024(%rdi), %xmm7
-; AVX512-FCP-NEXT: vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512-FCP-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 1024(%rdi), %xmm7
+; AVX512-FCP-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm7, %ymm7
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm16, %zmm24
; AVX512-FCP-NEXT: vmovdqa64 %zmm29, %zmm19 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 1600(%rdi), %xmm5
-; AVX512-FCP-NEXT: vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
-; AVX512-FCP-NEXT: vmovdqa 1536(%rdi), %xmm11
-; AVX512-FCP-NEXT: vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 1600(%rdi), %xmm5
+; AVX512-FCP-NEXT: vinserti32x4 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm11
+; AVX512-FCP-NEXT: vinserti32x4 $1, 1664(%rdi), %ymm11, %ymm11
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm29, %zmm19, %zmm19
; AVX512-FCP-NEXT: vmovdqa64 %zmm21, %zmm23 {%k1}
@@ -21114,14 +21114,14 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm16, %zmm1
; AVX512DQ-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 3200(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 3200(%rdi), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-NEXT: vmovdqa 3136(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 3072(%rdi), %ymm7
+; AVX512DQ-NEXT: vmovdqa64 3136(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 3072(%rdi), %ymm7
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -21133,8 +21133,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512DQ-NEXT: vpermt2q %zmm4, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa64 640(%rdi), %ymm20
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %ymm22
@@ -21148,8 +21148,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512DQ-NEXT: vpermt2q %zmm12, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm14
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %ymm21
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm13
@@ -21165,15 +21165,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm10, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqa 1728(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 1664(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 1728(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 1664(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-NEXT: vmovdqa 1600(%rdi), %ymm4
-; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 1536(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 1600(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 1536(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -21189,15 +21189,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqa 1216(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 1152(%rdi), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 1216(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 1152(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-NEXT: vmovdqa 1088(%rdi), %ymm4
-; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 1088(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -21236,11 +21236,11 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqa 2240(%rdi), %ymm12
-; AVX512DQ-NEXT: vmovdqa 2176(%rdi), %ymm11
+; AVX512DQ-NEXT: vmovdqa64 2240(%rdi), %ymm12
+; AVX512DQ-NEXT: vmovdqa64 2176(%rdi), %ymm11
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
-; AVX512DQ-NEXT: vmovdqa 2112(%rdi), %ymm10
-; AVX512DQ-NEXT: vmovdqa 2048(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqa64 2112(%rdi), %ymm10
+; AVX512DQ-NEXT: vmovdqa64 2048(%rdi), %ymm3
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -21259,8 +21259,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqa64 3776(%rdi), %ymm17
; AVX512DQ-NEXT: vmovdqa64 3712(%rdi), %ymm23
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
-; AVX512DQ-NEXT: vmovdqa 3648(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 3584(%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 3648(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 3584(%rdi), %ymm0
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm5, %zmm2, %zmm2
@@ -21318,10 +21318,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512DQ-NEXT: vpermt2q %zmm13, %zmm2, %zmm6
; AVX512DQ-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -21332,10 +21332,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
; AVX512DQ-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -21932,32 +21932,32 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vmovdqa64 %zmm26, %zmm17 {%k1}
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdi), %ymm8, %ymm8
; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm28, %zmm17, %zmm26
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
-; AVX512DQ-NEXT: vmovdqa 576(%rdi), %xmm13
-; AVX512DQ-NEXT: vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512DQ-NEXT: vmovdqa64 576(%rdi), %xmm13
+; AVX512DQ-NEXT: vinserti32x4 $1, 704(%rdi), %ymm13, %ymm13
; AVX512DQ-NEXT: vmovdqa64 512(%rdi), %xmm28
; AVX512DQ-NEXT: vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm1
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vmovdqa64 %zmm24, %zmm16 {%k1}
-; AVX512DQ-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512DQ-NEXT: vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
-; AVX512DQ-NEXT: vmovdqa 1024(%rdi), %xmm7
-; AVX512DQ-NEXT: vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512DQ-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512DQ-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512DQ-NEXT: vmovdqa64 1024(%rdi), %xmm7
+; AVX512DQ-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm7, %ymm7
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm5, %zmm16, %zmm24
; AVX512DQ-NEXT: vmovdqa64 %zmm29, %zmm19 {%k1}
-; AVX512DQ-NEXT: vmovdqa 1600(%rdi), %xmm5
-; AVX512DQ-NEXT: vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
-; AVX512DQ-NEXT: vmovdqa 1536(%rdi), %xmm11
-; AVX512DQ-NEXT: vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512DQ-NEXT: vmovdqa64 1600(%rdi), %xmm5
+; AVX512DQ-NEXT: vinserti32x4 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512DQ-NEXT: vmovdqa64 1536(%rdi), %xmm11
+; AVX512DQ-NEXT: vinserti32x4 $1, 1664(%rdi), %ymm11, %ymm11
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm29, %zmm19, %zmm19
; AVX512DQ-NEXT: vmovdqa64 %zmm21, %zmm23 {%k1}
@@ -22184,14 +22184,14 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm16, %zmm1
; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 3200(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 3200(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 3136(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 3072(%rdi), %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 3136(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 3072(%rdi), %ymm7
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -22203,8 +22203,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512DQ-FCP-NEXT: vpermt2q %zmm4, %zmm2, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 640(%rdi), %ymm20
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %ymm22
@@ -22218,8 +22218,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512DQ-FCP-NEXT: vpermt2q %zmm12, %zmm2, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %ymm21
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm13
@@ -22235,15 +22235,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm10, %zmm2, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 1728(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1664(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1728(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1664(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 1600(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1536(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1600(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -22259,15 +22259,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 1216(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1152(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1216(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1152(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 1088(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -22306,11 +22306,11 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 2240(%rdi), %ymm12
-; AVX512DQ-FCP-NEXT: vmovdqa 2176(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 2240(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 2176(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 2112(%rdi), %ymm10
-; AVX512DQ-FCP-NEXT: vmovdqa 2048(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 2112(%rdi), %ymm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 2048(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -22329,8 +22329,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqa64 3776(%rdi), %ymm17
; AVX512DQ-FCP-NEXT: vmovdqa64 3712(%rdi), %ymm23
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 3648(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 3584(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 3648(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 3584(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm2, %zmm2
@@ -22388,10 +22388,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vpermt2q %zmm13, %zmm2, %zmm6
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -22402,10 +22402,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -23002,32 +23002,32 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm26, %zmm17 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm8, %ymm8
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm28, %zmm17, %zmm26
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 576(%rdi), %xmm13
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 576(%rdi), %xmm13
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm13, %ymm13
; AVX512DQ-FCP-NEXT: vmovdqa64 512(%rdi), %xmm28
; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm24, %zmm16 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 1024(%rdi), %xmm7
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 1024(%rdi), %xmm7
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm7, %ymm7
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm16, %zmm24
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm29, %zmm19 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 1600(%rdi), %xmm5
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 1536(%rdi), %xmm11
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 1600(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm11
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 1664(%rdi), %ymm11, %ymm11
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm29, %zmm19, %zmm19
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm21, %zmm23 {%k1}
@@ -23254,14 +23254,14 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 %zmm16, %zmm1
; AVX512BW-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 3200(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 3200(%rdi), %ymm0
; AVX512BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-NEXT: vmovdqa 3136(%rdi), %ymm3
-; AVX512BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 3072(%rdi), %ymm7
+; AVX512BW-NEXT: vmovdqa64 3136(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 3072(%rdi), %ymm7
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -23273,8 +23273,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512BW-NEXT: vpermt2q %zmm4, %zmm2, %zmm1
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vmovdqa64 640(%rdi), %ymm20
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
; AVX512BW-NEXT: vmovdqa64 576(%rdi), %ymm22
@@ -23288,8 +23288,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512BW-NEXT: vpermt2q %zmm12, %zmm2, %zmm1
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm14
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm14
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %ymm21
; AVX512BW-NEXT: vmovdqa (%rdi), %ymm13
@@ -23305,15 +23305,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpermt2q %zmm10, %zmm2, %zmm1
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT: vmovdqa 1728(%rdi), %ymm3
-; AVX512BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 1664(%rdi), %ymm0
-; AVX512BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 1728(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 1664(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-NEXT: vmovdqa 1600(%rdi), %ymm4
-; AVX512BW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 1536(%rdi), %ymm3
-; AVX512BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 1600(%rdi), %ymm4
+; AVX512BW-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 1536(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -23329,15 +23329,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
; AVX512BW-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT: vmovdqa 1216(%rdi), %ymm3
-; AVX512BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 1152(%rdi), %ymm0
-; AVX512BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 1216(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 1152(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-NEXT: vmovdqa 1088(%rdi), %ymm4
-; AVX512BW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 1088(%rdi), %ymm4
+; AVX512BW-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -23376,11 +23376,11 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT: vmovdqa 2240(%rdi), %ymm12
-; AVX512BW-NEXT: vmovdqa 2176(%rdi), %ymm11
+; AVX512BW-NEXT: vmovdqa64 2240(%rdi), %ymm12
+; AVX512BW-NEXT: vmovdqa64 2176(%rdi), %ymm11
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
-; AVX512BW-NEXT: vmovdqa 2112(%rdi), %ymm10
-; AVX512BW-NEXT: vmovdqa 2048(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqa64 2112(%rdi), %ymm10
+; AVX512BW-NEXT: vmovdqa64 2048(%rdi), %ymm3
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -23399,8 +23399,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqa64 3776(%rdi), %ymm17
; AVX512BW-NEXT: vmovdqa64 3712(%rdi), %ymm23
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
-; AVX512BW-NEXT: vmovdqa 3648(%rdi), %ymm1
-; AVX512BW-NEXT: vmovdqa 3584(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa64 3648(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 3584(%rdi), %ymm0
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm5, %zmm2, %zmm2
@@ -23458,10 +23458,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512BW-NEXT: vpermt2q %zmm13, %zmm2, %zmm6
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -23472,10 +23472,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
; AVX512BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -24072,32 +24072,32 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vmovdqa64 %zmm26, %zmm17 {%k1}
; AVX512BW-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm8, %ymm8
; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm28, %zmm17, %zmm26
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
-; AVX512BW-NEXT: vmovdqa 576(%rdi), %xmm13
-; AVX512BW-NEXT: vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512BW-NEXT: vmovdqa64 576(%rdi), %xmm13
+; AVX512BW-NEXT: vinserti32x4 $1, 704(%rdi), %ymm13, %ymm13
; AVX512BW-NEXT: vmovdqa64 512(%rdi), %xmm28
; AVX512BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm1
; AVX512BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vmovdqa64 %zmm24, %zmm16 {%k1}
-; AVX512BW-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512BW-NEXT: vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
-; AVX512BW-NEXT: vmovdqa 1024(%rdi), %xmm7
-; AVX512BW-NEXT: vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512BW-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512BW-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512BW-NEXT: vmovdqa64 1024(%rdi), %xmm7
+; AVX512BW-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm7, %ymm7
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm5, %zmm16, %zmm24
; AVX512BW-NEXT: vmovdqa64 %zmm29, %zmm19 {%k1}
-; AVX512BW-NEXT: vmovdqa 1600(%rdi), %xmm5
-; AVX512BW-NEXT: vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
-; AVX512BW-NEXT: vmovdqa 1536(%rdi), %xmm11
-; AVX512BW-NEXT: vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512BW-NEXT: vmovdqa64 1600(%rdi), %xmm5
+; AVX512BW-NEXT: vinserti32x4 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512BW-NEXT: vmovdqa64 1536(%rdi), %xmm11
+; AVX512BW-NEXT: vinserti32x4 $1, 1664(%rdi), %ymm11, %ymm11
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm29, %zmm19, %zmm19
; AVX512BW-NEXT: vmovdqa64 %zmm21, %zmm23 {%k1}
@@ -24324,14 +24324,14 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm16, %zmm1
; AVX512BW-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 3200(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 3200(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-FCP-NEXT: vmovdqa 3136(%rdi), %ymm3
-; AVX512BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 3072(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 3136(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 3072(%rdi), %ymm7
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -24343,8 +24343,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512BW-FCP-NEXT: vpermt2q %zmm4, %zmm2, %zmm1
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm20
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm22
@@ -24358,8 +24358,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512BW-FCP-NEXT: vpermt2q %zmm12, %zmm2, %zmm1
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm14
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm14
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %ymm21
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm13
@@ -24375,15 +24375,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm10, %zmm2, %zmm1
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 1728(%rdi), %ymm3
-; AVX512BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 1664(%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1728(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1664(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-FCP-NEXT: vmovdqa 1600(%rdi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 1536(%rdi), %ymm3
-; AVX512BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1600(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -24399,15 +24399,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 1216(%rdi), %ymm3
-; AVX512BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 1152(%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1216(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1152(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-FCP-NEXT: vmovdqa 1088(%rdi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -24446,11 +24446,11 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 2240(%rdi), %ymm12
-; AVX512BW-FCP-NEXT: vmovdqa 2176(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 2240(%rdi), %ymm12
+; AVX512BW-FCP-NEXT: vmovdqa64 2176(%rdi), %ymm11
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
-; AVX512BW-FCP-NEXT: vmovdqa 2112(%rdi), %ymm10
-; AVX512BW-FCP-NEXT: vmovdqa 2048(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqa64 2112(%rdi), %ymm10
+; AVX512BW-FCP-NEXT: vmovdqa64 2048(%rdi), %ymm3
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -24469,8 +24469,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqa64 3776(%rdi), %ymm17
; AVX512BW-FCP-NEXT: vmovdqa64 3712(%rdi), %ymm23
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
-; AVX512BW-FCP-NEXT: vmovdqa 3648(%rdi), %ymm1
-; AVX512BW-FCP-NEXT: vmovdqa 3584(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 3648(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 3584(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm2, %zmm2
@@ -24528,10 +24528,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512BW-FCP-NEXT: vpermt2q %zmm13, %zmm2, %zmm6
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -24542,10 +24542,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
; AVX512BW-FCP-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -25142,32 +25142,32 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm26, %zmm17 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512BW-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm8, %ymm8
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm28, %zmm17, %zmm26
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 576(%rdi), %xmm13
-; AVX512BW-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512BW-FCP-NEXT: vmovdqa64 576(%rdi), %xmm13
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm13, %ymm13
; AVX512BW-FCP-NEXT: vmovdqa64 512(%rdi), %xmm28
; AVX512BW-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm24, %zmm16 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512BW-FCP-NEXT: vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 1024(%rdi), %xmm7
-; AVX512BW-FCP-NEXT: vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 1024(%rdi), %xmm7
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm7, %ymm7
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm16, %zmm24
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm29, %zmm19 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 1600(%rdi), %xmm5
-; AVX512BW-FCP-NEXT: vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
-; AVX512BW-FCP-NEXT: vmovdqa 1536(%rdi), %xmm11
-; AVX512BW-FCP-NEXT: vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 1600(%rdi), %xmm5
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm11
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 1664(%rdi), %ymm11, %ymm11
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm29, %zmm19, %zmm19
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm21, %zmm23 {%k1}
@@ -25394,14 +25394,14 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm16, %zmm1
; AVX512DQ-BW-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 3200(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 3200(%rdi), %ymm0
; AVX512DQ-BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-BW-NEXT: vmovdqa 3136(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 3072(%rdi), %ymm7
+; AVX512DQ-BW-NEXT: vmovdqa64 3136(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 3072(%rdi), %ymm7
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -25413,8 +25413,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512DQ-BW-NEXT: vpermt2q %zmm4, %zmm2, %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa64 640(%rdi), %ymm20
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %ymm22
@@ -25428,8 +25428,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512DQ-BW-NEXT: vpermt2q %zmm12, %zmm2, %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm14
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm14
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %ymm21
; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm13
@@ -25445,15 +25445,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermt2q %zmm10, %zmm2, %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 1728(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 1664(%rdi), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 1728(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 1664(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-BW-NEXT: vmovdqa 1600(%rdi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 1536(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 1600(%rdi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 1536(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -25469,15 +25469,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 1216(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 1152(%rdi), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 1216(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 1152(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-BW-NEXT: vmovdqa 1088(%rdi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 1088(%rdi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -25516,11 +25516,11 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 2240(%rdi), %ymm12
-; AVX512DQ-BW-NEXT: vmovdqa 2176(%rdi), %ymm11
+; AVX512DQ-BW-NEXT: vmovdqa64 2240(%rdi), %ymm12
+; AVX512DQ-BW-NEXT: vmovdqa64 2176(%rdi), %ymm11
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
-; AVX512DQ-BW-NEXT: vmovdqa 2112(%rdi), %ymm10
-; AVX512DQ-BW-NEXT: vmovdqa 2048(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqa64 2112(%rdi), %ymm10
+; AVX512DQ-BW-NEXT: vmovdqa64 2048(%rdi), %ymm3
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -25539,8 +25539,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqa64 3776(%rdi), %ymm17
; AVX512DQ-BW-NEXT: vmovdqa64 3712(%rdi), %ymm23
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
-; AVX512DQ-BW-NEXT: vmovdqa 3648(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqa 3584(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 3648(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 3584(%rdi), %ymm0
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm5, %zmm2, %zmm2
@@ -25598,10 +25598,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512DQ-BW-NEXT: vpermt2q %zmm13, %zmm2, %zmm6
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -25612,10 +25612,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
; AVX512DQ-BW-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -26212,32 +26212,32 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm26, %zmm17 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdi), %ymm8, %ymm8
; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm28, %zmm17, %zmm26
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 576(%rdi), %xmm13
-; AVX512DQ-BW-NEXT: vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512DQ-BW-NEXT: vmovdqa64 576(%rdi), %xmm13
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 704(%rdi), %ymm13, %ymm13
; AVX512DQ-BW-NEXT: vmovdqa64 512(%rdi), %xmm28
; AVX512DQ-BW-NEXT: vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm24, %zmm16 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 1024(%rdi), %xmm7
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512DQ-BW-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 1024(%rdi), %xmm7
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm7, %ymm7
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm5, %zmm16, %zmm24
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm29, %zmm19 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 1600(%rdi), %xmm5
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
-; AVX512DQ-BW-NEXT: vmovdqa 1536(%rdi), %xmm11
-; AVX512DQ-BW-NEXT: vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512DQ-BW-NEXT: vmovdqa64 1600(%rdi), %xmm5
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 1536(%rdi), %xmm11
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 1664(%rdi), %ymm11, %ymm11
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm29, %zmm19, %zmm19
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm21, %zmm23 {%k1}
@@ -26464,14 +26464,14 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm16, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3264(%rdi), %ymm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3200(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3264(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3200(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3136(%rdi), %ymm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3072(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3136(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3072(%rdi), %ymm7
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -26483,8 +26483,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm4, %zmm2, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 704(%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 704(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 640(%rdi), %ymm20
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %ymm22
@@ -26498,8 +26498,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm15, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm12, %zmm2, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm14
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm14
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %ymm21
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm13
@@ -26515,15 +26515,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm10, %zmm2, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1728(%rdi), %ymm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1664(%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1728(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1664(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1600(%rdi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1536(%rdi), %ymm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1600(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1536(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -26539,15 +26539,15 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1216(%rdi), %ymm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1152(%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1216(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1152(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1088(%rdi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1024(%rdi), %ymm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1088(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1024(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -26586,11 +26586,11 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm3, %zmm2, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2240(%rdi), %ymm12
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2176(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2240(%rdi), %ymm12
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2176(%rdi), %ymm11
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2112(%rdi), %ymm10
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 2048(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2112(%rdi), %ymm10
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 2048(%rdi), %ymm3
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm0
@@ -26609,8 +26609,8 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3776(%rdi), %ymm17
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3712(%rdi), %ymm23
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3648(%rdi), %ymm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 3584(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3648(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 3584(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm2, %zmm2
@@ -26668,10 +26668,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm13, %zmm2, %zmm6
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -26682,10 +26682,10 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
@@ -27282,32 +27282,32 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm26, %zmm17 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %xmm8
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 192(%rdi), %ymm8, %ymm8
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm28, %zmm17, %zmm26
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 576(%rdi), %xmm13
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 576(%rdi), %xmm13
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 704(%rdi), %ymm13, %ymm13
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 512(%rdi), %xmm28
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm24, %zmm16 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1088(%rdi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1024(%rdi), %xmm7
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1088(%rdi), %xmm4
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1024(%rdi), %xmm7
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 1152(%rdi), %ymm7, %ymm7
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm16, %zmm24
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm29, %zmm19 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1600(%rdi), %xmm5
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 1536(%rdi), %xmm11
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1600(%rdi), %xmm5
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 1536(%rdi), %xmm11
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 1664(%rdi), %ymm11, %ymm11
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm29, %zmm19, %zmm19
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm21, %zmm23 {%k1}
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-3.ll
index a238371f0acbf..9d905157f9a51 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-3.ll
@@ -2409,20 +2409,20 @@ define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512-NEXT: vmovdqa 96(%rdi), %xmm3
; AVX512-NEXT: vmovdqa 112(%rdi), %xmm4
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm5
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm5
; AVX512-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
; AVX512-NEXT: # ymm6 = mem[0,1,0,1]
; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm0
-; AVX512-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
+; AVX512-NEXT: vinserti32x4 $1, 144(%rdi), %ymm3, %ymm3
; AVX512-NEXT: vpshufb %ymm6, %ymm3, %ymm3
; AVX512-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
; AVX512-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm4
+; AVX512-NEXT: vinserti32x4 $1, 160(%rdi), %ymm4, %ymm4
; AVX512-NEXT: vpshufb %ymm6, %ymm4, %ymm4
; AVX512-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
; AVX512-NEXT: vpshufb %ymm6, %ymm2, %ymm2
-; AVX512-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm5
+; AVX512-NEXT: vinserti32x4 $1, 176(%rdi), %ymm5, %ymm5
; AVX512-NEXT: vpshufb %ymm6, %ymm5, %ymm5
; AVX512-NEXT: vpalignr {{.*#+}} ymm6 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]
; AVX512-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]
@@ -2456,20 +2456,20 @@ define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm3
; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm4
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %xmm5
; AVX512-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
; AVX512-FCP-NEXT: # ymm6 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
-; AVX512-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
+; AVX512-FCP-NEXT: vinserti32x4 $1, 144(%rdi), %ymm3, %ymm3
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
; AVX512-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm4
+; AVX512-FCP-NEXT: vinserti32x4 $1, 160(%rdi), %ymm4, %ymm4
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm4
; AVX512-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
-; AVX512-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm5
+; AVX512-FCP-NEXT: vinserti32x4 $1, 176(%rdi), %ymm5, %ymm5
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm5
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm6 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]
; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]
@@ -2503,20 +2503,20 @@ define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm3
; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm4
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm5
; AVX512DQ-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
; AVX512DQ-NEXT: # ymm6 = mem[0,1,0,1]
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
+; AVX512DQ-NEXT: vinserti32x4 $1, 144(%rdi), %ymm3, %ymm3
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm3, %ymm3
; AVX512DQ-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm4
+; AVX512DQ-NEXT: vinserti32x4 $1, 160(%rdi), %ymm4, %ymm4
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm4, %ymm4
; AVX512DQ-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm2, %ymm2
-; AVX512DQ-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm5
+; AVX512DQ-NEXT: vinserti32x4 $1, 176(%rdi), %ymm5, %ymm5
; AVX512DQ-NEXT: vpshufb %ymm6, %ymm5, %ymm5
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm6 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]
; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]
@@ -2550,20 +2550,20 @@ define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
; AVX512DQ-FCP-NEXT: # ymm6 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 144(%rdi), %ymm3, %ymm3
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
; AVX512DQ-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 160(%rdi), %ymm4, %ymm4
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm4
; AVX512DQ-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm5
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 176(%rdi), %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm6 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]
; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]
@@ -2597,14 +2597,14 @@ define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm3
; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm4
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512BW-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512BW-NEXT: vinserti32x4 $1, 144(%rdi), %ymm3, %ymm3
; AVX512BW-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
; AVX512BW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512BW-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm3
+; AVX512BW-NEXT: vinserti32x4 $1, 160(%rdi), %ymm4, %ymm3
; AVX512BW-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
; AVX512BW-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512BW-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm3
+; AVX512BW-NEXT: vinserti32x4 $1, 176(%rdi), %ymm5, %ymm3
; AVX512BW-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
; AVX512BW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
@@ -2634,14 +2634,14 @@ define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512BW-FCP-NEXT: vmovdqa 96(%rdi), %xmm3
; AVX512BW-FCP-NEXT: vmovdqa 112(%rdi), %xmm4
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512BW-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 144(%rdi), %ymm3, %ymm3
; AVX512BW-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512BW-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm3
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 160(%rdi), %ymm4, %ymm3
; AVX512BW-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512BW-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm3
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 176(%rdi), %ymm5, %ymm3
; AVX512BW-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
; AVX512BW-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
@@ -2671,14 +2671,14 @@ define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm3
; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm4
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512DQ-BW-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 144(%rdi), %ymm3, %ymm3
; AVX512DQ-BW-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-BW-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm3
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 160(%rdi), %ymm4, %ymm3
; AVX512DQ-BW-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm3
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 176(%rdi), %ymm5, %ymm3
; AVX512DQ-BW-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
; AVX512DQ-BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
@@ -2708,14 +2708,14 @@ define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqa 96(%rdi), %xmm3
; AVX512DQ-BW-FCP-NEXT: vmovdqa 112(%rdi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 144(%rdi), %ymm3, %ymm3
; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm0
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm3
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 160(%rdi), %ymm4, %ymm3
; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm1
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm3
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 176(%rdi), %ymm5, %ymm3
; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm2
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2
; AVX512DQ-BW-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll
index 2477b8df3799b..2e553fc632f59 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll
@@ -2993,9 +2993,9 @@ define void @load_i8_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm2
; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm3
; AVX512-NEXT: vpshufb %ymm7, %ymm3, %ymm5
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm4
; AVX512-NEXT: vpshufb %ymm7, %ymm4, %ymm6
; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,4,0,4,0,4,8,12]
; AVX512-NEXT: vpermt2d %ymm5, %ymm1, %ymm6
@@ -3063,9 +3063,9 @@ define void @load_i8_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm3
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm3, %ymm5
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm4
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm6
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,4,0,4,0,4,8,12]
; AVX512-FCP-NEXT: vpermt2d %ymm5, %ymm1, %ymm6
@@ -3133,9 +3133,9 @@ define void @load_i8_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm2
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm3
; AVX512DQ-NEXT: vpshufb %ymm7, %ymm3, %ymm5
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm4
; AVX512DQ-NEXT: vpshufb %ymm7, %ymm4, %ymm6
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,4,0,4,0,4,8,12]
; AVX512DQ-NEXT: vpermt2d %ymm5, %ymm1, %ymm6
@@ -3203,9 +3203,9 @@ define void @load_i8_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %zmm2
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm3, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm6
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,4,0,4,0,4,8,12]
; AVX512DQ-FCP-NEXT: vpermt2d %ymm5, %ymm1, %ymm6
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-5.ll
index 98c8605164ca3..37f3487645c10 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-5.ll
@@ -3215,9 +3215,9 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpor %xmm7, %xmm9, %xmm9
; AVX512-NEXT: vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm9 = (ymm9 & ymm11) | ymm6
-; AVX512-NEXT: vmovdqa 144(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 144(%rdi), %xmm7
; AVX512-NEXT: vpshufb %xmm8, %xmm7, %xmm6
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm8
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm8
; AVX512-NEXT: vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512-NEXT: vpor %xmm6, %xmm10, %xmm6
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
@@ -3290,7 +3290,7 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm4))
; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
; AVX512-NEXT: vpermd %ymm1, %ymm2, %ymm1
@@ -3325,9 +3325,9 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpor %xmm7, %xmm9, %xmm9
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = (ymm9 & ymm11) | ymm6
-; AVX512-FCP-NEXT: vmovdqa 144(%rdi), %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 144(%rdi), %xmm7
; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm7, %xmm6
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %xmm8
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %xmm8
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512-FCP-NEXT: vpor %xmm6, %xmm10, %xmm6
; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
@@ -3400,7 +3400,7 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm4))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
; AVX512-FCP-NEXT: vpermd %ymm1, %ymm2, %ymm1
@@ -3435,9 +3435,9 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpor %xmm7, %xmm9, %xmm9
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm9 = (ymm9 & ymm11) | ymm6
-; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 144(%rdi), %xmm7
; AVX512DQ-NEXT: vpshufb %xmm8, %xmm7, %xmm6
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm8
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm8
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512DQ-NEXT: vpor %xmm6, %xmm10, %xmm6
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
@@ -3510,7 +3510,7 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm4))
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
; AVX512DQ-NEXT: vpermd %ymm1, %ymm2, %ymm1
@@ -3545,9 +3545,9 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpor %xmm7, %xmm9, %xmm9
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm9 = (ymm9 & ymm11) | ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa 144(%rdi), %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 144(%rdi), %xmm7
; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm7, %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %xmm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %xmm8
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm10, %xmm6
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
@@ -3620,7 +3620,7 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm4))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm2, %ymm1
@@ -3656,9 +3656,9 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: movl $67100672, %eax # imm = 0x3FFE000
; AVX512BW-NEXT: kmovd %eax, %k3
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm5 {%k3} = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,1,6,11,16,21,26,31,20,25,30,19,24,29,u,u,u,u,u,u]
-; AVX512BW-NEXT: vmovdqa 144(%rdi), %xmm6
+; AVX512BW-NEXT: vmovdqa64 144(%rdi), %xmm6
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm4 = xmm6[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm6[1,6,11]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm7
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm7
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512BW-NEXT: vpor %xmm4, %xmm8, %xmm4
; AVX512BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
@@ -3732,7 +3732,7 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512BW-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
; AVX512BW-NEXT: vpermd %ymm1, %ymm2, %ymm1
@@ -3768,9 +3768,9 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: movl $67100672, %eax # imm = 0x3FFE000
; AVX512BW-FCP-NEXT: kmovd %eax, %k3
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm3 {%k3} = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,1,6,11,16,21,26,31,20,25,30,19,24,29,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT: vmovdqa 144(%rdi), %xmm6
+; AVX512BW-FCP-NEXT: vmovdqa64 144(%rdi), %xmm6
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm6[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm6[1,6,11]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm7
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm7
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512BW-FCP-NEXT: vpor %xmm5, %xmm8, %xmm5
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,21,22,23]
@@ -3839,7 +3839,7 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
; AVX512BW-FCP-NEXT: vpermd %ymm1, %ymm2, %ymm1
@@ -3875,9 +3875,9 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: movl $67100672, %eax # imm = 0x3FFE000
; AVX512DQ-BW-NEXT: kmovd %eax, %k3
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm5 {%k3} = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,1,6,11,16,21,26,31,20,25,30,19,24,29,u,u,u,u,u,u]
-; AVX512DQ-BW-NEXT: vmovdqa 144(%rdi), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 144(%rdi), %xmm6
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm4 = xmm6[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm6[1,6,11]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm7
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm7
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512DQ-BW-NEXT: vpor %xmm4, %xmm8, %xmm4
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm4
@@ -3951,7 +3951,7 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
; AVX512DQ-BW-NEXT: vpermd %ymm1, %ymm2, %ymm1
@@ -3987,9 +3987,9 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: movl $67100672, %eax # imm = 0x3FFE000
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k3
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm3 {%k3} = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,1,6,11,16,21,26,31,20,25,30,19,24,29,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 144(%rdi), %xmm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 144(%rdi), %xmm6
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm6[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm6[1,6,11]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm7
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512DQ-BW-FCP-NEXT: vpor %xmm5, %xmm8, %xmm5
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,21,22,23]
@@ -4058,7 +4058,7 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm1, %ymm2, %ymm1
@@ -6432,29 +6432,29 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 {{.*#+}} ymm20 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm20) | ymm7
; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm26
-; AVX512-NEXT: vmovdqa 224(%rdi), %ymm8
+; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm8
; AVX512-NEXT: vmovdqa %ymm4, %ymm11
; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm26 ^ (ymm11 & (ymm8 ^ ymm26))
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm9
+; AVX512-NEXT: vmovdqa64 208(%rdi), %xmm9
; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm9 ^ (mem & (ymm11 ^ ymm9))
; AVX512-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,ymm11[3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm12
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm12
; AVX512-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,5,10,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,25,30,u,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm17 = [0,5,0,5,0,5,0,5]
; AVX512-NEXT: vpermd %ymm12, %ymm17, %ymm15
; AVX512-NEXT: vmovdqa64 {{.*#+}} ymm16 = [0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm15 = (ymm15 & ~ymm16) | ymm11
-; AVX512-NEXT: vmovdqa 144(%rdi), %xmm12
+; AVX512-NEXT: vmovdqa64 144(%rdi), %xmm12
; AVX512-NEXT: vpshufb %xmm6, %xmm12, %xmm6
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm13
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm13
; AVX512-NEXT: vpshufb {{.*#+}} xmm11 = xmm13[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512-NEXT: vpor %xmm6, %xmm11, %xmm6
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vinserti64x4 $1, %ymm15, %zmm6, %zmm6
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm21 & (zmm6 ^ zmm10))
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm14
-; AVX512-NEXT: vmovdqa 288(%rdi), %ymm11
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm14
+; AVX512-NEXT: vmovdqa64 288(%rdi), %ymm11
; AVX512-NEXT: vmovdqa %ymm5, %ymm10
; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm14 ^ (ymm10 & (ymm11 ^ ymm14))
; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm0
@@ -6475,9 +6475,9 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm6 = ymm26 ^ (ymm6 & (ymm8 ^ ymm26))
; AVX512-NEXT: vpternlogq {{.*#+}} ymm6 = ymm9 ^ (mem & (ymm6 ^ ymm9))
; AVX512-NEXT: vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,ymm6[4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vmovdqa 160(%rdi), %xmm15
+; AVX512-NEXT: vmovdqa64 160(%rdi), %xmm15
; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm15[1,6,11],zero,zero,zero,zero,xmm15[u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vmovdqa 176(%rdi), %xmm6
+; AVX512-NEXT: vmovdqa64 176(%rdi), %xmm6
; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm6[0,5,10,15,u,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = (ymm1 & ~ymm16) | ymm10
@@ -6601,7 +6601,7 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 ^ (mem & (ymm5 ^ ymm4))
; AVX512-NEXT: vpshufb {{.*#+}} ymm4 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3,4,5,6,7]
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vpermd %ymm4, %ymm17, %ymm4
; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm1 ^ (zmm2 & (zmm4 ^ zmm1))
@@ -6637,29 +6637,29 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} ymm20 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm20) | ymm7
; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm26
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm8
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm8
; AVX512-FCP-NEXT: vmovdqa %ymm4, %ymm11
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm26 ^ (ymm11 & (ymm8 ^ ymm26))
-; AVX512-FCP-NEXT: vmovdqa 208(%rdi), %xmm9
+; AVX512-FCP-NEXT: vmovdqa64 208(%rdi), %xmm9
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm9 ^ (mem & (ymm11 ^ ymm9))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,ymm11[3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm12
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,5,10,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,25,30,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm17 = [0,5,0,5,0,5,0,5]
; AVX512-FCP-NEXT: vpermd %ymm12, %ymm17, %ymm14
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = (ymm14 & ~ymm16) | ymm11
-; AVX512-FCP-NEXT: vmovdqa 144(%rdi), %xmm12
+; AVX512-FCP-NEXT: vmovdqa64 144(%rdi), %xmm12
; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm12, %xmm6
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %xmm13
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %xmm13
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm11 = xmm13[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512-FCP-NEXT: vpor %xmm6, %xmm11, %xmm6
; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm14, %zmm6, %zmm6
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm21 & (zmm6 ^ zmm10))
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm14
-; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm14
+; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %ymm11
; AVX512-FCP-NEXT: vmovdqa %ymm5, %ymm10
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm14 ^ (ymm10 & (ymm11 ^ ymm14))
; AVX512-FCP-NEXT: vextracti128 $1, %ymm10, %xmm15
@@ -6680,9 +6680,9 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm26 ^ (ymm6 & (ymm8 ^ ymm26))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm9 ^ (mem & (ymm6 ^ ymm9))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,ymm6[4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %xmm15
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %xmm15
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm15[1,6,11],zero,zero,zero,zero,xmm15[u,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT: vmovdqa 176(%rdi), %xmm6
+; AVX512-FCP-NEXT: vmovdqa64 176(%rdi), %xmm6
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm6[0,5,10,15,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = (ymm1 & ~ymm16) | ymm0
@@ -6806,7 +6806,7 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 ^ (mem & (ymm5 ^ ymm4))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3,4,5,6,7]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpermd %ymm4, %ymm17, %ymm4
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm1 ^ (zmm2 & (zmm4 ^ zmm1))
@@ -6842,29 +6842,29 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} ymm20 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm20) | ymm7
; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm26
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %ymm8
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm8
; AVX512DQ-NEXT: vmovdqa %ymm4, %ymm11
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm26 ^ (ymm11 & (ymm8 ^ ymm26))
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm9
+; AVX512DQ-NEXT: vmovdqa64 208(%rdi), %xmm9
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm9 ^ (mem & (ymm11 ^ ymm9))
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,ymm11[3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm12
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm12
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,5,10,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,25,30,u,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm17 = [0,5,0,5,0,5,0,5]
; AVX512DQ-NEXT: vpermd %ymm12, %ymm17, %ymm15
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} ymm16 = [0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm15 = (ymm15 & ~ymm16) | ymm11
-; AVX512DQ-NEXT: vmovdqa 144(%rdi), %xmm12
+; AVX512DQ-NEXT: vmovdqa64 144(%rdi), %xmm12
; AVX512DQ-NEXT: vpshufb %xmm6, %xmm12, %xmm6
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm13
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm13
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm11 = xmm13[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512DQ-NEXT: vpor %xmm6, %xmm11, %xmm6
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm15, %zmm6, %zmm6
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm21 & (zmm6 ^ zmm10))
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm14
-; AVX512DQ-NEXT: vmovdqa 288(%rdi), %ymm11
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 288(%rdi), %ymm11
; AVX512DQ-NEXT: vmovdqa %ymm5, %ymm10
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm10 = ymm14 ^ (ymm10 & (ymm11 ^ ymm14))
; AVX512DQ-NEXT: vextracti128 $1, %ymm10, %xmm0
@@ -6885,9 +6885,9 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm6 = ymm26 ^ (ymm6 & (ymm8 ^ ymm26))
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm6 = ymm9 ^ (mem & (ymm6 ^ ymm9))
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,ymm6[4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm15
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %xmm15
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm15[1,6,11],zero,zero,zero,zero,xmm15[u,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 176(%rdi), %xmm6
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm6[0,5,10,15,u,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = (ymm1 & ~ymm16) | ymm10
@@ -7011,7 +7011,7 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 ^ (mem & (ymm5 ^ ymm4))
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm4 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3,4,5,6,7]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpermd %ymm4, %ymm17, %ymm4
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm4 = zmm1 ^ (zmm2 & (zmm4 ^ zmm1))
@@ -7047,29 +7047,29 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} ymm20 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = (ymm10 & ymm20) | ymm7
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm26
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm8
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm8
; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, %ymm11
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm26 ^ (ymm11 & (ymm8 ^ ymm26))
-; AVX512DQ-FCP-NEXT: vmovdqa 208(%rdi), %xmm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 208(%rdi), %xmm9
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm9 ^ (mem & (ymm11 ^ ymm9))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,ymm11[3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm12
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm12 = ymm12[0,5,10,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,25,30,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm17 = [0,5,0,5,0,5,0,5]
; AVX512DQ-FCP-NEXT: vpermd %ymm12, %ymm17, %ymm14
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = (ymm14 & ~ymm16) | ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa 144(%rdi), %xmm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 144(%rdi), %xmm12
; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm12, %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %xmm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %xmm13
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm11 = xmm13[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm11, %xmm6
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm14, %zmm6, %zmm6
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm21 & (zmm6 ^ zmm10))
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vmovdqa %ymm5, %ymm10
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm14 ^ (ymm10 & (ymm11 ^ ymm14))
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm10, %xmm15
@@ -7090,9 +7090,9 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm26 ^ (ymm6 & (ymm8 ^ ymm26))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm9 ^ (mem & (ymm6 ^ ymm9))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,ymm6[4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %xmm15
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm15[1,6,11],zero,zero,zero,zero,xmm15[u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vmovdqa 176(%rdi), %xmm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 176(%rdi), %xmm6
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm6[0,5,10,15,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = (ymm1 & ~ymm16) | ymm0
@@ -7216,7 +7216,7 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm5 ^ (mem & (ymm5 ^ ymm4))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm4
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm17, %ymm4
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm1 ^ (zmm2 & (zmm4 ^ zmm1))
@@ -7252,32 +7252,32 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: movl $67100672, %eax # imm = 0x3FFE000
; AVX512BW-NEXT: kmovd %eax, %k5
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm9 {%k5} = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,1,6,11,16,21,26,31,20,25,30,19,24,29,u,u,u,u,u,u]
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm6
-; AVX512BW-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 224(%rdi), %ymm5
; AVX512BW-NEXT: vpblendmw %ymm6, %ymm5, %ymm4 {%k1}
; AVX512BW-NEXT: vpermq {{.*#+}} ymm7 = ymm4[2,3,0,1]
; AVX512BW-NEXT: movl $4228, %eax # imm = 0x1084
; AVX512BW-NEXT: kmovd %eax, %k3
; AVX512BW-NEXT: vmovdqu8 %ymm7, %ymm4 {%k3}
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[u,u,u,u,u,u,u,3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %ymm4
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm8 = ymm4[0,5,10,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,25,30,u,u,u,u,u,u,u,u,u]
; AVX512BW-NEXT: vpbroadcastq {{.*#+}} ymm19 = [0,5,0,5,0,5,0,5]
; AVX512BW-NEXT: vpermd %ymm8, %ymm19, %ymm8
; AVX512BW-NEXT: movl $127, %eax
; AVX512BW-NEXT: kmovd %eax, %k4
; AVX512BW-NEXT: vmovdqu8 %ymm8, %ymm7 {%k4}
-; AVX512BW-NEXT: vmovdqa 144(%rdi), %xmm11
+; AVX512BW-NEXT: vmovdqa64 144(%rdi), %xmm11
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm8 = xmm11[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm11[1,6,11]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm12
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm12
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512BW-NEXT: vpor %xmm8, %xmm10, %xmm8
; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512BW-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
; AVX512BW-NEXT: vmovdqu16 %zmm7, %zmm9 {%k5}
; AVX512BW-NEXT: vextracti64x4 $1, %zmm9, %ymm10
-; AVX512BW-NEXT: vmovdqa 256(%rdi), %ymm8
-; AVX512BW-NEXT: vmovdqa 288(%rdi), %ymm7
+; AVX512BW-NEXT: vmovdqa64 256(%rdi), %ymm8
+; AVX512BW-NEXT: vmovdqa64 288(%rdi), %ymm7
; AVX512BW-NEXT: vpblendmw %ymm8, %ymm7, %ymm13 {%k2}
; AVX512BW-NEXT: vextracti128 $1, %ymm13, %xmm14
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u],zero,zero,zero,xmm14[3,8,13],zero,zero,zero,xmm14[1,6,11]
@@ -7305,9 +7305,9 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: kmovd %eax, %k6
; AVX512BW-NEXT: vmovdqu8 %ymm13, %ymm10 {%k6}
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm15 = ymm10[u,u,u,u,u,u,u,4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm10
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm10
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm16 = xmm10[1,6,11],zero,zero,zero,zero,xmm10[u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm13
+; AVX512BW-NEXT: vmovdqa64 176(%rdi), %xmm13
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm17 = zero,zero,zero,xmm13[0,5,10,15,u,u,u,u,u,u,u,u,u]
; AVX512BW-NEXT: vporq %xmm16, %xmm17, %xmm16
; AVX512BW-NEXT: vmovdqu8 %ymm16, %ymm15 {%k4}
@@ -7429,7 +7429,7 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX512BW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm1[3,4,5,6,7]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512BW-NEXT: vpermd %ymm2, %ymm19, %ymm2
; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
@@ -7474,31 +7474,31 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: movl $67100672, %eax # imm = 0x3FFE000
; AVX512BW-FCP-NEXT: kmovd %eax, %k5
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm9 {%k5} = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,1,6,11,16,21,26,31,20,25,30,19,24,29,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm6
-; AVX512BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 224(%rdi), %ymm5
; AVX512BW-FCP-NEXT: vpblendmw %ymm6, %ymm5, %ymm4 {%k1}
; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm4[2,3,0,1]
; AVX512BW-FCP-NEXT: movl $4228, %eax # imm = 0x1084
; AVX512BW-FCP-NEXT: kmovd %eax, %k3
; AVX512BW-FCP-NEXT: vmovdqu8 %ymm7, %ymm4 {%k3}
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[u,u,u,u,u,u,u,3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm4
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm4[0,5,10,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,25,30,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpbroadcastq {{.*#+}} ymm19 = [0,5,0,5,0,5,0,5]
; AVX512BW-FCP-NEXT: vpermd %ymm8, %ymm19, %ymm8
; AVX512BW-FCP-NEXT: movl $127, %eax
; AVX512BW-FCP-NEXT: kmovd %eax, %k4
; AVX512BW-FCP-NEXT: vmovdqu8 %ymm8, %ymm7 {%k4}
-; AVX512BW-FCP-NEXT: vmovdqa 144(%rdi), %xmm10
+; AVX512BW-FCP-NEXT: vmovdqa64 144(%rdi), %xmm10
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm8 = xmm10[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm10[1,6,11]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm11
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm11
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512BW-FCP-NEXT: vpor %xmm8, %xmm12, %xmm8
; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
; AVX512BW-FCP-NEXT: vmovdqu16 %zmm7, %zmm9 {%k5}
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm8
-; AVX512BW-FCP-NEXT: vmovdqa 288(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm8
+; AVX512BW-FCP-NEXT: vmovdqa64 288(%rdi), %ymm7
; AVX512BW-FCP-NEXT: vpblendmw %ymm8, %ymm7, %ymm12 {%k2}
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm12, %xmm13
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u],zero,zero,zero,xmm13[3,8,13],zero,zero,zero,xmm13[1,6,11]
@@ -7526,9 +7526,9 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: kmovd %eax, %k6
; AVX512BW-FCP-NEXT: vmovdqu8 %ymm13, %ymm12 {%k6}
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm12[u,u,u,u,u,u,u,4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %xmm12
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %xmm12
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm16 = xmm12[1,6,11],zero,zero,zero,zero,xmm12[u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT: vmovdqa 176(%rdi), %xmm13
+; AVX512BW-FCP-NEXT: vmovdqa64 176(%rdi), %xmm13
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm17 = zero,zero,zero,xmm13[0,5,10,15,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vporq %xmm16, %xmm17, %xmm16
; AVX512BW-FCP-NEXT: vmovdqu8 %ymm16, %ymm15 {%k4}
@@ -7650,7 +7650,7 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX512BW-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm1[3,4,5,6,7]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpermd %ymm2, %ymm19, %ymm2
; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
@@ -7695,32 +7695,32 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: movl $67100672, %eax # imm = 0x3FFE000
; AVX512DQ-BW-NEXT: kmovd %eax, %k5
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm9 {%k5} = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,1,6,11,16,21,26,31,20,25,30,19,24,29,u,u,u,u,u,u]
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm6
-; AVX512DQ-BW-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 224(%rdi), %ymm5
; AVX512DQ-BW-NEXT: vpblendmw %ymm6, %ymm5, %ymm4 {%k1}
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm7 = ymm4[2,3,0,1]
; AVX512DQ-BW-NEXT: movl $4228, %eax # imm = 0x1084
; AVX512DQ-BW-NEXT: kmovd %eax, %k3
; AVX512DQ-BW-NEXT: vmovdqu8 %ymm7, %ymm4 {%k3}
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[u,u,u,u,u,u,u,3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %ymm4
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm8 = ymm4[0,5,10,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,25,30,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-NEXT: vpbroadcastq {{.*#+}} ymm19 = [0,5,0,5,0,5,0,5]
; AVX512DQ-BW-NEXT: vpermd %ymm8, %ymm19, %ymm8
; AVX512DQ-BW-NEXT: movl $127, %eax
; AVX512DQ-BW-NEXT: kmovd %eax, %k4
; AVX512DQ-BW-NEXT: vmovdqu8 %ymm8, %ymm7 {%k4}
-; AVX512DQ-BW-NEXT: vmovdqa 144(%rdi), %xmm11
+; AVX512DQ-BW-NEXT: vmovdqa64 144(%rdi), %xmm11
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm8 = xmm11[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm11[1,6,11]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm12
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm12
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512DQ-BW-NEXT: vpor %xmm8, %xmm10, %xmm8
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
; AVX512DQ-BW-NEXT: vmovdqu16 %zmm7, %zmm9 {%k5}
; AVX512DQ-BW-NEXT: vextracti64x4 $1, %zmm9, %ymm10
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rdi), %ymm8
-; AVX512DQ-BW-NEXT: vmovdqa 288(%rdi), %ymm7
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %ymm8
+; AVX512DQ-BW-NEXT: vmovdqa64 288(%rdi), %ymm7
; AVX512DQ-BW-NEXT: vpblendmw %ymm8, %ymm7, %ymm13 {%k2}
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm13, %xmm14
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u],zero,zero,zero,xmm14[3,8,13],zero,zero,zero,xmm14[1,6,11]
@@ -7748,9 +7748,9 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: kmovd %eax, %k6
; AVX512DQ-BW-NEXT: vmovdqu8 %ymm13, %ymm10 {%k6}
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm15 = ymm10[u,u,u,u,u,u,u,4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm10
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %xmm10
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm16 = xmm10[1,6,11],zero,zero,zero,zero,xmm10[u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-NEXT: vmovdqa 176(%rdi), %xmm13
+; AVX512DQ-BW-NEXT: vmovdqa64 176(%rdi), %xmm13
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm17 = zero,zero,zero,xmm13[0,5,10,15,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-NEXT: vporq %xmm16, %xmm17, %xmm16
; AVX512DQ-BW-NEXT: vmovdqu8 %ymm16, %ymm15 {%k4}
@@ -7872,7 +7872,7 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX512DQ-BW-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm1[3,4,5,6,7]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-NEXT: vpermd %ymm2, %ymm19, %ymm2
; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
@@ -7917,31 +7917,31 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: movl $67100672, %eax # imm = 0x3FFE000
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k5
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm9 {%k5} = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,1,6,11,16,21,26,31,20,25,30,19,24,29,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 224(%rdi), %ymm5
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm6, %ymm5, %ymm4 {%k1}
; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm4[2,3,0,1]
; AVX512DQ-BW-FCP-NEXT: movl $4228, %eax # imm = 0x1084
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k3
; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %ymm7, %ymm4 {%k3}
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[u,u,u,u,u,u,u,3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm4
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm8 = ymm4[0,5,10,15,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,25,30,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpbroadcastq {{.*#+}} ymm19 = [0,5,0,5,0,5,0,5]
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm8, %ymm19, %ymm8
; AVX512DQ-BW-FCP-NEXT: movl $127, %eax
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k4
; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %ymm8, %ymm7 {%k4}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 144(%rdi), %xmm10
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 144(%rdi), %xmm10
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm8 = xmm10[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm10[1,6,11]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm11
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
; AVX512DQ-BW-FCP-NEXT: vpor %xmm8, %xmm12, %xmm8
; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7
; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm7, %zmm9 {%k5}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm8
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 288(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm8
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 288(%rdi), %ymm7
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm8, %ymm7, %ymm12 {%k2}
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm12, %xmm13
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u],zero,zero,zero,xmm13[3,8,13],zero,zero,zero,xmm13[1,6,11]
@@ -7969,9 +7969,9 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k6
; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %ymm13, %ymm12 {%k6}
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm12[u,u,u,u,u,u,u,4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %xmm12
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %xmm12
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm16 = xmm12[1,6,11],zero,zero,zero,zero,xmm12[u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 176(%rdi), %xmm13
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 176(%rdi), %xmm13
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm17 = zero,zero,zero,xmm13[0,5,10,15,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vporq %xmm16, %xmm17, %xmm16
; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %ymm16, %ymm15 {%k4}
@@ -8093,7 +8093,7 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2
; AVX512DQ-BW-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm1[3,4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm2, %ymm19, %ymm2
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
index 023f2241fd31f..251b989b3b496 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
@@ -3770,7 +3770,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 (%rdi), %ymm17
; AVX512-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512-NEXT: vmovdqa %ymm0, %ymm7
; AVX512-NEXT: vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm17 ^ ymm3))
; AVX512-NEXT: vpshufb {{.*#+}} xmm4 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
@@ -3785,7 +3785,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm13
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm13
; AVX512-NEXT: vmovdqa %ymm0, %ymm14
; AVX512-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
; AVX512-NEXT: vextracti128 $1, %ymm14, %xmm15
@@ -3880,7 +3880,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %ymm17
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm7
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm17 ^ ymm3))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
@@ -3895,7 +3895,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm13
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm14
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
; AVX512-FCP-NEXT: vextracti128 $1, %ymm14, %xmm15
@@ -3990,7 +3990,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %ymm17
; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm7
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm17 ^ ymm3))
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm4 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
@@ -4005,7 +4005,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm13
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm14
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
; AVX512DQ-NEXT: vextracti128 $1, %ymm14, %xmm15
@@ -4100,7 +4100,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %ymm17
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm7
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm3 ^ (ymm7 & (ymm17 ^ ymm3))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm7[0,6,12],zero,zero,zero,xmm7[4,10],zero,zero,zero,xmm7[u,u,u,u,u]
@@ -4115,7 +4115,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,8,14,4,10,16,22,28,18,24,30],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm11 | (ymm4 & ymm16)
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm14
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm14 = ymm6 ^ (ymm14 & (ymm13 ^ ymm6))
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm14, %xmm15
@@ -4209,7 +4209,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa (%rdi), %ymm4
; AVX512BW-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512BW-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],mem[2,3]
; AVX512BW-NEXT: vinserti128 $1, 96(%rdi), %ymm3, %ymm8
; AVX512BW-NEXT: movw $-28124, %r10w # imm = 0x9224
@@ -4225,7 +4225,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: movl $4192256, %r10d # imm = 0x3FF800
; AVX512BW-NEXT: kmovd %r10d, %k3
; AVX512BW-NEXT: vpshufb {{.*#+}} ymm5 {%k3} = ymm6[u,u,u,u,u,u,u,u,u,u,u,2,8,14,4,10,16,22,28,18,24,30,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %ymm3
; AVX512BW-NEXT: vpblendmw %ymm2, %ymm3, %ymm10 {%k1}
; AVX512BW-NEXT: vextracti128 $1, %ymm10, %xmm11
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[u,u,u,u,u,u],zero,zero,xmm11[0,6,12],zero,zero,zero,xmm11[4,10]
@@ -4317,7 +4317,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm5
; AVX512BW-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm1[2,3],mem[2,3]
; AVX512BW-FCP-NEXT: vinserti128 $1, 96(%rdi), %ymm1, %ymm7
; AVX512BW-FCP-NEXT: movw $-28124, %r10w # imm = 0x9224
@@ -4333,7 +4333,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: movl $4192256, %r10d # imm = 0x3FF800
; AVX512BW-FCP-NEXT: kmovd %r10d, %k3
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 {%k3} = ymm8[u,u,u,u,u,u,u,u,u,u,u,2,8,14,4,10,16,22,28,18,24,30,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm4
; AVX512BW-FCP-NEXT: vpblendmw %ymm3, %ymm4, %ymm10 {%k1}
; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm10, %xmm11
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[u,u,u,u,u,u],zero,zero,xmm11[0,6,12],zero,zero,zero,xmm11[4,10]
@@ -4422,7 +4422,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm4
; AVX512DQ-BW-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],mem[2,3]
; AVX512DQ-BW-NEXT: vinserti128 $1, 96(%rdi), %ymm3, %ymm8
; AVX512DQ-BW-NEXT: movw $-28124, %r10w # imm = 0x9224
@@ -4438,7 +4438,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: movl $4192256, %r10d # imm = 0x3FF800
; AVX512DQ-BW-NEXT: kmovd %r10d, %k3
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm5 {%k3} = ymm6[u,u,u,u,u,u,u,u,u,u,u,2,8,14,4,10,16,22,28,18,24,30,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %ymm3
; AVX512DQ-BW-NEXT: vpblendmw %ymm2, %ymm3, %ymm10 {%k1}
; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm10, %xmm11
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[u,u,u,u,u,u],zero,zero,xmm11[0,6,12],zero,zero,zero,xmm11[4,10]
@@ -4530,7 +4530,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm5
; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %ymm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm1[2,3],mem[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 96(%rdi), %ymm1, %ymm7
; AVX512DQ-BW-FCP-NEXT: movw $-28124, %r10w # imm = 0x9224
@@ -4546,7 +4546,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: movl $4192256, %r10d # imm = 0x3FF800
; AVX512DQ-BW-FCP-NEXT: kmovd %r10d, %k3
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 {%k3} = ymm8[u,u,u,u,u,u,u,u,u,u,u,2,8,14,4,10,16,22,28,18,24,30,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm4
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm3, %ymm4, %ymm10 {%k1}
; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm10, %xmm11
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[u,u,u,u,u,u],zero,zero,xmm11[0,6,12],zero,zero,zero,xmm11[4,10]
@@ -7450,7 +7450,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[3,9,15],zero,zero,xmm1[1,7,13]
; AVX512-NEXT: vpor %xmm1, %xmm15, %xmm0
; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm1
; AVX512-NEXT: vshufi64x2 {{.*#+}} ymm19 = ymm1[2,3],mem[2,3]
; AVX512-NEXT: vinserti32x4 $1, 288(%rdi), %ymm1, %ymm20
; AVX512-NEXT: vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
@@ -7704,7 +7704,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[3,9,15],zero,zero,xmm1[1,7,13]
; AVX512-FCP-NEXT: vpor %xmm1, %xmm15, %xmm0
; AVX512-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm1
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} ymm19 = ymm1[2,3],mem[2,3]
; AVX512-FCP-NEXT: vinserti32x4 $1, 288(%rdi), %ymm1, %ymm20
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
@@ -7958,7 +7958,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[3,9,15],zero,zero,xmm1[1,7,13]
; AVX512DQ-NEXT: vpor %xmm1, %xmm15, %xmm0
; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm1
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} ymm19 = ymm1[2,3],mem[2,3]
; AVX512DQ-NEXT: vinserti32x4 $1, 288(%rdi), %ymm1, %ymm20
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
@@ -8212,7 +8212,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u],zero,zero,zero,xmm1[3,9,15],zero,zero,xmm1[1,7,13]
; AVX512DQ-FCP-NEXT: vpor %xmm1, %xmm15, %xmm0
; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} ymm19 = ymm1[2,3],mem[2,3]
; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 288(%rdi), %ymm1, %ymm20
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0]
@@ -8366,7 +8366,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-LABEL: load_i8_stride6_vf64:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT: vmovdqa 224(%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa64 224(%rdi), %ymm0
; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm23
; AVX512BW-NEXT: movw $18724, %r10w # imm = 0x4924
; AVX512BW-NEXT: kmovd %r10d, %k1
@@ -8381,7 +8381,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512BW-NEXT: vmovdqa 64(%rdi), %ymm6
; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm26
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512BW-NEXT: vpblendmw %ymm26, %ymm1, %ymm15 {%k1}
; AVX512BW-NEXT: vextracti32x4 $1, %ymm15, %xmm16
; AVX512BW-NEXT: vmovdqa64 {{.*#+}} xmm17 = [u,u,u,u,u,u,128,128,0,6,12,128,128,128,4,10]
@@ -8406,13 +8406,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: kmovd %r10d, %k2
; AVX512BW-NEXT: vpshufb %ymm6, %ymm19, %ymm2 {%k2}
; AVX512BW-NEXT: vmovdqu16 %zmm11, %zmm2 {%k2}
-; AVX512BW-NEXT: vmovdqa 256(%rdi), %ymm11
-; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm11[2,3],mem[2,3]
-; AVX512BW-NEXT: vinserti128 $1, 288(%rdi), %ymm11, %ymm14
+; AVX512BW-NEXT: vmovdqa64 256(%rdi), %ymm11
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} ymm4 = ymm11[2,3],mem[2,3]
+; AVX512BW-NEXT: vinserti32x4 $1, 288(%rdi), %ymm11, %ymm14
; AVX512BW-NEXT: vpblendmw %ymm4, %ymm14, %ymm22 {%k4}
; AVX512BW-NEXT: vpshufb %ymm6, %ymm22, %ymm7
-; AVX512BW-NEXT: vmovdqa 320(%rdi), %ymm11
-; AVX512BW-NEXT: vmovdqa 352(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 320(%rdi), %ymm11
+; AVX512BW-NEXT: vmovdqa64 352(%rdi), %ymm6
; AVX512BW-NEXT: vpblendmw %ymm11, %ymm6, %ymm24 {%k1}
; AVX512BW-NEXT: vextracti32x4 $1, %ymm24, %xmm25
; AVX512BW-NEXT: vpshufb %xmm17, %xmm25, %xmm17
@@ -8602,8 +8602,8 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-LABEL: load_i8_stride6_vf64:
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 224(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm7
; AVX512BW-FCP-NEXT: movw $18724, %r10w # imm = 0x4924
; AVX512BW-FCP-NEXT: kmovd %r10d, %k1
; AVX512BW-FCP-NEXT: vpblendmw %ymm0, %ymm7, %ymm9 {%k1}
@@ -8616,8 +8616,8 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm10
; AVX512BW-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdi), %ymm6
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm8
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm8
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpblendmw %ymm8, %ymm1, %ymm15 {%k1}
; AVX512BW-FCP-NEXT: vextracti32x4 $1, %ymm15, %xmm16
; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm17 = [u,u,u,u,u,u,128,128,0,6,12,128,128,128,4,10]
@@ -8642,13 +8642,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: kmovd %r10d, %k2
; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm19, %ymm2 {%k2}
; AVX512BW-FCP-NEXT: vmovdqu16 %zmm11, %zmm2 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm11
-; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm11[2,3],mem[2,3]
-; AVX512BW-FCP-NEXT: vinserti128 $1, 288(%rdi), %ymm11, %ymm14
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} ymm4 = ymm11[2,3],mem[2,3]
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 288(%rdi), %ymm11, %ymm14
; AVX512BW-FCP-NEXT: vpblendmw %ymm4, %ymm14, %ymm22 {%k4}
; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm22, %ymm23
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rdi), %ymm11
-; AVX512BW-FCP-NEXT: vmovdqa 352(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 352(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpblendmw %ymm11, %ymm6, %ymm24 {%k1}
; AVX512BW-FCP-NEXT: vextracti32x4 $1, %ymm24, %xmm25
; AVX512BW-FCP-NEXT: vpshufb %xmm17, %xmm25, %xmm17
@@ -8835,7 +8835,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-LABEL: load_i8_stride6_vf64:
; AVX512DQ-BW: # %bb.0:
; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT: vmovdqa 224(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 224(%rdi), %ymm0
; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm23
; AVX512DQ-BW-NEXT: movw $18724, %r10w # imm = 0x4924
; AVX512DQ-BW-NEXT: kmovd %r10d, %k1
@@ -8850,7 +8850,7 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm26
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpblendmw %ymm26, %ymm1, %ymm15 {%k1}
; AVX512DQ-BW-NEXT: vextracti32x4 $1, %ymm15, %xmm16
; AVX512DQ-BW-NEXT: vmovdqa64 {{.*#+}} xmm17 = [u,u,u,u,u,u,128,128,0,6,12,128,128,128,4,10]
@@ -8875,13 +8875,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: kmovd %r10d, %k2
; AVX512DQ-BW-NEXT: vpshufb %ymm6, %ymm19, %ymm2 {%k2}
; AVX512DQ-BW-NEXT: vmovdqu16 %zmm11, %zmm2 {%k2}
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rdi), %ymm11
-; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm11[2,3],mem[2,3]
-; AVX512DQ-BW-NEXT: vinserti128 $1, 288(%rdi), %ymm11, %ymm14
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %ymm11
+; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} ymm4 = ymm11[2,3],mem[2,3]
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 288(%rdi), %ymm11, %ymm14
; AVX512DQ-BW-NEXT: vpblendmw %ymm4, %ymm14, %ymm22 {%k4}
; AVX512DQ-BW-NEXT: vpshufb %ymm6, %ymm22, %ymm7
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rdi), %ymm11
-; AVX512DQ-BW-NEXT: vmovdqa 352(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %ymm11
+; AVX512DQ-BW-NEXT: vmovdqa64 352(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpblendmw %ymm11, %ymm6, %ymm24 {%k1}
; AVX512DQ-BW-NEXT: vextracti32x4 $1, %ymm24, %xmm25
; AVX512DQ-BW-NEXT: vpshufb %xmm17, %xmm25, %xmm17
@@ -9071,8 +9071,8 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-LABEL: load_i8_stride6_vf64:
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 224(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm7
; AVX512DQ-BW-FCP-NEXT: movw $18724, %r10w # imm = 0x4924
; AVX512DQ-BW-FCP-NEXT: kmovd %r10d, %k1
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm0, %ymm7, %ymm9 {%k1}
@@ -9085,8 +9085,8 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm10
; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdi), %ymm3
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdi), %ymm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm8
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm8
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm8, %ymm1, %ymm15 {%k1}
; AVX512DQ-BW-FCP-NEXT: vextracti32x4 $1, %ymm15, %xmm16
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm17 = [u,u,u,u,u,u,128,128,0,6,12,128,128,128,4,10]
@@ -9111,13 +9111,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: kmovd %r10d, %k2
; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm19, %ymm2 {%k2}
; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm11, %zmm2 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm11
-; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm11[2,3],mem[2,3]
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 288(%rdi), %ymm11, %ymm14
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} ymm4 = ymm11[2,3],mem[2,3]
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 288(%rdi), %ymm11, %ymm14
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm4, %ymm14, %ymm22 {%k4}
; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm22, %ymm23
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rdi), %ymm11
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 352(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 352(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm11, %ymm6, %ymm24 {%k1}
; AVX512DQ-BW-FCP-NEXT: vextracti32x4 $1, %ymm24, %xmm25
; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm17, %xmm25, %xmm17
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
index 319735da0a6c4..3ebaf765d6882 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
@@ -5615,18 +5615,18 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm17
-; AVX512-NEXT: vmovdqa 160(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 160(%rdi), %ymm2
; AVX512-NEXT: vmovdqa %ymm0, %ymm1
; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm17 ^ (ymm1 & (ymm2 ^ ymm17))
; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u]
; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,5,12],zero,zero,xmm1[1,8,15],zero,zero,xmm1[u,u,u,u]
; AVX512-NEXT: vpor %xmm3, %xmm1, %xmm12
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm4
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm4
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,4,11,0,0,4,11,0,0,4,11,0,0,4,11]
; AVX512-NEXT: vpshufb %xmm3, %xmm4, %xmm1
; AVX512-NEXT: vmovdqa64 %xmm3, %xmm20
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm5
+; AVX512-NEXT: vmovdqa64 208(%rdi), %xmm5
; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm5[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [0,1,2,11,0,1,2,11]
@@ -5815,8 +5815,8 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 160(%rdi), %ymm3
; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm1
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm2 ^ (ymm1 & (ymm3 ^ ymm2))
; AVX512-FCP-NEXT: vextracti128 $1, %ymm1, %xmm4
@@ -5824,7 +5824,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,5,12],zero,zero,xmm1[1,8,15],zero,zero,xmm1[u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm4, %xmm1, %xmm6
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,0,0,0,1,2,4,6]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm12
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm12
; AVX512-FCP-NEXT: vpermd %ymm12, %ymm1, %ymm1
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm13 = [0,1,2,15,0,1,2,15]
@@ -5905,9 +5905,9 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,1,8,15],zero,zero,xmm6[4,11],zero,zero,xmm6[u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm6, %xmm12, %xmm6
; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-FCP-NEXT: vmovdqa 208(%rdi), %xmm14
+; AVX512-FCP-NEXT: vmovdqa64 208(%rdi), %xmm14
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm14[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm14[5,12]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %xmm13
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm13
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm13[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
; AVX512-FCP-NEXT: vpor %xmm12, %xmm15, %xmm12
; AVX512-FCP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
@@ -6016,18 +6016,18 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm17
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %ymm2
; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm1
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm17 ^ (ymm1 & (ymm2 ^ ymm17))
; AVX512DQ-NEXT: vextracti128 $1, %ymm1, %xmm3
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,5,12],zero,zero,xmm1[1,8,15],zero,zero,xmm1[u,u,u,u]
; AVX512DQ-NEXT: vpor %xmm3, %xmm1, %xmm12
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm4
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm4
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm3 = [0,0,4,11,0,0,4,11,0,0,4,11,0,0,4,11]
; AVX512DQ-NEXT: vpshufb %xmm3, %xmm4, %xmm1
; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm20
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 208(%rdi), %xmm5
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm5[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [0,1,2,11,0,1,2,11]
@@ -6216,8 +6216,8 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 160(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm1
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm2 ^ (ymm1 & (ymm3 ^ ymm2))
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm1, %xmm4
@@ -6225,7 +6225,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,5,12],zero,zero,xmm1[1,8,15],zero,zero,xmm1[u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm4, %xmm1, %xmm6
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,0,0,0,1,2,4,6]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm12
; AVX512DQ-FCP-NEXT: vpermd %ymm12, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm13 = [0,1,2,15,0,1,2,15]
@@ -6306,9 +6306,9 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,1,8,15],zero,zero,xmm6[4,11],zero,zero,xmm6[u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm6, %xmm12, %xmm6
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa 208(%rdi), %xmm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 208(%rdi), %xmm14
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm14[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm14[5,12]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %xmm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm13
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm13[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
; AVX512DQ-FCP-NEXT: vpor %xmm12, %xmm15, %xmm12
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm12
@@ -6430,8 +6430,8 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpermw %zmm2, %zmm3, %zmm11
; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [16,17,10,3,20,13,6,23,24,25,0,27,28,0,30,31]
; AVX512BW-NEXT: vpermw %zmm2, %zmm3, %zmm15
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm3
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %ymm2
; AVX512BW-NEXT: movw $8772, %r11w # imm = 0x2244
; AVX512BW-NEXT: kmovd %r11d, %k1
; AVX512BW-NEXT: vpblendmw %ymm3, %ymm2, %ymm4 {%k1}
@@ -6439,10 +6439,10 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[u,u,u],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13,u,u,u,u]
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u,u,u]
; AVX512BW-NEXT: vporq %xmm5, %xmm4, %xmm16
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm4
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %xmm4
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,11,4,11,4,11,4,11,4,11,4,11,4,11,4,11]
; AVX512BW-NEXT: vpshufb %xmm13, %xmm4, %xmm5
-; AVX512BW-NEXT: vmovdqa 208(%rdi), %xmm7
+; AVX512BW-NEXT: vmovdqa64 208(%rdi), %xmm7
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm6 = xmm7[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm14 = [0,1,2,11,0,1,2,11]
@@ -6613,8 +6613,8 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpermw %zmm2, %zmm3, %zmm8
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [16,17,10,3,20,13,6,23,24,25,18,27,28,21,30,31]
; AVX512BW-FCP-NEXT: vpermw %zmm2, %zmm3, %zmm13
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm3
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm3
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm2
; AVX512BW-FCP-NEXT: movw $8772, %r11w # imm = 0x2244
; AVX512BW-FCP-NEXT: kmovd %r11d, %k1
; AVX512BW-FCP-NEXT: vpblendmw %ymm3, %ymm2, %ymm4 {%k1}
@@ -6623,7 +6623,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u,u,u]
; AVX512BW-FCP-NEXT: vpor %xmm5, %xmm4, %xmm14
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,2,4,6]
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm11
; AVX512BW-FCP-NEXT: vpermd %ymm11, %ymm4, %ymm4
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
; AVX512BW-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm12 = [0,1,2,15,0,1,2,15]
@@ -6693,9 +6693,9 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[u,u,1,8,15],zero,zero,xmm11[4,11],zero,zero,xmm11[u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpor %xmm12, %xmm11, %xmm11
; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm13
-; AVX512BW-FCP-NEXT: vmovdqa 208(%rdi), %xmm11
+; AVX512BW-FCP-NEXT: vmovdqa64 208(%rdi), %xmm11
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm14 = xmm11[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm11[5,12]
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm12
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm12
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
; AVX512BW-FCP-NEXT: vpor %xmm14, %xmm15, %xmm14
; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
@@ -6796,8 +6796,8 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpermw %zmm2, %zmm3, %zmm11
; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [16,17,10,3,20,13,6,23,24,25,0,27,28,0,30,31]
; AVX512DQ-BW-NEXT: vpermw %zmm2, %zmm3, %zmm15
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm3
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %ymm2
; AVX512DQ-BW-NEXT: movw $8772, %r11w # imm = 0x2244
; AVX512DQ-BW-NEXT: kmovd %r11d, %k1
; AVX512DQ-BW-NEXT: vpblendmw %ymm3, %ymm2, %ymm4 {%k1}
@@ -6805,10 +6805,10 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[u,u,u],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13,u,u,u,u]
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u,u,u]
; AVX512DQ-BW-NEXT: vporq %xmm5, %xmm4, %xmm16
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %xmm4
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm13 = [4,11,4,11,4,11,4,11,4,11,4,11,4,11,4,11]
; AVX512DQ-BW-NEXT: vpshufb %xmm13, %xmm4, %xmm5
-; AVX512DQ-BW-NEXT: vmovdqa 208(%rdi), %xmm7
+; AVX512DQ-BW-NEXT: vmovdqa64 208(%rdi), %xmm7
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm6 = xmm7[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
; AVX512DQ-BW-NEXT: vbroadcasti128 {{.*#+}} ymm14 = [0,1,2,11,0,1,2,11]
@@ -6979,8 +6979,8 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpermw %zmm2, %zmm3, %zmm8
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [16,17,10,3,20,13,6,23,24,25,18,27,28,21,30,31]
; AVX512DQ-BW-FCP-NEXT: vpermw %zmm2, %zmm3, %zmm13
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm2
; AVX512DQ-BW-FCP-NEXT: movw $8772, %r11w # imm = 0x2244
; AVX512DQ-BW-FCP-NEXT: kmovd %r11d, %k1
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm3, %ymm2, %ymm4 {%k1}
@@ -6989,7 +6989,7 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpor %xmm5, %xmm4, %xmm14
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,2,4,6]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm11
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm11, %ymm4, %ymm4
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
; AVX512DQ-BW-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm12 = [0,1,2,15,0,1,2,15]
@@ -7059,9 +7059,9 @@ define void @load_i8_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[u,u,1,8,15],zero,zero,xmm11[4,11],zero,zero,xmm11[u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpor %xmm12, %xmm11, %xmm11
; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm13
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 208(%rdi), %xmm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 208(%rdi), %xmm11
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm14 = xmm11[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm11[5,12]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm12
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm12
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
; AVX512DQ-BW-FCP-NEXT: vpor %xmm14, %xmm15, %xmm14
; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm14
@@ -11692,20 +11692,20 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u]
; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,5,12],zero,zero,xmm2[1,8,15],zero,zero,xmm2[u,u,u,u]
; AVX512-NEXT: vpor %xmm3, %xmm2, %xmm2
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm5
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm5
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,4,11,0,0,4,11,0,0,4,11,0,0,4,11]
; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm4
; AVX512-NEXT: vmovdqa64 %xmm0, %xmm28
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm3
+; AVX512-NEXT: vmovdqa64 208(%rdi), %xmm3
; AVX512-NEXT: vpshufb {{.*#+}} xmm6 = xmm3[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vmovdqa64 %xmm3, %xmm25
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
; AVX512-NEXT: vbroadcasti32x4 {{.*#+}} ymm20 = [0,1,2,11,0,1,2,11]
; AVX512-NEXT: # ymm20 = mem[0,1,2,3,0,1,2,3]
; AVX512-NEXT: vpermt2d %ymm4, %ymm20, %ymm2
-; AVX512-NEXT: vmovdqa 240(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 240(%rdi), %xmm0
; AVX512-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm0[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm3
+; AVX512-NEXT: vmovdqa64 224(%rdi), %xmm3
; AVX512-NEXT: vpshufb {{.*#+}} xmm7 = xmm3[0,7,14],zero,zero,xmm3[u,u,u,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vmovdqa64 %xmm3, %xmm24
; AVX512-NEXT: vpor %xmm6, %xmm7, %xmm6
@@ -11720,7 +11720,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u],zero,zero,xmm6[1,8,15],zero,zero,xmm6[4,11,u,u]
; AVX512-NEXT: vpor %xmm7, %xmm6, %xmm7
-; AVX512-NEXT: vmovdqa 352(%rdi), %ymm13
+; AVX512-NEXT: vmovdqa64 352(%rdi), %ymm13
; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm16
; AVX512-NEXT: vmovdqa %ymm14, %ymm8
; AVX512-NEXT: vpternlogq {{.*#+}} ymm8 = ymm13 ^ (ymm8 & (ymm16 ^ ymm13))
@@ -12111,30 +12111,30 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm2, %xmm0, %xmm0
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,2,4,6]
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm3
; AVX512-FCP-NEXT: vpermd %ymm3, %ymm2, %ymm2
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [0,1,2,15,0,1,2,15]
; AVX512-FCP-NEXT: # ymm15 = mem[0,1,0,1]
; AVX512-FCP-NEXT: vpermt2d %ymm2, %ymm15, %ymm0
-; AVX512-FCP-NEXT: vmovdqa 240(%rdi), %xmm4
+; AVX512-FCP-NEXT: vmovdqa64 240(%rdi), %xmm4
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm4[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %xmm5
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm5[0,7,14],zero,zero,xmm5[u,u,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm2, %xmm7, %xmm2
; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm8
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm22 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm22 & (zmm8 ^ zmm1))
; AVX512-FCP-NEXT: vmovdqa64 288(%rdi), %ymm16
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm11
; AVX512-FCP-NEXT: vmovdqa %ymm10, %ymm0
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm16 ^ (ymm0 & (ymm11 ^ ymm16))
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[u,u,u,u,u,3,10],zero,zero,zero,xmm0[6,13],zero,zero,xmm0[u,u]
; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u],zero,zero,xmm0[1,8,15],zero,zero,xmm0[4,11,u,u]
; AVX512-FCP-NEXT: vpor %xmm2, %xmm0, %xmm0
-; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %ymm14
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %ymm14
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm2
; AVX512-FCP-NEXT: vmovdqa %ymm9, %ymm7
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm14 ^ (ymm7 & (ymm2 ^ ymm14))
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm13 = ymm7[2,3,0,1]
@@ -12275,10 +12275,10 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,1,8,15],zero,zero,xmm3[4,11],zero,zero,xmm3[u,u,u,u,u]
; AVX512-FCP-NEXT: vpor %xmm7, %xmm3, %xmm7
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT: vmovdqa 208(%rdi), %xmm3
+; AVX512-FCP-NEXT: vmovdqa64 208(%rdi), %xmm3
; AVX512-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm13
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %xmm15
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm15
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm15[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
; AVX512-FCP-NEXT: vpor %xmm3, %xmm13, %xmm3
; AVX512-FCP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
@@ -12429,7 +12429,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 ^ (ymm12 & (ymm8 ^ ymm3))
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,2,4,6,0,0,0,0]
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm9
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm9
; AVX512-FCP-NEXT: vpermd %ymm9, %ymm3, %ymm3
; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,7,10,13,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm3, %zmm4, %zmm3
@@ -12509,20 +12509,20 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u]
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,5,12],zero,zero,xmm2[1,8,15],zero,zero,xmm2[u,u,u,u]
; AVX512DQ-NEXT: vpor %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm5
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,4,11,0,0,4,11,0,0,4,11,0,0,4,11]
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm4
; AVX512DQ-NEXT: vmovdqa64 %xmm0, %xmm30
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm3
+; AVX512DQ-NEXT: vmovdqa64 208(%rdi), %xmm3
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm6 = xmm3[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm24
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
; AVX512DQ-NEXT: vbroadcasti32x4 {{.*#+}} ymm20 = [0,1,2,11,0,1,2,11]
; AVX512DQ-NEXT: # ymm20 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-NEXT: vpermt2d %ymm4, %ymm20, %ymm2
-; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 240(%rdi), %xmm0
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm0[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm3
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %xmm3
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm7 = xmm3[0,7,14],zero,zero,xmm3[u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm23
; AVX512DQ-NEXT: vpor %xmm6, %xmm7, %xmm6
@@ -12537,7 +12537,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u],zero,zero,xmm6[1,8,15],zero,zero,xmm6[4,11,u,u]
; AVX512DQ-NEXT: vpor %xmm7, %xmm6, %xmm7
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %ymm13
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %ymm13
; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm16
; AVX512DQ-NEXT: vmovdqa %ymm14, %ymm8
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm8 = ymm13 ^ (ymm8 & (ymm16 ^ ymm13))
@@ -12930,30 +12930,30 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm2, %xmm0, %xmm0
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,2,4,6]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm2, %ymm2
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [0,1,2,15,0,1,2,15]
; AVX512DQ-FCP-NEXT: # ymm15 = mem[0,1,0,1]
; AVX512DQ-FCP-NEXT: vpermt2d %ymm2, %ymm15, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 240(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 240(%rdi), %xmm4
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm4[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %xmm5
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm5[0,7,14],zero,zero,xmm5[u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm2, %xmm7, %xmm2
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm8
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm22 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm22 & (zmm8 ^ zmm1))
; AVX512DQ-FCP-NEXT: vmovdqa64 288(%rdi), %ymm16
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vmovdqa %ymm10, %ymm0
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm16 ^ (ymm0 & (ymm11 ^ ymm16))
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[u,u,u,u,u,3,10],zero,zero,zero,xmm0[6,13],zero,zero,xmm0[u,u]
; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u],zero,zero,xmm0[1,8,15],zero,zero,xmm0[4,11,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm2, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %ymm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa %ymm9, %ymm7
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm7 = ymm14 ^ (ymm7 & (ymm2 ^ ymm14))
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm13 = ymm7[2,3,0,1]
@@ -13094,10 +13094,10 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,1,8,15],zero,zero,xmm3[4,11],zero,zero,xmm3[u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpor %xmm7, %xmm3, %xmm7
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vmovdqa 208(%rdi), %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 208(%rdi), %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %xmm8, %xmm3, %xmm13
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm15
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm15[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
; AVX512DQ-FCP-NEXT: vpor %xmm3, %xmm13, %xmm3
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm7
@@ -13248,7 +13248,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm8 ^ (ymm12 & (ymm8 ^ ymm3))
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,2,4,6,0,0,0,0]
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm9
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm9
; AVX512DQ-FCP-NEXT: vpermd %ymm9, %ymm3, %ymm3
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,7,10,13,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm3, %zmm4, %zmm3
@@ -13323,8 +13323,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: movw $992, %ax # imm = 0x3E0
; AVX512BW-NEXT: kmovd %eax, %k1
; AVX512BW-NEXT: vmovdqu16 %ymm3, %ymm26 {%k1}
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm13
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %ymm12
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm13
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %ymm12
; AVX512BW-NEXT: movw $8772, %ax # imm = 0x2244
; AVX512BW-NEXT: kmovd %eax, %k6
; AVX512BW-NEXT: vpblendmw %ymm13, %ymm12, %ymm3 {%k6}
@@ -13332,18 +13332,18 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,u,u],zero,zero,xmm4[3,10],zero,zero,zero,xmm4[6,13,u,u,u,u]
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,5,12],zero,zero,xmm3[1,8,15],zero,zero,xmm3[u,u,u,u]
; AVX512BW-NEXT: vpor %xmm4, %xmm3, %xmm3
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm14
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %xmm14
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm21 = [4,11,4,11,4,11,4,11,4,11,4,11,4,11,4,11]
; AVX512BW-NEXT: vpshufb %xmm21, %xmm14, %xmm4
-; AVX512BW-NEXT: vmovdqa 208(%rdi), %xmm15
+; AVX512BW-NEXT: vmovdqa64 208(%rdi), %xmm15
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm5 = xmm15[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} ymm22 = [0,1,2,11,0,1,2,11]
; AVX512BW-NEXT: # ymm22 = mem[0,1,2,3,0,1,2,3]
; AVX512BW-NEXT: vpermt2d %ymm4, %ymm22, %ymm3
-; AVX512BW-NEXT: vmovdqa 240(%rdi), %xmm5
+; AVX512BW-NEXT: vmovdqa64 240(%rdi), %xmm5
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm5[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vmovdqa 224(%rdi), %xmm7
+; AVX512BW-NEXT: vmovdqa64 224(%rdi), %xmm7
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm6 = xmm7[0,7,14],zero,zero,xmm7[u,u,u,u,u,u,u,u,u,u,u]
; AVX512BW-NEXT: vpor %xmm4, %xmm6, %xmm4
; AVX512BW-NEXT: vinserti32x4 $2, %xmm4, %zmm3, %zmm3
@@ -13351,7 +13351,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: kmovq %rax, %k5
; AVX512BW-NEXT: vmovdqu8 %zmm3, %zmm26 {%k5}
; AVX512BW-NEXT: vmovdqa64 288(%rdi), %ymm28
-; AVX512BW-NEXT: vmovdqa 256(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqa64 256(%rdi), %ymm3
; AVX512BW-NEXT: movw $9288, %ax # imm = 0x2448
; AVX512BW-NEXT: kmovd %eax, %k4
; AVX512BW-NEXT: vpblendmw %ymm28, %ymm3, %ymm6 {%k4}
@@ -13359,8 +13359,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vextracti128 $1, %ymm6, %xmm6
; AVX512BW-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u],zero,zero,xmm6[1,8,15],zero,zero,xmm6[4,11,u,u]
; AVX512BW-NEXT: vporq %xmm8, %xmm6, %xmm19
-; AVX512BW-NEXT: vmovdqa 352(%rdi), %ymm10
-; AVX512BW-NEXT: vmovdqa 320(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 352(%rdi), %ymm10
+; AVX512BW-NEXT: vmovdqa64 320(%rdi), %ymm9
; AVX512BW-NEXT: vpblendmw %ymm10, %ymm9, %ymm6 {%k6}
; AVX512BW-NEXT: vpermq {{.*#+}} ymm8 = ymm6[2,3,0,1]
; AVX512BW-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm8[2],ymm6[3,4,5],ymm8[6],ymm6[7,8,9],ymm8[10],ymm6[11,12,13],ymm8[14],ymm6[15]
@@ -13368,8 +13368,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: movw $3968, %ax # imm = 0xF80
; AVX512BW-NEXT: kmovd %eax, %k7
; AVX512BW-NEXT: vmovdqu16 %ymm6, %ymm19 {%k7}
-; AVX512BW-NEXT: vmovdqa 416(%rdi), %ymm8
-; AVX512BW-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 416(%rdi), %ymm8
+; AVX512BW-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512BW-NEXT: movw $4644, %ax # imm = 0x1224
; AVX512BW-NEXT: kmovd %eax, %k3
; AVX512BW-NEXT: vpblendmw %ymm8, %ymm6, %ymm20 {%k3}
@@ -13697,8 +13697,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: movw $992, %ax # imm = 0x3E0
; AVX512BW-FCP-NEXT: kmovd %eax, %k1
; AVX512BW-FCP-NEXT: vmovdqu16 %ymm4, %ymm1 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm11
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm9
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm9
; AVX512BW-FCP-NEXT: movw $8772, %ax # imm = 0x2244
; AVX512BW-FCP-NEXT: kmovd %eax, %k1
; AVX512BW-FCP-NEXT: vpblendmw %ymm11, %ymm9, %ymm4 {%k1}
@@ -13724,8 +13724,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: movabsq $137438429184, %rax # imm = 0x1FFFF80000
; AVX512BW-FCP-NEXT: kmovq %rax, %k5
; AVX512BW-FCP-NEXT: vmovdqu8 %zmm4, %zmm1 {%k5}
-; AVX512BW-FCP-NEXT: vmovdqa 288(%rdi), %ymm6
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 288(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm4
; AVX512BW-FCP-NEXT: movw $9288, %ax # imm = 0x2448
; AVX512BW-FCP-NEXT: kmovd %eax, %k6
; AVX512BW-FCP-NEXT: vpblendmw %ymm6, %ymm4, %ymm7 {%k6}
@@ -13737,8 +13737,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: movw $3968, %ax # imm = 0xF80
; AVX512BW-FCP-NEXT: kmovd %eax, %k7
; AVX512BW-FCP-NEXT: vmovdqu16 %ymm5, %ymm21 {%k7}
-; AVX512BW-FCP-NEXT: vmovdqa 416(%rdi), %ymm7
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rdi), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 416(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm5
; AVX512BW-FCP-NEXT: movw $4644, %ax # imm = 0x1224
; AVX512BW-FCP-NEXT: kmovd %eax, %k4
; AVX512BW-FCP-NEXT: vpblendmw %ymm7, %ymm5, %ymm22 {%k4}
@@ -14044,8 +14044,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: movw $992, %ax # imm = 0x3E0
; AVX512DQ-BW-NEXT: kmovd %eax, %k1
; AVX512DQ-BW-NEXT: vmovdqu16 %ymm0, %ymm16 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm11
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm11
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %ymm5
; AVX512DQ-BW-NEXT: movw $8772, %ax # imm = 0x2244
; AVX512DQ-BW-NEXT: kmovd %eax, %k6
; AVX512DQ-BW-NEXT: vpblendmw %ymm11, %ymm5, %ymm0 {%k6}
@@ -14053,10 +14053,10 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u]
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u,u,u]
; AVX512DQ-BW-NEXT: vpor %xmm3, %xmm0, %xmm0
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm21 = [4,11,4,11,4,11,4,11,4,11,4,11,4,11,4,11]
; AVX512DQ-BW-NEXT: vpshufb %xmm21, %xmm7, %xmm3
-; AVX512DQ-BW-NEXT: vmovdqa 208(%rdi), %xmm8
+; AVX512DQ-BW-NEXT: vmovdqa64 208(%rdi), %xmm8
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm4 = xmm8[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
; AVX512DQ-BW-NEXT: vbroadcasti32x4 {{.*#+}} ymm22 = [0,1,2,11,0,1,2,11]
@@ -14064,15 +14064,15 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpermt2d %ymm3, %ymm22, %ymm0
; AVX512DQ-BW-NEXT: vmovdqa64 240(%rdi), %xmm27
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm27[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-NEXT: vmovdqa 224(%rdi), %xmm4
+; AVX512DQ-BW-NEXT: vmovdqa64 224(%rdi), %xmm4
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm12 = xmm4[0,7,14],zero,zero,xmm4[u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-NEXT: vpor %xmm6, %xmm12, %xmm6
; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm6, %zmm0, %zmm0
; AVX512DQ-BW-NEXT: movabsq $137438429184, %rax # imm = 0x1FFFF80000
; AVX512DQ-BW-NEXT: kmovq %rax, %k5
; AVX512DQ-BW-NEXT: vmovdqu8 %zmm0, %zmm16 {%k5}
-; AVX512DQ-BW-NEXT: vmovdqa 288(%rdi), %ymm13
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rdi), %ymm12
+; AVX512DQ-BW-NEXT: vmovdqa64 288(%rdi), %ymm13
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %ymm12
; AVX512DQ-BW-NEXT: movw $9288, %ax # imm = 0x2448
; AVX512DQ-BW-NEXT: kmovd %eax, %k3
; AVX512DQ-BW-NEXT: vpblendmw %ymm13, %ymm12, %ymm0 {%k3}
@@ -14081,7 +14081,7 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u],zero,zero,xmm0[1,8,15],zero,zero,xmm0[4,11,u,u]
; AVX512DQ-BW-NEXT: vporq %xmm6, %xmm0, %xmm19
; AVX512DQ-BW-NEXT: vmovdqa64 352(%rdi), %ymm17
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %ymm0
; AVX512DQ-BW-NEXT: vpblendmw %ymm17, %ymm0, %ymm6 {%k6}
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm15 = ymm6[2,3,0,1]
; AVX512DQ-BW-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm15[2],ymm6[3,4,5],ymm15[6],ymm6[7,8,9],ymm15[10],ymm6[11,12,13],ymm15[14],ymm6[15]
@@ -14089,8 +14089,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: movw $3968, %ax # imm = 0xF80
; AVX512DQ-BW-NEXT: kmovd %eax, %k7
; AVX512DQ-BW-NEXT: vmovdqu16 %ymm6, %ymm19 {%k7}
-; AVX512DQ-BW-NEXT: vmovdqa 416(%rdi), %ymm15
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 416(%rdi), %ymm15
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512DQ-BW-NEXT: movw $4644, %ax # imm = 0x1224
; AVX512DQ-BW-NEXT: kmovd %eax, %k4
; AVX512DQ-BW-NEXT: vpblendmw %ymm15, %ymm6, %ymm20 {%k4}
@@ -14415,8 +14415,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: movw $992, %ax # imm = 0x3E0
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %ymm4, %ymm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm11
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm9
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm9
; AVX512DQ-BW-FCP-NEXT: movw $8772, %ax # imm = 0x2244
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm11, %ymm9, %ymm4 {%k1}
@@ -14442,8 +14442,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: movabsq $137438429184, %rax # imm = 0x1FFFF80000
; AVX512DQ-BW-FCP-NEXT: kmovq %rax, %k5
; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm4, %zmm1 {%k5}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 288(%rdi), %ymm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 288(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm4
; AVX512DQ-BW-FCP-NEXT: movw $9288, %ax # imm = 0x2448
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k6
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm6, %ymm4, %ymm7 {%k6}
@@ -14455,8 +14455,8 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: movw $3968, %ax # imm = 0xF80
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k7
; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %ymm5, %ymm21 {%k7}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 416(%rdi), %ymm7
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 416(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm5
; AVX512DQ-BW-FCP-NEXT: movw $4644, %ax # imm = 0x1224
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k4
; AVX512DQ-BW-FCP-NEXT: vpblendmw %ymm7, %ymm5, %ymm22 {%k4}
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
index 6c0970f9d67de..24ecb4f1536a8 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
@@ -5970,21 +5970,21 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm18
; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm1
; AVX512-NEXT: vpmovqb %zmm1, %xmm2
-; AVX512-NEXT: vmovdqa 240(%rdi), %xmm1
+; AVX512-NEXT: vmovdqa64 240(%rdi), %xmm1
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
; AVX512-NEXT: vpshufb %xmm8, %xmm1, %xmm4
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm3
+; AVX512-NEXT: vmovdqa64 224(%rdi), %xmm3
; AVX512-NEXT: vpshufb %xmm8, %xmm3, %xmm5
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm4
+; AVX512-NEXT: vmovdqa64 208(%rdi), %xmm4
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
; AVX512-NEXT: vpshufb %xmm11, %xmm4, %xmm6
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512-NEXT: vpshufb %xmm11, %xmm7, %xmm9
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm17 = [2,11,2,11,2,11,2,11]
; AVX512-NEXT: vpermt2d %ymm5, %ymm17, %ymm6
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm5
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm5
; AVX512-NEXT: vpmovqb %ymm5, %xmm9
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm16 = [0,9,0,9,0,9,0,9]
; AVX512-NEXT: vpermt2d %ymm2, %ymm16, %ymm9
@@ -6004,8 +6004,8 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpblendd {{.*#+}} xmm8 = xmm11[0,1],xmm8[2,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm2[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm19
-; AVX512-NEXT: vmovdqa 160(%rdi), %xmm13
-; AVX512-NEXT: vmovdqa 176(%rdi), %xmm14
+; AVX512-NEXT: vmovdqa64 160(%rdi), %xmm13
+; AVX512-NEXT: vmovdqa64 176(%rdi), %xmm14
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
; AVX512-NEXT: vpshufb %xmm2, %xmm1, %xmm8
; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm11
@@ -6239,7 +6239,7 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vpermd %ymm18, %ymm0, %ymm3
; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm3, %ymm6
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm15 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm11
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm11
; AVX512-FCP-NEXT: vpermd %ymm11, %ymm0, %ymm4
; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm4, %ymm0
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm6[5],ymm0[6,7]
@@ -6411,21 +6411,21 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm18
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm1
; AVX512DQ-NEXT: vpmovqb %zmm1, %xmm2
-; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm1
+; AVX512DQ-NEXT: vmovdqa64 240(%rdi), %xmm1
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm8 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
; AVX512DQ-NEXT: vpshufb %xmm8, %xmm1, %xmm4
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm3
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %xmm3
; AVX512DQ-NEXT: vpshufb %xmm8, %xmm3, %xmm5
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm4
+; AVX512DQ-NEXT: vmovdqa64 208(%rdi), %xmm4
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm11 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
; AVX512DQ-NEXT: vpshufb %xmm11, %xmm4, %xmm6
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512DQ-NEXT: vpshufb %xmm11, %xmm7, %xmm9
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm17 = [2,11,2,11,2,11,2,11]
; AVX512DQ-NEXT: vpermt2d %ymm5, %ymm17, %ymm6
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm5
; AVX512DQ-NEXT: vpmovqb %ymm5, %xmm9
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm16 = [0,9,0,9,0,9,0,9]
; AVX512DQ-NEXT: vpermt2d %ymm2, %ymm16, %ymm9
@@ -6445,8 +6445,8 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm8 = xmm11[0,1],xmm8[2,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm19
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm13
-; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm14
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %xmm13
+; AVX512DQ-NEXT: vmovdqa64 176(%rdi), %xmm14
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
; AVX512DQ-NEXT: vpshufb %xmm2, %xmm1, %xmm8
; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm11
@@ -6680,7 +6680,7 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vpermd %ymm18, %ymm0, %ymm3
; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm3, %ymm6
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm15 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm11
; AVX512DQ-FCP-NEXT: vpermd %ymm11, %ymm0, %ymm4
; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm4, %ymm0
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm6[5],ymm0[6,7]
@@ -6856,7 +6856,7 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm1
; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm2
; AVX512BW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512BW-NEXT: vpmovqb %ymm3, %xmm5
; AVX512BW-NEXT: vpbroadcastq {{.*#+}} ymm16 = [0,9,0,9,0,9,0,9]
; AVX512BW-NEXT: vpermt2d %ymm2, %ymm16, %ymm5
@@ -6882,8 +6882,8 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm8[0,1],xmm1[2,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512BW-NEXT: vmovdqa64 %ymm1, %ymm19
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm10
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm11
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm10
+; AVX512BW-NEXT: vmovdqa64 176(%rdi), %xmm11
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
; AVX512BW-NEXT: vpshufb %xmm1, %xmm11, %xmm2
; AVX512BW-NEXT: vpshufb %xmm1, %xmm10, %xmm1
@@ -7061,17 +7061,17 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm4
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
-; AVX512BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 224(%rdi), %ymm5
; AVX512BW-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm7
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28]
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm8
; AVX512BW-FCP-NEXT: vpermd %ymm8, %ymm0, %ymm30
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm30[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm11
+; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm11
; AVX512BW-FCP-NEXT: vpermd %ymm11, %ymm0, %ymm3
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512BW-FCP-NEXT: vpermd %ymm15, %ymm0, %ymm2
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4],ymm6[5],ymm9[6,7]
@@ -7234,7 +7234,7 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm2
; AVX512DQ-BW-NEXT: vpmovqb %zmm2, %xmm2
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512DQ-BW-NEXT: vpmovqb %ymm3, %xmm5
; AVX512DQ-BW-NEXT: vpbroadcastq {{.*#+}} ymm16 = [0,9,0,9,0,9,0,9]
; AVX512DQ-BW-NEXT: vpermt2d %ymm2, %ymm16, %ymm5
@@ -7260,8 +7260,8 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm8[0,1],xmm1[2,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
; AVX512DQ-BW-NEXT: vmovdqa64 %ymm1, %ymm19
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm10
-; AVX512DQ-BW-NEXT: vmovdqa 176(%rdi), %xmm11
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %xmm10
+; AVX512DQ-BW-NEXT: vmovdqa64 176(%rdi), %xmm11
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm1 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm11, %xmm2
; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm10, %xmm1
@@ -7439,17 +7439,17 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm4
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 224(%rdi), %ymm5
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm7
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm8
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm8
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm8, %ymm0, %ymm30
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm30[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 160(%rdi), %ymm11
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm11
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm11, %ymm0, %ymm3
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm15
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm15
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm15, %ymm0, %ymm2
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm9 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4],ymm6[5],ymm9[6,7]
@@ -12659,30 +12659,30 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpmovqb %zmm0, %xmm3
; AVX512-NEXT: vmovdqa64 256(%rdi), %zmm18
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512-NEXT: vmovdqa 368(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 368(%rdi), %xmm7
; AVX512-NEXT: vpshufb %xmm0, %xmm7, %xmm1
-; AVX512-NEXT: vmovdqa 352(%rdi), %xmm8
+; AVX512-NEXT: vmovdqa64 352(%rdi), %xmm8
; AVX512-NEXT: vpshufb %xmm0, %xmm8, %xmm2
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512-NEXT: vmovdqa 336(%rdi), %xmm13
+; AVX512-NEXT: vmovdqa64 336(%rdi), %xmm13
; AVX512-NEXT: vpshufb %xmm1, %xmm13, %xmm4
-; AVX512-NEXT: vmovdqa 320(%rdi), %xmm15
+; AVX512-NEXT: vmovdqa64 320(%rdi), %xmm15
; AVX512-NEXT: vpshufb %xmm1, %xmm15, %xmm5
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
; AVX512-NEXT: vpmovqb %zmm18, %xmm4
; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3]
-; AVX512-NEXT: vmovdqa 496(%rdi), %xmm5
+; AVX512-NEXT: vmovdqa64 496(%rdi), %xmm5
; AVX512-NEXT: vpshufb %xmm0, %xmm5, %xmm4
; AVX512-NEXT: vmovdqa64 %xmm5, %xmm20
-; AVX512-NEXT: vmovdqa 480(%rdi), %xmm11
+; AVX512-NEXT: vmovdqa64 480(%rdi), %xmm11
; AVX512-NEXT: vpshufb %xmm0, %xmm11, %xmm5
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-NEXT: vmovdqa 464(%rdi), %xmm6
+; AVX512-NEXT: vmovdqa64 464(%rdi), %xmm6
; AVX512-NEXT: vpshufb %xmm1, %xmm6, %xmm5
; AVX512-NEXT: vmovdqa64 %xmm6, %xmm22
-; AVX512-NEXT: vmovdqa 448(%rdi), %xmm9
+; AVX512-NEXT: vmovdqa64 448(%rdi), %xmm9
; AVX512-NEXT: vpshufb %xmm1, %xmm9, %xmm6
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm12 = [2,11,2,11,2,11,2,11]
@@ -12695,16 +12695,16 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vbroadcasti64x4 {{.*#+}} zmm28 = [0,1,10,11,0,1,10,11]
; AVX512-NEXT: # zmm28 = mem[0,1,2,3,0,1,2,3]
; AVX512-NEXT: vpermt2q %zmm3, %zmm28, %zmm2
-; AVX512-NEXT: vmovdqa 240(%rdi), %xmm3
+; AVX512-NEXT: vmovdqa64 240(%rdi), %xmm3
; AVX512-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpshufb %xmm0, %xmm3, %xmm3
-; AVX512-NEXT: vmovdqa 224(%rdi), %xmm10
+; AVX512-NEXT: vmovdqa64 224(%rdi), %xmm10
; AVX512-NEXT: vpshufb %xmm0, %xmm10, %xmm4
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT: vmovdqa 208(%rdi), %xmm5
+; AVX512-NEXT: vmovdqa64 208(%rdi), %xmm5
; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm4
; AVX512-NEXT: vmovdqa64 %xmm5, %xmm25
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm5
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm5
; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm5
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
@@ -12764,9 +12764,9 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vmovdqa64 %xmm9, %xmm23
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512-NEXT: vpermt2d %ymm3, %ymm12, %ymm4
-; AVX512-NEXT: vmovdqa 416(%rdi), %xmm0
+; AVX512-NEXT: vmovdqa64 416(%rdi), %xmm0
; AVX512-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT: vmovdqa 432(%rdi), %xmm11
+; AVX512-NEXT: vmovdqa64 432(%rdi), %xmm11
; AVX512-NEXT: vmovd {{.*#+}} xmm2 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
; AVX512-NEXT: vpshufb %xmm2, %xmm11, %xmm3
; AVX512-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -12792,8 +12792,8 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
; AVX512-NEXT: vpermt2d %ymm3, %ymm12, %ymm13
; AVX512-NEXT: vmovdqa64 %ymm12, %ymm17
-; AVX512-NEXT: vmovdqa 160(%rdi), %xmm3
-; AVX512-NEXT: vmovdqa 176(%rdi), %xmm9
+; AVX512-NEXT: vmovdqa64 160(%rdi), %xmm3
+; AVX512-NEXT: vmovdqa64 176(%rdi), %xmm9
; AVX512-NEXT: vpshufb %xmm2, %xmm9, %xmm0
; AVX512-NEXT: vpshufb %xmm2, %xmm3, %xmm2
; AVX512-NEXT: vmovdqa64 %xmm3, %xmm25
@@ -13290,37 +13290,37 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %zmm25
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
-; AVX512-FCP-NEXT: vmovdqa 480(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 480(%rdi), %ymm1
; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm3
; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 %ymm3, %ymm24
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm5
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm2
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512-FCP-NEXT: vmovdqa 416(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 416(%rdi), %ymm2
; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm4
; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm4, %ymm2
; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm23
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm12 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm3
; AVX512-FCP-NEXT: vmovdqu %ymm3, (%rsp) # 32-byte Spill
; AVX512-FCP-NEXT: vpermd %ymm3, %ymm0, %ymm6
; AVX512-FCP-NEXT: vpshufb %ymm12, %ymm6, %ymm3
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-FCP-NEXT: vmovdqa 352(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %ymm2
; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpmovqd %ymm2, %xmm4
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm2
; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm21
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm3
; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpmovqd %ymm3, %xmm9
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
@@ -13332,12 +13332,12 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm30 = [0,1,10,11,0,1,10,11]
; AVX512-FCP-NEXT: # zmm30 = mem[0,1,2,3,0,1,2,3]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm30, %zmm14
-; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm1
; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm2
; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm2, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 %ymm2, %ymm18
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm3
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm3, %ymm2
@@ -13727,30 +13727,30 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpmovqb %zmm0, %xmm3
; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %zmm18
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512DQ-NEXT: vmovdqa 368(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 368(%rdi), %xmm7
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm7, %xmm1
-; AVX512DQ-NEXT: vmovdqa 352(%rdi), %xmm8
+; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %xmm8
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm8, %xmm2
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512DQ-NEXT: vmovdqa 336(%rdi), %xmm13
+; AVX512DQ-NEXT: vmovdqa64 336(%rdi), %xmm13
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm13, %xmm4
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %xmm15
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %xmm15
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm15, %xmm5
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm4[0,1,2],xmm2[3]
; AVX512DQ-NEXT: vpmovqb %zmm18, %xmm4
; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3]
-; AVX512DQ-NEXT: vmovdqa 496(%rdi), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 496(%rdi), %xmm5
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm5, %xmm4
; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm20
-; AVX512DQ-NEXT: vmovdqa 480(%rdi), %xmm11
+; AVX512DQ-NEXT: vmovdqa64 480(%rdi), %xmm11
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm11, %xmm5
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-NEXT: vmovdqa 464(%rdi), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 464(%rdi), %xmm6
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm6, %xmm5
; AVX512DQ-NEXT: vmovdqa64 %xmm6, %xmm22
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %xmm9
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %xmm9
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm9, %xmm6
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm12 = [2,11,2,11,2,11,2,11]
@@ -13763,16 +13763,16 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vbroadcasti64x4 {{.*#+}} zmm28 = [0,1,10,11,0,1,10,11]
; AVX512DQ-NEXT: # zmm28 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-NEXT: vpermt2q %zmm3, %zmm28, %zmm2
-; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm3
+; AVX512DQ-NEXT: vmovdqa64 240(%rdi), %xmm3
; AVX512DQ-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm3, %xmm3
-; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm10
+; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %xmm10
; AVX512DQ-NEXT: vpshufb %xmm0, %xmm10, %xmm4
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT: vmovdqa 208(%rdi), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 208(%rdi), %xmm5
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm4
; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm25
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm5
; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm5
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
@@ -13832,9 +13832,9 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vmovdqa64 %xmm9, %xmm23
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; AVX512DQ-NEXT: vpermt2d %ymm3, %ymm12, %ymm4
-; AVX512DQ-NEXT: vmovdqa 416(%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 416(%rdi), %xmm0
; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT: vmovdqa 432(%rdi), %xmm11
+; AVX512DQ-NEXT: vmovdqa64 432(%rdi), %xmm11
; AVX512DQ-NEXT: vmovd {{.*#+}} xmm2 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
; AVX512DQ-NEXT: vpshufb %xmm2, %xmm11, %xmm3
; AVX512DQ-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -13860,8 +13860,8 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
; AVX512DQ-NEXT: vpermt2d %ymm3, %ymm12, %ymm13
; AVX512DQ-NEXT: vmovdqa64 %ymm12, %ymm17
-; AVX512DQ-NEXT: vmovdqa 160(%rdi), %xmm3
-; AVX512DQ-NEXT: vmovdqa 176(%rdi), %xmm9
+; AVX512DQ-NEXT: vmovdqa64 160(%rdi), %xmm3
+; AVX512DQ-NEXT: vmovdqa64 176(%rdi), %xmm9
; AVX512DQ-NEXT: vpshufb %xmm2, %xmm9, %xmm0
; AVX512DQ-NEXT: vpshufb %xmm2, %xmm3, %xmm2
; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm25
@@ -14358,37 +14358,37 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %zmm25
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm8 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
-; AVX512DQ-FCP-NEXT: vmovdqa 480(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 480(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm3
; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, %ymm24
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm7 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm5
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm2
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm11 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512DQ-FCP-NEXT: vmovdqa 416(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 416(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm4
; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm4, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm23
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm12 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, (%rsp) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm0, %ymm6
; AVX512DQ-FCP-NEXT: vpshufb %ymm12, %ymm6, %ymm3
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 352(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpmovqd %ymm2, %xmm4
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm10 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm21
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpmovqd %ymm3, %xmm9
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
@@ -14400,12 +14400,12 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm30 = [0,1,10,11,0,1,10,11]
; AVX512DQ-FCP-NEXT: # zmm30 = mem[0,1,2,3,0,1,2,3]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm30, %zmm14
-; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm2
; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm2, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, %ymm18
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm0, %ymm3
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm3, %ymm2
@@ -14796,13 +14796,13 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpmovqb %zmm1, %xmm2
; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm8
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT: vmovdqa 368(%rdi), %xmm10
+; AVX512BW-NEXT: vmovdqa64 368(%rdi), %xmm10
; AVX512BW-NEXT: vpshufb %xmm7, %xmm10, %xmm1
-; AVX512BW-NEXT: vmovdqa 352(%rdi), %xmm12
+; AVX512BW-NEXT: vmovdqa64 352(%rdi), %xmm12
; AVX512BW-NEXT: vpshufb %xmm7, %xmm12, %xmm3
; AVX512BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm1 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT: vmovdqa 336(%rdi), %xmm14
+; AVX512BW-NEXT: vmovdqa64 336(%rdi), %xmm14
; AVX512BW-NEXT: vpshufb %xmm1, %xmm14, %xmm4
; AVX512BW-NEXT: vmovdqa64 320(%rdi), %xmm18
; AVX512BW-NEXT: vpshufb %xmm1, %xmm18, %xmm5
@@ -14810,7 +14810,7 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
; AVX512BW-NEXT: vpmovqb %zmm8, %xmm4
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3]
-; AVX512BW-NEXT: vmovdqa 384(%rdi), %ymm9
+; AVX512BW-NEXT: vmovdqa64 384(%rdi), %ymm9
; AVX512BW-NEXT: vpmovqb %ymm9, %xmm3
; AVX512BW-NEXT: vpbroadcastq {{.*#+}} ymm29 = [0,9,0,9,0,9,0,9]
; AVX512BW-NEXT: vpermt2d %ymm2, %ymm29, %ymm3
@@ -15016,8 +15016,8 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-NEXT: vpmovqb %zmm18, %xmm4
; AVX512BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm1[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm1
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm1
+; AVX512BW-NEXT: vmovdqa64 176(%rdi), %xmm0
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
; AVX512BW-NEXT: vpshufb %xmm3, %xmm0, %xmm2
; AVX512BW-NEXT: vpshufb %xmm3, %xmm1, %xmm3
@@ -15201,25 +15201,25 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm19
; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm16 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,2,2,3,0,2,4,6]
-; AVX512BW-FCP-NEXT: vmovdqa 480(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 480(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpermd %ymm6, %ymm13, %ymm3
; AVX512BW-FCP-NEXT: vpshufb %ymm16, %ymm3, %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 448(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpermd %ymm1, %ymm13, %ymm23
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm23[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
-; AVX512BW-FCP-NEXT: vmovdqa 416(%rdi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 416(%rdi), %ymm4
; AVX512BW-FCP-NEXT: vpermd %ymm4, %ymm13, %ymm24
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rdi), %ymm10
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm10
; AVX512BW-FCP-NEXT: vpermd %ymm10, %ymm13, %ymm25
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm25[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4],ymm2[5],ymm5[6,7]
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-FCP-NEXT: vmovdqa 352(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 352(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vpmovqd %ymm0, %xmm26
; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm26[u,u,u,u,u,u,u,u,u,u,u,u,0,4,8,12]
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rdi), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdi), %ymm5
; AVX512BW-FCP-NEXT: vpmovqd %ymm5, %xmm31
; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} xmm14 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512BW-FCP-NEXT: vpshufb %xmm14, %xmm31, %xmm8
@@ -15370,7 +15370,7 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512BW-FCP-NEXT: vpermt2q %zmm2, %zmm27, %zmm0
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm8
+; AVX512BW-FCP-NEXT: vmovdqa64 224(%rdi), %ymm8
; AVX512BW-FCP-NEXT: vpermd %ymm8, %ymm13, %ymm10
; AVX512BW-FCP-NEXT: vpshufb %ymm16, %ymm10, %ymm0
; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -15380,7 +15380,7 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm0[7]
; AVX512BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm27
; AVX512BW-FCP-NEXT: vpermd %ymm27, %ymm13, %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm7
; AVX512BW-FCP-NEXT: vpermd %ymm7, %ymm13, %ymm3
; AVX512BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512BW-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
@@ -15550,13 +15550,13 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpmovqb %zmm1, %xmm2
; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %zmm8
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm7 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT: vmovdqa 368(%rdi), %xmm10
+; AVX512DQ-BW-NEXT: vmovdqa64 368(%rdi), %xmm10
; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm10, %xmm1
-; AVX512DQ-BW-NEXT: vmovdqa 352(%rdi), %xmm12
+; AVX512DQ-BW-NEXT: vmovdqa64 352(%rdi), %xmm12
; AVX512DQ-BW-NEXT: vpshufb %xmm7, %xmm12, %xmm3
; AVX512DQ-BW-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} zmm1 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT: vmovdqa 336(%rdi), %xmm14
+; AVX512DQ-BW-NEXT: vmovdqa64 336(%rdi), %xmm14
; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm14, %xmm4
; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %xmm18
; AVX512DQ-BW-NEXT: vpshufb %xmm1, %xmm18, %xmm5
@@ -15564,7 +15564,7 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
; AVX512DQ-BW-NEXT: vpmovqb %zmm8, %xmm4
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,3]
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdi), %ymm9
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %ymm9
; AVX512DQ-BW-NEXT: vpmovqb %ymm9, %xmm3
; AVX512DQ-BW-NEXT: vpbroadcastq {{.*#+}} ymm29 = [0,9,0,9,0,9,0,9]
; AVX512DQ-BW-NEXT: vpermt2d %ymm2, %ymm29, %ymm3
@@ -15770,8 +15770,8 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-NEXT: vpmovqb %zmm18, %xmm4
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm1[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 160(%rdi), %xmm1
-; AVX512DQ-BW-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX512DQ-BW-NEXT: vmovdqa64 160(%rdi), %xmm1
+; AVX512DQ-BW-NEXT: vmovdqa64 176(%rdi), %xmm0
; AVX512DQ-BW-NEXT: vpbroadcastw {{.*#+}} xmm3 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm0, %xmm2
; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm1, %xmm3
@@ -15955,25 +15955,25 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm19
; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm16 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,2,2,3,0,2,4,6]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 480(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 480(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm6, %ymm13, %ymm3
; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm16, %ymm3, %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm1, %ymm13, %ymm23
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm23[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 416(%rdi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 416(%rdi), %ymm4
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm4, %ymm13, %ymm24
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rdi), %ymm10
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm10
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm10, %ymm13, %ymm25
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm25[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,20,24,28,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4],ymm2[5],ymm5[6,7]
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 352(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 352(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vpmovqd %ymm0, %xmm26
; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm26[u,u,u,u,u,u,u,u,u,u,u,u,0,4,8,12]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdi), %ymm5
; AVX512DQ-BW-FCP-NEXT: vpmovqd %ymm5, %xmm31
; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} xmm14 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm14, %xmm31, %xmm8
@@ -16124,7 +16124,7 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm2, %zmm27, %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 224(%rdi), %ymm8
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 224(%rdi), %ymm8
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm8, %ymm13, %ymm10
; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm16, %ymm10, %ymm0
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -16134,7 +16134,7 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm0[7]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 160(%rdi), %ymm27
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm27, %ymm13, %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm7
; AVX512DQ-BW-FCP-NEXT: vpermd %ymm7, %ymm13, %ymm3
; AVX512DQ-BW-FCP-NEXT: vpbroadcastd {{.*#+}} ymm0 = [0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12]
; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-2.ll
index 8129dc05b02ab..c2f5a84bf08b4 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-2.ll
@@ -1116,13 +1116,13 @@ define void @store_i16_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
; AVX512-NEXT: vmovdqa %xmm14, 80(%rdx)
; AVX512-NEXT: vmovdqa %xmm11, 112(%rdx)
; AVX512-NEXT: vmovdqa %xmm15, 96(%rdx)
-; AVX512-NEXT: vmovdqa %xmm6, 192(%rdx)
-; AVX512-NEXT: vmovdqa %xmm8, 208(%rdx)
-; AVX512-NEXT: vmovdqa %xmm5, 240(%rdx)
-; AVX512-NEXT: vmovdqa %xmm4, 224(%rdx)
-; AVX512-NEXT: vmovdqa %xmm3, 128(%rdx)
-; AVX512-NEXT: vmovdqa %xmm2, 144(%rdx)
-; AVX512-NEXT: vmovdqa %xmm1, 176(%rdx)
+; AVX512-NEXT: vmovdqa64 %xmm6, 192(%rdx)
+; AVX512-NEXT: vmovdqa64 %xmm8, 208(%rdx)
+; AVX512-NEXT: vmovdqa64 %xmm5, 240(%rdx)
+; AVX512-NEXT: vmovdqa64 %xmm4, 224(%rdx)
+; AVX512-NEXT: vmovdqa64 %xmm3, 128(%rdx)
+; AVX512-NEXT: vmovdqa64 %xmm2, 144(%rdx)
+; AVX512-NEXT: vmovdqa64 %xmm1, 176(%rdx)
; AVX512-NEXT: vmovdqa64 %xmm16, 160(%rdx)
; AVX512-NEXT: retq
;
@@ -1169,13 +1169,13 @@ define void @store_i16_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
; AVX512-FCP-NEXT: vmovdqa %xmm14, 80(%rdx)
; AVX512-FCP-NEXT: vmovdqa %xmm11, 112(%rdx)
; AVX512-FCP-NEXT: vmovdqa %xmm15, 96(%rdx)
-; AVX512-FCP-NEXT: vmovdqa %xmm6, 192(%rdx)
-; AVX512-FCP-NEXT: vmovdqa %xmm8, 208(%rdx)
-; AVX512-FCP-NEXT: vmovdqa %xmm5, 240(%rdx)
-; AVX512-FCP-NEXT: vmovdqa %xmm4, 224(%rdx)
-; AVX512-FCP-NEXT: vmovdqa %xmm3, 128(%rdx)
-; AVX512-FCP-NEXT: vmovdqa %xmm2, 144(%rdx)
-; AVX512-FCP-NEXT: vmovdqa %xmm1, 176(%rdx)
+; AVX512-FCP-NEXT: vmovdqa64 %xmm6, 192(%rdx)
+; AVX512-FCP-NEXT: vmovdqa64 %xmm8, 208(%rdx)
+; AVX512-FCP-NEXT: vmovdqa64 %xmm5, 240(%rdx)
+; AVX512-FCP-NEXT: vmovdqa64 %xmm4, 224(%rdx)
+; AVX512-FCP-NEXT: vmovdqa64 %xmm3, 128(%rdx)
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, 144(%rdx)
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, 176(%rdx)
; AVX512-FCP-NEXT: vmovdqa64 %xmm16, 160(%rdx)
; AVX512-FCP-NEXT: retq
;
@@ -1222,13 +1222,13 @@ define void @store_i16_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
; AVX512DQ-NEXT: vmovdqa %xmm14, 80(%rdx)
; AVX512DQ-NEXT: vmovdqa %xmm11, 112(%rdx)
; AVX512DQ-NEXT: vmovdqa %xmm15, 96(%rdx)
-; AVX512DQ-NEXT: vmovdqa %xmm6, 192(%rdx)
-; AVX512DQ-NEXT: vmovdqa %xmm8, 208(%rdx)
-; AVX512DQ-NEXT: vmovdqa %xmm5, 240(%rdx)
-; AVX512DQ-NEXT: vmovdqa %xmm4, 224(%rdx)
-; AVX512DQ-NEXT: vmovdqa %xmm3, 128(%rdx)
-; AVX512DQ-NEXT: vmovdqa %xmm2, 144(%rdx)
-; AVX512DQ-NEXT: vmovdqa %xmm1, 176(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %xmm6, 192(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %xmm8, 208(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %xmm5, 240(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %xmm4, 224(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %xmm3, 128(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %xmm2, 144(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %xmm1, 176(%rdx)
; AVX512DQ-NEXT: vmovdqa64 %xmm16, 160(%rdx)
; AVX512DQ-NEXT: retq
;
@@ -1275,13 +1275,13 @@ define void @store_i16_stride2_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %out.v
; AVX512DQ-FCP-NEXT: vmovdqa %xmm14, 80(%rdx)
; AVX512DQ-FCP-NEXT: vmovdqa %xmm11, 112(%rdx)
; AVX512DQ-FCP-NEXT: vmovdqa %xmm15, 96(%rdx)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, 192(%rdx)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm8, 208(%rdx)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm5, 240(%rdx)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, 224(%rdx)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, 128(%rdx)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, 144(%rdx)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, 176(%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm6, 192(%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm8, 208(%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm5, 240(%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, 224(%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, 128(%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, 144(%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, 176(%rdx)
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, 160(%rdx)
; AVX512DQ-FCP-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
index 1fd85afea7d2d..9b1f3a114929b 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
@@ -1777,7 +1777,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = m64bcst ^ (ymm1 & (ymm0 ^ m64bcst))
-; AVX512-NEXT: vmovdqa %ymm1, 128(%r9)
+; AVX512-NEXT: vmovdqa64 %ymm1, 128(%r9)
; AVX512-NEXT: vmovdqa64 %zmm3, 64(%r9)
; AVX512-NEXT: vmovdqa64 %zmm6, (%r9)
; AVX512-NEXT: vzeroupper
@@ -1843,7 +1843,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = m64bcst ^ (ymm1 & (ymm0 ^ m64bcst))
-; AVX512-FCP-NEXT: vmovdqa %ymm1, 128(%r9)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm1, 128(%r9)
; AVX512-FCP-NEXT: vmovdqa64 %zmm2, (%r9)
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512-FCP-NEXT: vzeroupper
@@ -1912,7 +1912,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = m64bcst ^ (ymm1 & (ymm0 ^ m64bcst))
-; AVX512DQ-NEXT: vmovdqa %ymm1, 128(%r9)
+; AVX512DQ-NEXT: vmovdqa64 %ymm1, 128(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm3, 64(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm6, (%r9)
; AVX512DQ-NEXT: vzeroupper
@@ -1978,7 +1978,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = m64bcst ^ (ymm1 & (ymm0 ^ m64bcst))
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, 128(%r9)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, 128(%r9)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, (%r9)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512DQ-FCP-NEXT: vzeroupper
@@ -2005,7 +2005,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm0 = [28,1,2,3,4,29,6,7,8,9,30,11,12,13,14,31]
; AVX512BW-NEXT: vpermi2w %ymm2, %ymm3, %ymm0
-; AVX512BW-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512BW-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -2030,7 +2030,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [28,1,2,3,4,29,6,7,8,9,30,11,12,13,14,31]
; AVX512BW-FCP-NEXT: vpermi2w %ymm2, %ymm3, %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512BW-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
@@ -2055,7 +2055,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm0 = [28,1,2,3,4,29,6,7,8,9,30,11,12,13,14,31]
; AVX512DQ-BW-NEXT: vpermi2w %ymm2, %ymm3, %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -2080,7 +2080,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [28,1,2,3,4,29,6,7,8,9,30,11,12,13,14,31]
; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm2, %ymm3, %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%in.vec0 = load <16 x i16>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
index f2505ad876e42..3563b8c97ee5f 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
@@ -8657,9 +8657,9 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: subq $1288, %rsp # imm = 0x508
; AVX512-FCP-NEXT: vmovdqa 96(%rcx), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 96(%rdx), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
@@ -8667,11 +8667,11 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm21 = [4,5,4,5,4,5,4,5,21,22,21,22,21,22,23,23]
; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm21, %zmm1
; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm3
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]
@@ -8682,7 +8682,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm20 = [8,21,10,11,20,13,14,23]
; AVX512-FCP-NEXT: vmovdqa 96(%r8), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm2, %zmm3
@@ -8693,7 +8693,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm3[0,1,2,3]
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 96(%r9), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm2, %ymm1
; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]
@@ -8703,7 +8703,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm25, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 64(%rcx), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 96(%r9), %xmm2
; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
@@ -8712,17 +8712,17 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm26 = [0,0,0,0,8,8,0,9]
; AVX512-FCP-NEXT: vpermi2q %zmm1, %zmm2, %zmm26
; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm21, %zmm1
; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm3
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]
@@ -8730,7 +8730,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
; AVX512-FCP-NEXT: vmovdqa64 %zmm2, %zmm1
; AVX512-FCP-NEXT: vmovdqa 64(%r8), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm3
; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm20, %zmm1
; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm4[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
@@ -8738,7 +8738,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[0,1,2,3]
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 64(%r9), %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm2, %ymm1
; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm2, %ymm2
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm25, %zmm2
@@ -8814,16 +8814,16 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm31, %zmm3
; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[8],ymm1[8],ymm3[9],ymm1[9],ymm3[10],ymm1[10],ymm3[11],ymm1[11]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]
; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm21, %zmm1
; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm3
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm6
; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -8842,7 +8842,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,2,3]
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%r9), %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm1
; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm3, %ymm3
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm25, %zmm3
@@ -8953,7 +8953,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm0 = [0,0,0,1,0,10,10,0]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm0, %zmm2
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[2],mem[2],ymm1[3],mem[3],ymm1[8],mem[8],ymm1[9],mem[9],ymm1[10],mem[10],ymm1[11],mem[11]
; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
@@ -8963,7 +8963,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm6 = ymm6[0],mem[0],ymm6[1],mem[1],ymm6[2],mem[2],ymm6[3],mem[3],ymm6[8],mem[8],ymm6[9],mem[9],ymm6[10],mem[10],ymm6[11],mem[11]
; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm29, %zmm6
@@ -8981,7 +8981,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm6
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm0, %zmm6
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[2],mem[2],ymm1[3],mem[3],ymm1[8],mem[8],ymm1[9],mem[9],ymm1[10],mem[10],ymm1[11],mem[11]
; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
@@ -8991,7 +8991,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
; AVX512-FCP-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[2],mem[2],ymm10[3],mem[3],ymm10[8],mem[8],ymm10[9],mem[9],ymm10[10],mem[10],ymm10[11],mem[11]
; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm29, %zmm10
@@ -9019,7 +9019,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm10
; AVX512-FCP-NEXT: vmovdqa64 %xmm19, %xmm12
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm10[4],xmm12[4],xmm10[5],xmm12[5],xmm10[6],xmm12[6],xmm10[7],xmm12[7]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm12 = ymm12[0],mem[0],ymm12[1],mem[1],ymm12[2],mem[2],ymm12[3],mem[3],ymm12[8],mem[8],ymm12[9],mem[9],ymm12[10],mem[10],ymm12[11],mem[11]
; AVX512-FCP-NEXT: vpermt2d %zmm10, %zmm29, %zmm12
@@ -9509,9 +9509,9 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: subq $1288, %rsp # imm = 0x508
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rcx), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdx), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]
; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
@@ -9519,11 +9519,11 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm21 = [4,5,4,5,4,5,4,5,21,22,21,22,21,22,23,23]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm21, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm3
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]
@@ -9534,7 +9534,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm20 = [8,21,10,11,20,13,14,23]
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r8), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, %zmm3
@@ -9545,7 +9545,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm3[0,1,2,3]
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r9), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]
; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm2, %ymm1
; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]
@@ -9555,7 +9555,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm25, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rcx), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r9), %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
@@ -9564,17 +9564,17 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm26 = [0,0,0,0,8,8,0,9]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm1, %zmm2, %zmm26
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdx), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm21, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm3
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]
@@ -9582,7 +9582,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r8), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm3
; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm20, %zmm1
; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm4[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]
@@ -9590,7 +9590,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[0,1,2,3]
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r9), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm2, %ymm1
; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm2, %ymm2
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm25, %zmm2
@@ -9666,16 +9666,16 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm31, %zmm3
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[8],ymm1[8],ymm3[9],ymm1[9],ymm3[10],ymm1[10],ymm3[11],ymm1[11]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm21, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -9694,7 +9694,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,2,3]
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm1
; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm3, %ymm3
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm25, %zmm3
@@ -9805,7 +9805,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm0 = [0,0,0,1,0,10,10,0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm0, %zmm2
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[2],mem[2],ymm1[3],mem[3],ymm1[8],mem[8],ymm1[9],mem[9],ymm1[10],mem[10],ymm1[11],mem[11]
; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
@@ -9815,7 +9815,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm6 = ymm6[0],mem[0],ymm6[1],mem[1],ymm6[2],mem[2],ymm6[3],mem[3],ymm6[8],mem[8],ymm6[9],mem[9],ymm6[10],mem[10],ymm6[11],mem[11]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm29, %zmm6
@@ -9833,7 +9833,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm5, %xmm6
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm0, %zmm6
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[2],mem[2],ymm1[3],mem[3],ymm1[8],mem[8],ymm1[9],mem[9],ymm1[10],mem[10],ymm1[11],mem[11]
; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
@@ -9843,7 +9843,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
; AVX512DQ-FCP-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[2],mem[2],ymm10[3],mem[3],ymm10[8],mem[8],ymm10[9],mem[9],ymm10[10],mem[10],ymm10[11],mem[11]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm29, %zmm10
@@ -9871,7 +9871,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm10
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm19, %xmm12
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm10[4],xmm12[4],xmm10[5],xmm12[5],xmm10[6],xmm12[6],xmm10[7],xmm12[7]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm12 = ymm12[0],mem[0],ymm12[1],mem[1],ymm12[2],mem[2],ymm12[3],mem[3],ymm12[8],mem[8],ymm12[9],mem[9],ymm12[10],mem[10],ymm12[11],mem[11]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm10, %zmm29, %zmm12
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
index c36e9a36bfe59..7747401277d85 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
@@ -2944,7 +2944,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm3))
-; AVX512-NEXT: vmovdqa %ymm0, 192(%rcx)
+; AVX512-NEXT: vmovdqa64 %ymm0, 192(%rcx)
; AVX512-NEXT: vmovdqa64 %zmm5, (%rcx)
; AVX512-NEXT: vmovdqa64 %zmm2, 128(%rcx)
; AVX512-NEXT: vmovdqa64 %zmm12, 64(%rcx)
@@ -3071,7 +3071,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm3 ^ (mem & (ymm0 ^ ymm3))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
-; AVX512-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm10, 128(%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm8, (%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm6, 64(%rax)
@@ -3203,7 +3203,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,3,2]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm3))
-; AVX512DQ-NEXT: vmovdqa %ymm0, 192(%rcx)
+; AVX512DQ-NEXT: vmovdqa64 %ymm0, 192(%rcx)
; AVX512DQ-NEXT: vmovdqa64 %zmm5, (%rcx)
; AVX512DQ-NEXT: vmovdqa64 %zmm2, 128(%rcx)
; AVX512DQ-NEXT: vmovdqa64 %zmm12, 64(%rcx)
@@ -3330,7 +3330,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm3 ^ (mem & (ymm0 ^ ymm3))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm2))
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm10, 128(%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, (%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, 64(%rax)
@@ -3379,7 +3379,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: movw $-7741, %cx # imm = 0xE1C3
; AVX512BW-NEXT: kmovd %ecx, %k1
; AVX512BW-NEXT: vmovdqu16 %ymm5, %ymm4 {%k1}
-; AVX512BW-NEXT: vmovdqa %ymm4, 192(%rax)
+; AVX512BW-NEXT: vmovdqa64 %ymm4, 192(%rax)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -3425,7 +3425,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: movw $-7741, %cx # imm = 0xE1C3
; AVX512BW-FCP-NEXT: kmovd %ecx, %k1
; AVX512BW-FCP-NEXT: vmovdqu16 %ymm5, %ymm4 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa %ymm4, 192(%rax)
+; AVX512BW-FCP-NEXT: vmovdqa64 %ymm4, 192(%rax)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
@@ -3471,7 +3471,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: movw $-7741, %cx # imm = 0xE1C3
; AVX512DQ-BW-NEXT: kmovd %ecx, %k1
; AVX512DQ-BW-NEXT: vmovdqu16 %ymm5, %ymm4 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa %ymm4, 192(%rax)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm4, 192(%rax)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -3517,7 +3517,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: movw $-7741, %cx # imm = 0xE1C3
; AVX512DQ-BW-FCP-NEXT: kmovd %ecx, %k1
; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %ymm5, %ymm4 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm4, 192(%rax)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm4, 192(%rax)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%in.vec0 = load <16 x i16>, ptr %in.vecptr0, align 64
@@ -12438,7 +12438,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpshufb %ymm3, %ymm2, %ymm1
; AVX512-NEXT: vmovdqa64 %ymm2, %ymm19
; AVX512-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa 64(%rcx), %ymm0
; AVX512-NEXT: vmovdqa {{.*#+}} ymm5 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
; AVX512-NEXT: vpshufb %ymm5, %ymm0, %ymm2
@@ -12447,7 +12447,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqa {{.*#+}} ymm8 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
; AVX512-NEXT: vpshufb %ymm8, %ymm14, %ymm4
; AVX512-NEXT: vpor %ymm2, %ymm4, %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa 64(%rsi), %ymm0
; AVX512-NEXT: vmovdqa {{.*#+}} ymm9 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512-NEXT: vpshufb %ymm9, %ymm0, %ymm11
@@ -12456,7 +12456,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqa {{.*#+}} ymm10 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
; AVX512-NEXT: vpshufb %ymm10, %ymm7, %ymm12
; AVX512-NEXT: vpor %ymm11, %ymm12, %ymm11
-; AVX512-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa (%r9), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufb %ymm13, %ymm0, %ymm11
@@ -12464,7 +12464,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufb %ymm3, %ymm0, %ymm12
; AVX512-NEXT: vpor %ymm11, %ymm12, %ymm11
-; AVX512-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa (%rcx), %ymm0
; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufb %ymm5, %ymm0, %ymm11
@@ -12472,7 +12472,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpshufb %ymm8, %ymm0, %ymm12
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm30
; AVX512-NEXT: vpor %ymm11, %ymm12, %ymm11
-; AVX512-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa (%rsi), %ymm0
; AVX512-NEXT: vpshufb %ymm9, %ymm0, %ymm11
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm24
@@ -12480,7 +12480,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpshufb %ymm10, %ymm0, %ymm12
; AVX512-NEXT: vmovdqa64 %ymm0, %ymm26
; AVX512-NEXT: vpor %ymm11, %ymm12, %ymm11
-; AVX512-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa 32(%rcx), %ymm15
; AVX512-NEXT: vpshufb %ymm5, %ymm15, %ymm11
; AVX512-NEXT: vmovdqa 32(%rdx), %ymm6
@@ -12507,13 +12507,13 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vmovdqa 96(%rsi), %ymm8
; AVX512-NEXT: vpshufb %ymm9, %ymm8, %ymm3
-; AVX512-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa 96(%rdi), %ymm9
; AVX512-NEXT: vpshufb %ymm10, %ymm9, %ymm5
; AVX512-NEXT: vpor %ymm3, %ymm5, %ymm3
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vmovdqa 96(%r9), %ymm5
-; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm5[2,1,2,3,6,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,0,3,3,4,5,6,7,8,8,11,11,12,13,14,15]
; AVX512-NEXT: vpmovsxbq {{.*#+}} zmm29 = [2,0,2,0,11,11,0,11]
@@ -12537,7 +12537,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} ymm3 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[2,2,2,2,6,6,6,6]
-; AVX512-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufd {{.*#+}} ymm5 = ymm9[3,3,3,3,7,7,7,7]
; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7,8,9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm5 = ymm8[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
@@ -12549,9 +12549,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
; AVX512-NEXT: # ymm3 = mem[0,1,0,1]
; AVX512-NEXT: vmovdqa %ymm11, %ymm12
-; AVX512-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufb %ymm3, %ymm11, %ymm9
-; AVX512-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufd {{.*#+}} ymm11 = ymm13[2,2,2,2,6,6,6,6]
; AVX512-NEXT: vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm11[2],ymm9[3,4],ymm11[5],ymm9[6,7,8,9],ymm11[10],ymm9[11,12],ymm11[13],ymm9[14,15]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm11 = ymm12[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
@@ -12563,7 +12563,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm22 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
; AVX512-NEXT: vpternlogq {{.*#+}} zmm9 = zmm5 ^ (zmm22 & (zmm9 ^ zmm5))
; AVX512-NEXT: vmovdqa 96(%r8), %ymm5
-; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[1,2,2,3,5,6,6,7]
; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,1,3,2]
; AVX512-NEXT: vpbroadcastd 124(%r8), %ymm11
@@ -12571,10 +12571,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm27 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm27 & (zmm5 ^ zmm9))
; AVX512-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} ymm5 = ymm0[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
; AVX512-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[2,2,2,2,6,6,6,6]
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufd {{.*#+}} ymm9 = ymm1[3,3,3,3,7,7,7,7]
; AVX512-NEXT: vpblendw {{.*#+}} ymm5 = ymm9[0,1,2],ymm5[3],ymm9[4,5],ymm5[6],ymm9[7,8,9,10],ymm5[11],ymm9[12,13],ymm5[14],ymm9[15]
; AVX512-NEXT: vpshufhw {{.*#+}} ymm9 = ymm0[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
@@ -12950,14 +12950,14 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: # ymm13 = mem[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
; AVX512-NEXT: vpshufd {{.*#+}} ymm13 = ymm13[0,0,2,1,4,4,6,5]
; AVX512-NEXT: vpblendw {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpshufd {{.*#+}} ymm13 = ymm0[0,0,2,1,4,4,6,5]
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = ymm0[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,u,u,u,u],zero,zero
; AVX512-NEXT: vpermq {{.*#+}} ymm13 = ymm13[2,1,3,3]
; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
; AVX512-NEXT: vpandnq %ymm13, %ymm18, %ymm13
; AVX512-NEXT: vinserti64x4 $1, %ymm13, %zmm14, %zmm13
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17],zero,zero,zero,zero
; AVX512-NEXT: vprold $16, %ymm0, %ymm19
; AVX512-NEXT: vpermq {{.*#+}} ymm19 = ymm19[2,2,2,2]
@@ -13026,61 +13026,61 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: subq $1992, %rsp # imm = 0x7C8
; AVX512-FCP-NEXT: vmovdqa 64(%r9), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm0
; AVX512-FCP-NEXT: vmovdqa 64(%r8), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 64(%rcx), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm3
; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm4
; AVX512-FCP-NEXT: vpor %ymm3, %ymm4, %ymm3
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm5
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm6
-; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm6
; AVX512-FCP-NEXT: vpor %ymm5, %ymm6, %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm5
; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm6
-; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm6, %ymm6
; AVX512-FCP-NEXT: vpor %ymm5, %ymm6, %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm5
; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm6
-; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm6, %ymm6
; AVX512-FCP-NEXT: vpor %ymm5, %ymm6, %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm5, %ymm5
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm6
-; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm6
; AVX512-FCP-NEXT: vpor %ymm5, %ymm6, %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm5
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm5
; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm6
; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -13115,7 +13115,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm4 = [2,0,2,0,11,11,0,11]
; AVX512-FCP-NEXT: vmovdqa 96(%r9), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
; AVX512-FCP-NEXT: vpermi2q %zmm0, %zmm1, %zmm4
@@ -13125,7 +13125,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm9 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
; AVX512-FCP-NEXT: vpandn %ymm0, %ymm9, %ymm0
; AVX512-FCP-NEXT: vmovdqa 64(%rax), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
@@ -13133,7 +13133,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpbroadcastd 8(%rax), %ymm0
; AVX512-FCP-NEXT: vpandn %ymm0, %ymm9, %ymm0
; AVX512-FCP-NEXT: vmovdqa (%rax), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -13144,10 +13144,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermi2q %zmm0, %zmm2, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm0
; AVX512-FCP-NEXT: vmovdqa64 %ymm1, %ymm22
-; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm3[3,3,3,3,7,7,7,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm7[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
@@ -13155,9 +13155,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm29 = [0,2,2,3,10,9,11,11]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm29, %zmm1
-; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm5[3,3,3,3,7,7,7,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6,7,8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14,15]
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
@@ -13169,7 +13169,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm1 ^ (zmm19 & (zmm2 ^ zmm1))
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [5,0,0,0,6,0,0,6]
; AVX512-FCP-NEXT: vmovdqa 96(%r8), %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm0
; AVX512-FCP-NEXT: vpbroadcastd 124(%r8), %ymm1
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
@@ -13205,7 +13205,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa 96(%r8), %xmm1
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm0
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13,0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0
@@ -13295,9 +13295,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm28 ^ (zmm13 & (zmm28 ^ zmm0))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm28 ^ (zmm4 & (zmm28 ^ zmm1))
; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm1
@@ -13323,7 +13323,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa 32(%r8), %xmm4
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm1, %ymm1
; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm2
; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
@@ -13345,7 +13345,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm4 = ymm31[2,2,2,2,6,6,6,6]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0],ymm1[1],ymm4[2,3],ymm1[4],ymm4[5,6,7,8],ymm1[9],ymm4[10,11],ymm1[12],ymm4[13,14,15]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm29, %zmm1
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-FCP-NEXT: vpmovsxdq {{.*#+}} ymm5 = [218894094,0,488382238,488382238]
; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm0
; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
@@ -13369,7 +13369,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm7 = [2,1,3,2,10,10,10,11]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm7, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm0
; AVX512-FCP-NEXT: vmovdqa %ymm3, %ymm4
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
@@ -13382,9 +13382,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm8 = ymm24[1,1,1,1,5,5,5,5]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm8[2],ymm1[3,4],ymm8[5],ymm1[6,7,8,9],ymm8[10],ymm1[11,12],ymm8[13],ymm1[14,15]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm27, %zmm1
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm9, %ymm0
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm8 = ymm13[2,2,2,2,6,6,6,6]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm8[2],ymm0[3,4],ymm8[5],ymm0[6,7,8,9],ymm8[10],ymm0[11,12],ymm8[13],ymm0[14,15]
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm8 = ymm9[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
@@ -13393,9 +13393,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm7, %zmm15
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm23 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm1 ^ (zmm23 & (zmm15 ^ zmm1))
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-FCP-NEXT: vprold $16, %ymm2, %ymm0
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm6[1,2,2,3,5,6,6,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} ymm20 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
@@ -13412,7 +13412,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermd %zmm3, %zmm0, %zmm16
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm31 & (zmm16 ^ zmm1))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm15))
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm0
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm19[2,2,2,2,6,6,6,6]
@@ -13422,9 +13422,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm15 = ymm19[1,1,1,1,5,5,5,5]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm15[2],ymm1[3,4],ymm15[5],ymm1[6,7,8,9],ymm15[10],ymm1[11,12],ymm15[13],ymm1[14,15]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm27, %zmm1
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufb %ymm14, %ymm3, %ymm0
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm15 = ymm4[2,2,2,2,6,6,6,6]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm0[0,1],ymm15[2],ymm0[3,4],ymm15[5],ymm0[6,7,8,9],ymm15[10],ymm0[11,12],ymm15[13],ymm0[14,15]
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm3[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
@@ -13459,16 +13459,16 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm7 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm14 = ymm6[3,3,3,3,7,7,7,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm14[2],ymm7[3,4],ymm14[5],ymm7[6,7,8,9],ymm14[10],ymm7[11,12],ymm14[13],ymm7[14,15]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} ymm21 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15,0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm6
; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm14
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm27 = [2,2,2,3,8,8,8,9]
; AVX512-FCP-NEXT: vpermt2q %zmm14, %zmm27, %zmm7
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm23 & (zmm0 ^ zmm1))
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
; AVX512-FCP-NEXT: vprold $16, %ymm6, %ymm1
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm14 = ymm2[1,2,2,3,5,6,6,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm14[0,1],ymm1[2],ymm14[3,4],ymm1[5],ymm14[6,7,8,9],ymm1[10],ymm14[11,12],ymm1[13],ymm14[14,15]
; AVX512-FCP-NEXT: vmovdqa64 %ymm20, %ymm8
@@ -13524,7 +13524,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q %zmm9, %zmm25, %zmm10
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm10 = zmm5 ^ (zmm24 & (zmm10 ^ zmm5))
; AVX512-FCP-NEXT: vmovdqa64 %ymm21, %ymm3
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm4, %xmm3
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm9 = ymm2[3,3,3,3,7,7,7,7]
@@ -13553,7 +13553,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512-FCP-NEXT: # zmm0 = (zmm0 & zmm17) | mem
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm0 ^ (zmm9 & (zmm5 ^ zmm0))
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
; AVX512-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm1 = mem[0,1,1,3,4,5,5,7]
@@ -13572,24 +13572,24 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm9, %ymm9
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm9, %zmm9
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm4[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
; AVX512-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm12 = mem[0,1,1,3,4,5,5,7]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm12[0,1],ymm11[2],ymm12[3,4],ymm11[5],ymm12[6,7,8,9],ymm11[10],ymm12[11,12],ymm11[13],ymm12[14,15]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm12
; AVX512-FCP-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Folded Reload
; AVX512-FCP-NEXT: # ymm13 = mem[1,1,1,1,5,5,5,5]
; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7,8,9],ymm13[10],ymm12[11,12],ymm13[13],ymm12[14,15]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,4,0,0,0,5,0,0]
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512-FCP-NEXT: vpermd %ymm4, %ymm13, %ymm13
; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm14 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
; AVX512-FCP-NEXT: vpandn %ymm13, %ymm14, %ymm13
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm4[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17,u,u,u,u],zero,zero
; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm13, %zmm6, %zmm6
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero
; AVX512-FCP-NEXT: vprold $16, %ymm4, %ymm21
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm21 = ymm21[2,2,2,2]
@@ -13666,7 +13666,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpshufb %ymm3, %ymm2, %ymm1
; AVX512DQ-NEXT: vmovdqa64 %ymm2, %ymm19
; AVX512DQ-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa 64(%rcx), %ymm0
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
; AVX512DQ-NEXT: vpshufb %ymm5, %ymm0, %ymm2
@@ -13675,7 +13675,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm8 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
; AVX512DQ-NEXT: vpshufb %ymm8, %ymm14, %ymm4
; AVX512DQ-NEXT: vpor %ymm2, %ymm4, %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa 64(%rsi), %ymm0
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm9 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512DQ-NEXT: vpshufb %ymm9, %ymm0, %ymm11
@@ -13684,7 +13684,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm10 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
; AVX512DQ-NEXT: vpshufb %ymm10, %ymm7, %ymm12
; AVX512DQ-NEXT: vpor %ymm11, %ymm12, %ymm11
-; AVX512DQ-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa (%r9), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufb %ymm13, %ymm0, %ymm11
@@ -13692,7 +13692,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufb %ymm3, %ymm0, %ymm12
; AVX512DQ-NEXT: vpor %ymm11, %ymm12, %ymm11
-; AVX512DQ-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa (%rcx), %ymm0
; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufb %ymm5, %ymm0, %ymm11
@@ -13700,7 +13700,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpshufb %ymm8, %ymm0, %ymm12
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm30
; AVX512DQ-NEXT: vpor %ymm11, %ymm12, %ymm11
-; AVX512DQ-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa (%rsi), %ymm0
; AVX512DQ-NEXT: vpshufb %ymm9, %ymm0, %ymm11
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm24
@@ -13708,7 +13708,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpshufb %ymm10, %ymm0, %ymm12
; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm26
; AVX512DQ-NEXT: vpor %ymm11, %ymm12, %ymm11
-; AVX512DQ-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa 32(%rcx), %ymm15
; AVX512DQ-NEXT: vpshufb %ymm5, %ymm15, %ymm11
; AVX512DQ-NEXT: vmovdqa 32(%rdx), %ymm6
@@ -13735,13 +13735,13 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vmovdqa 96(%rsi), %ymm8
; AVX512DQ-NEXT: vpshufb %ymm9, %ymm8, %ymm3
-; AVX512DQ-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm9
; AVX512DQ-NEXT: vpshufb %ymm10, %ymm9, %ymm5
; AVX512DQ-NEXT: vpor %ymm3, %ymm5, %ymm3
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vmovdqa 96(%r9), %ymm5
-; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm5[2,1,2,3,6,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,0,3,3,4,5,6,7,8,8,11,11,12,13,14,15]
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm29 = [2,0,2,0,11,11,0,11]
@@ -13765,7 +13765,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm3 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm5 = ymm9[3,3,3,3,7,7,7,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7,8,9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm5 = ymm8[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
@@ -13777,9 +13777,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
; AVX512DQ-NEXT: # ymm3 = mem[0,1,0,1]
; AVX512DQ-NEXT: vmovdqa %ymm11, %ymm12
-; AVX512DQ-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufb %ymm3, %ymm11, %ymm9
-; AVX512DQ-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm11 = ymm13[2,2,2,2,6,6,6,6]
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm9 = ymm9[0,1],ymm11[2],ymm9[3,4],ymm11[5],ymm9[6,7,8,9],ymm11[10],ymm9[11,12],ymm11[13],ymm9[14,15]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm11 = ymm12[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
@@ -13791,7 +13791,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm22 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm9 = zmm5 ^ (zmm22 & (zmm9 ^ zmm5))
; AVX512DQ-NEXT: vmovdqa 96(%r8), %ymm5
-; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[1,2,2,3,5,6,6,7]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,1,3,2]
; AVX512DQ-NEXT: vpbroadcastd 124(%r8), %ymm11
@@ -13799,10 +13799,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm27 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm27 & (zmm5 ^ zmm9))
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm5 = ymm0[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[2,2,2,2,6,6,6,6]
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm9 = ymm1[3,3,3,3,7,7,7,7]
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm9[0,1,2],ymm5[3],ymm9[4,5],ymm5[6],ymm9[7,8,9,10],ymm5[11],ymm9[12,13],ymm5[14],ymm9[15]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm9 = ymm0[0,1,2,3,5,4,6,7,8,9,10,11,13,12,14,15]
@@ -14178,14 +14178,14 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: # ymm13 = mem[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm13 = ymm13[0,0,2,1,4,4,6,5]
; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm13 = ymm0[0,0,2,1,4,4,6,5]
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = ymm0[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,u,u,u,u],zero,zero
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm13 = ymm13[2,1,3,3]
; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
; AVX512DQ-NEXT: vpandnq %ymm13, %ymm18, %ymm13
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm13, %zmm14, %zmm13
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17],zero,zero,zero,zero
; AVX512DQ-NEXT: vprold $16, %ymm0, %ymm19
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm19 = ymm19[2,2,2,2]
@@ -14254,61 +14254,61 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: subq $1992, %rsp # imm = 0x7C8
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r9), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r8), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rcx), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdx), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm4
; AVX512DQ-FCP-NEXT: vpor %ymm3, %ymm4, %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm5
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19]
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm6
; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm6, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm6, %ymm6
; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm6, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm6, %ymm6
; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm6, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm6
; AVX512DQ-FCP-NEXT: vpor %ymm5, %ymm6, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm6
; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -14343,7 +14343,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm4 = [2,0,2,0,11,11,0,11]
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r9), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} ymm0 = ymm1[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
; AVX512DQ-FCP-NEXT: vpermi2q %zmm0, %zmm1, %zmm4
@@ -14353,7 +14353,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm9 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
; AVX512DQ-FCP-NEXT: vpandn %ymm0, %ymm9, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rax), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
@@ -14361,7 +14361,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpbroadcastd 8(%rax), %ymm0
; AVX512DQ-FCP-NEXT: vpandn %ymm0, %ymm9, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -14372,10 +14372,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermi2q %zmm0, %zmm2, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, %ymm22
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm3[3,3,3,3,7,7,7,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm7[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
@@ -14383,9 +14383,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm29 = [0,2,2,3,10,9,11,11]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm29, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm2 = ymm5[3,3,3,3,7,7,7,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6,7,8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14,15]
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
@@ -14397,7 +14397,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm1 ^ (zmm19 & (zmm2 ^ zmm1))
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [5,0,0,0,6,0,0,6]
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r8), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpbroadcastd 124(%r8), %ymm1
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
@@ -14433,7 +14433,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r8), %xmm1
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13,0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0
@@ -14523,9 +14523,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm28 ^ (zmm13 & (zmm28 ^ zmm0))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm28 = zmm28 ^ (zmm4 & (zmm28 ^ zmm1))
; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm1
@@ -14551,7 +14551,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %xmm4
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm2
; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm1
@@ -14573,7 +14573,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm4 = ymm31[2,2,2,2,6,6,6,6]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm4[0],ymm1[1],ymm4[2,3],ymm1[4],ymm4[5,6,7,8],ymm1[9],ymm4[10,11],ymm1[12],ymm4[13,14,15]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm29, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpmovsxdq {{.*#+}} ymm5 = [218894094,0,488382238,488382238]
; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
@@ -14597,7 +14597,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm7 = [2,1,3,2,10,10,10,11]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm7, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa %ymm3, %ymm4
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
@@ -14610,9 +14610,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm8 = ymm24[1,1,1,1,5,5,5,5]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm8[2],ymm1[3,4],ymm8[5],ymm1[6,7,8,9],ymm8[10],ymm1[11,12],ymm8[13],ymm1[14,15]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm27, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm9, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm8 = ymm13[2,2,2,2,6,6,6,6]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm8[2],ymm0[3,4],ymm8[5],ymm0[6,7,8,9],ymm8[10],ymm0[11,12],ymm8[13],ymm0[14,15]
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm8 = ymm9[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
@@ -14621,9 +14621,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm7, %zmm15
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm23 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm15 = zmm1 ^ (zmm23 & (zmm15 ^ zmm1))
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vprold $16, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm6[1,2,2,3,5,6,6,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} ymm20 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
@@ -14640,7 +14640,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermd %zmm3, %zmm0, %zmm16
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm31 & (zmm16 ^ zmm1))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm15))
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm0
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm1 = ymm19[2,2,2,2,6,6,6,6]
@@ -14650,9 +14650,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm15 = ymm19[1,1,1,1,5,5,5,5]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm15[2],ymm1[3,4],ymm15[5],ymm1[6,7,8,9],ymm15[10],ymm1[11,12],ymm15[13],ymm1[14,15]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm27, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufb %ymm14, %ymm3, %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm15 = ymm4[2,2,2,2,6,6,6,6]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm15 = ymm0[0,1],ymm15[2],ymm0[3,4],ymm15[5],ymm0[6,7,8,9],ymm15[10],ymm0[11,12],ymm15[13],ymm0[14,15]
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm3[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
@@ -14687,16 +14687,16 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm7 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm14 = ymm6[3,3,3,3,7,7,7,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm14[2],ymm7[3,4],ymm14[5],ymm7[6,7,8,9],ymm14[10],ymm7[11,12],ymm14[13],ymm7[14,15]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} ymm21 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15,0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm6
; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm14
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm27 = [2,2,2,3,8,8,8,9]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm14, %zmm27, %zmm7
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm23 & (zmm0 ^ zmm1))
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vprold $16, %ymm6, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm14 = ymm2[1,2,2,3,5,6,6,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm14[0,1],ymm1[2],ymm14[3,4],ymm1[5],ymm14[6,7,8,9],ymm1[10],ymm14[11,12],ymm1[13],ymm14[14,15]
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm20, %ymm8
@@ -14752,7 +14752,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q %zmm9, %zmm25, %zmm10
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm10 = zmm5 ^ (zmm24 & (zmm10 ^ zmm5))
; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm21, %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm4, %xmm3
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm9 = ymm2[3,3,3,3,7,7,7,7]
@@ -14781,7 +14781,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload
; AVX512DQ-FCP-NEXT: # zmm0 = (zmm0 & zmm17) | mem
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm0 ^ (zmm9 & (zmm5 ^ zmm0))
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
; AVX512DQ-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm1 = mem[0,1,1,3,4,5,5,7]
@@ -14800,24 +14800,24 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm9, %ymm9
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm9, %zmm9
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm4[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
; AVX512DQ-FCP-NEXT: vpshufd $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm12 = mem[0,1,1,3,4,5,5,7]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm12[0,1],ymm11[2],ymm12[3,4],ymm11[5],ymm12[6,7,8,9],ymm11[10],ymm12[11,12],ymm11[13],ymm12[14,15]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm12
; AVX512DQ-FCP-NEXT: vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: # ymm13 = mem[1,1,1,1,5,5,5,5]
; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7,8,9],ymm13[10],ymm12[11,12],ymm13[13],ymm12[14,15]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,4,0,0,0,5,0,0]
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm13, %ymm13
; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm14 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
; AVX512DQ-FCP-NEXT: vpandn %ymm13, %ymm14, %ymm13
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm4[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17,u,u,u,u],zero,zero
; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm13, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero
; AVX512DQ-FCP-NEXT: vprold $16, %ymm4, %ymm21
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm21 = ymm21[2,2,2,2]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
index 1eef9e25a4dbe..8e2c5fc17319c 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
@@ -953,7 +953,7 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vmovdqa64 %zmm0, (%r9)
; AVX512-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
-; AVX512-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -978,7 +978,7 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%r9)
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
-; AVX512-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -1003,7 +1003,7 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -1028,7 +1028,7 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%r9)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
@@ -1053,7 +1053,7 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%r9)
; AVX512BW-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
-; AVX512BW-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512BW-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -1078,7 +1078,7 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%r9)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
-; AVX512BW-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512BW-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
@@ -1103,7 +1103,7 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%r9)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -1128,7 +1128,7 @@ define void @store_i32_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%r9)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm1[2],ymm3[3,4,5,6],ymm1[7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%in.vec0 = load <8 x i32>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
index a8d7d5040bf51..9ded82018bc72 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
@@ -9771,7 +9771,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermi2d %zmm13, %zmm15, %zmm21
; AVX512-NEXT: vpermt2d %zmm13, %zmm0, %zmm15
-; AVX512-NEXT: vmovdqa 128(%rdx), %ymm0
+; AVX512-NEXT: vmovdqa64 128(%rdx), %ymm0
; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [3,11,0,8,7,15,4,12]
; AVX512-NEXT: vpermt2d (%rcx), %ymm2, %ymm14
; AVX512-NEXT: movb $36, %al
@@ -9786,7 +9786,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2d %zmm4, %zmm29, %zmm14
; AVX512-NEXT: vpermt2d 128(%rcx), %ymm2, %ymm0
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm14 {%k1} = zmm0[0,1,0,1,2,3,6,7]
-; AVX512-NEXT: vmovdqa 192(%rdx), %ymm0
+; AVX512-NEXT: vmovdqa64 192(%rdx), %ymm0
; AVX512-NEXT: vpermt2d 192(%rcx), %ymm2, %ymm0
; AVX512-NEXT: vmovdqa64 192(%rdi), %zmm12
; AVX512-NEXT: vmovdqa64 192(%rsi), %zmm7
@@ -9876,10 +9876,10 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqa 64(%rdi), %ymm1
; AVX512-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm22 {%k1} = zmm1[2,3,2,3,2,3,2,3]
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm18 {%k1} = zmm1[2,3,2,3,2,3,2,3]
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm21 {%k1} = zmm1[2,3,2,3,2,3,2,3]
; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,1,21,0,4,5,6,7,22,0,10,11,12,13,23,0]
@@ -10338,7 +10338,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermi2d %zmm13, %zmm15, %zmm21
; AVX512DQ-NEXT: vpermt2d %zmm13, %zmm0, %zmm15
-; AVX512DQ-NEXT: vmovdqa 128(%rdx), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 128(%rdx), %ymm0
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [3,11,0,8,7,15,4,12]
; AVX512DQ-NEXT: vpermt2d (%rcx), %ymm2, %ymm14
; AVX512DQ-NEXT: movb $36, %al
@@ -10353,7 +10353,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2d %zmm4, %zmm29, %zmm14
; AVX512DQ-NEXT: vpermt2d 128(%rcx), %ymm2, %ymm0
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm14 {%k1} = zmm0[0,1,0,1,2,3,6,7]
-; AVX512DQ-NEXT: vmovdqa 192(%rdx), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 192(%rdx), %ymm0
; AVX512DQ-NEXT: vpermt2d 192(%rcx), %ymm2, %ymm0
; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %zmm12
; AVX512DQ-NEXT: vmovdqa64 192(%rsi), %zmm7
@@ -10443,10 +10443,10 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm1
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm22 {%k1} = zmm1[2,3,2,3,2,3,2,3]
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm18 {%k1} = zmm1[2,3,2,3,2,3,2,3]
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm21 {%k1} = zmm1[2,3,2,3,2,3,2,3]
; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,1,21,0,4,5,6,7,22,0,10,11,12,13,23,0]
@@ -10905,7 +10905,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpermi2d %zmm13, %zmm15, %zmm21
; AVX512BW-NEXT: vpermt2d %zmm13, %zmm0, %zmm15
-; AVX512BW-NEXT: vmovdqa 128(%rdx), %ymm0
+; AVX512BW-NEXT: vmovdqa64 128(%rdx), %ymm0
; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm2 = [3,11,0,8,7,15,4,12]
; AVX512BW-NEXT: vpermt2d (%rcx), %ymm2, %ymm14
; AVX512BW-NEXT: movb $36, %al
@@ -10920,7 +10920,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2d %zmm4, %zmm29, %zmm14
; AVX512BW-NEXT: vpermt2d 128(%rcx), %ymm2, %ymm0
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm14 {%k1} = zmm0[0,1,0,1,2,3,6,7]
-; AVX512BW-NEXT: vmovdqa 192(%rdx), %ymm0
+; AVX512BW-NEXT: vmovdqa64 192(%rdx), %ymm0
; AVX512BW-NEXT: vpermt2d 192(%rcx), %ymm2, %ymm0
; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm12
; AVX512BW-NEXT: vmovdqa64 192(%rsi), %zmm7
@@ -11010,10 +11010,10 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vmovdqa 64(%rdi), %ymm1
; AVX512BW-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm22 {%k1} = zmm1[2,3,2,3,2,3,2,3]
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512BW-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm18 {%k1} = zmm1[2,3,2,3,2,3,2,3]
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512BW-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm21 {%k1} = zmm1[2,3,2,3,2,3,2,3]
; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,1,21,0,4,5,6,7,22,0,10,11,12,13,23,0]
@@ -11472,7 +11472,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermi2d %zmm13, %zmm15, %zmm21
; AVX512DQ-BW-NEXT: vpermt2d %zmm13, %zmm0, %zmm15
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdx), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdx), %ymm0
; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm2 = [3,11,0,8,7,15,4,12]
; AVX512DQ-BW-NEXT: vpermt2d (%rcx), %ymm2, %ymm14
; AVX512DQ-BW-NEXT: movb $36, %al
@@ -11487,7 +11487,7 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2d %zmm4, %zmm29, %zmm14
; AVX512DQ-BW-NEXT: vpermt2d 128(%rcx), %ymm2, %ymm0
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm14 {%k1} = zmm0[0,1,0,1,2,3,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdx), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdx), %ymm0
; AVX512DQ-BW-NEXT: vpermt2d 192(%rcx), %ymm2, %ymm0
; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm12
; AVX512DQ-BW-NEXT: vmovdqa64 192(%rsi), %zmm7
@@ -11577,10 +11577,10 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vmovdqa 64(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm22 {%k1} = zmm1[2,3,2,3,2,3,2,3]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm18 {%k1} = zmm1[2,3,2,3,2,3,2,3]
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpunpckhdq {{.*#+}} ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm21 {%k1} = zmm1[2,3,2,3,2,3,2,3]
; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,1,21,0,4,5,6,7,22,0,10,11,12,13,23,0]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
index 18f34154b7578..0225c72d1e1c0 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
@@ -1437,7 +1437,7 @@ define void @store_i32_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vmovdqa64 %zmm8, 64(%rax)
; AVX512-NEXT: vmovdqa64 %zmm7, (%rax)
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX512-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -1481,7 +1481,7 @@ define void @store_i32_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vmovdqa64 %zmm8, 64(%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm7, (%rax)
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX512-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -1525,7 +1525,7 @@ define void @store_i32_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vmovdqa64 %zmm8, 64(%rax)
; AVX512DQ-NEXT: vmovdqa64 %zmm7, (%rax)
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX512DQ-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -1569,7 +1569,7 @@ define void @store_i32_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, 64(%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm7, (%rax)
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
@@ -1613,7 +1613,7 @@ define void @store_i32_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: vmovdqa64 %zmm8, 64(%rax)
; AVX512BW-NEXT: vmovdqa64 %zmm7, (%rax)
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX512BW-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512BW-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -1657,7 +1657,7 @@ define void @store_i32_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, 64(%rax)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, (%rax)
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX512BW-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512BW-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
@@ -1701,7 +1701,7 @@ define void @store_i32_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, 64(%rax)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, (%rax)
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX512DQ-BW-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -1745,7 +1745,7 @@ define void @store_i32_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, 64(%rax)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, (%rax)
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4],ymm1[5,6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%in.vec0 = load <8 x i32>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll
index f41123c5c3cfd..13c93bc3c20fa 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll
@@ -466,7 +466,7 @@ define void @store_i64_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -490,7 +490,7 @@ define void @store_i64_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -514,7 +514,7 @@ define void @store_i64_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -538,7 +538,7 @@ define void @store_i64_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
@@ -562,7 +562,7 @@ define void @store_i64_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512BW-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512BW-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -586,7 +586,7 @@ define void @store_i64_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512BW-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
@@ -610,7 +610,7 @@ define void @store_i64_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -634,7 +634,7 @@ define void @store_i64_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, 64(%r9)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%in.vec0 = load <4 x i64>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
index 816c487f192ad..e79954705b0b6 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
@@ -6535,12 +6535,12 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k1}
-; AVX512-NEXT: vmovdqa 128(%rdx), %xmm5
+; AVX512-NEXT: vmovdqa64 128(%rdx), %xmm5
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm16 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rdx), %xmm5
+; AVX512-NEXT: vmovdqa64 192(%rdx), %xmm5
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm19 {%k1}
@@ -6567,13 +6567,13 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm24, %zmm0
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm9 {%k1}
; AVX512-NEXT: vpermt2q %zmm1, %zmm19, %zmm0
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512-NEXT: vinserti64x4 $0, %ymm1, %zmm7, %zmm1
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm2, %zmm28 {%k1}
; AVX512-NEXT: vpermt2q %zmm2, %zmm19, %zmm1
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm21, %zmm2
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
@@ -6707,10 +6707,10 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm10
-; AVX512-FCP-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm7
-; AVX512-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm20
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,0,4,12]
@@ -6850,7 +6850,7 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm25, %zmm14, %zmm25
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm16, %zmm25
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm3, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -7090,12 +7090,12 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k1}
-; AVX512DQ-NEXT: vmovdqa 128(%rdx), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 128(%rdx), %xmm5
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm16 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rdx), %xmm5
+; AVX512DQ-NEXT: vmovdqa64 192(%rdx), %xmm5
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm19 {%k1}
@@ -7122,13 +7122,13 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm24, %zmm0
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm9 {%k1}
; AVX512DQ-NEXT: vpermt2q %zmm1, %zmm19, %zmm0
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm1, %zmm7, %zmm1
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm2, %zmm28 {%k1}
; AVX512DQ-NEXT: vpermt2q %zmm2, %zmm19, %zmm1
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm21, %zmm2
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
@@ -7262,10 +7262,10 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm10
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm7
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm20
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,0,4,12]
@@ -7405,7 +7405,7 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm25, %zmm14, %zmm25
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm16, %zmm25
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm3, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -7645,12 +7645,12 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k1}
-; AVX512BW-NEXT: vmovdqa 128(%rdx), %xmm5
+; AVX512BW-NEXT: vmovdqa64 128(%rdx), %xmm5
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm16 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rdx), %xmm5
+; AVX512BW-NEXT: vmovdqa64 192(%rdx), %xmm5
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512BW-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm19 {%k1}
@@ -7677,13 +7677,13 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm24, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm9 {%k1}
; AVX512BW-NEXT: vpermt2q %zmm1, %zmm19, %zmm0
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm1, %zmm7, %zmm1
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm28 {%k1}
; AVX512BW-NEXT: vpermt2q %zmm2, %zmm19, %zmm1
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm21, %zmm2
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
@@ -7817,10 +7817,10 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm10
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm7
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm20
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,0,4,12]
@@ -7960,7 +7960,7 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm25, %zmm14, %zmm25
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm16, %zmm25
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm3, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -8200,12 +8200,12 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdx), %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdx), %xmm5
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm16 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdx), %xmm5
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdx), %xmm5
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm19 {%k1}
@@ -8232,13 +8232,13 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm24, %zmm0
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm9 {%k1}
; AVX512DQ-BW-NEXT: vpermt2q %zmm1, %zmm19, %zmm0
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm1, %zmm7, %zmm1
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm28 {%k1}
; AVX512DQ-BW-NEXT: vpermt2q %zmm2, %zmm19, %zmm1
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm21, %zmm2
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
@@ -8372,10 +8372,10 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm10
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm7
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm4, %zmm20
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,0,4,12]
@@ -8515,7 +8515,7 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm25, %zmm14, %zmm25
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm29 {%k1}
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm16, %zmm25
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm3, %zmm1
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
@@ -13756,42 +13756,42 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpmovsxbq {{.*#+}} zmm1 = [10,0,2,3,4,5,11,0]
; AVX512-NEXT: vpermt2q %zmm2, %zmm1, %zmm5
; AVX512-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm5, %zmm31
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm21 {%k2}
; AVX512-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm0, %zmm1, %zmm31
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm30
; AVX512-NEXT: vmovdqa64 %zmm4, %zmm16 {%k2}
; AVX512-NEXT: vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm4, %zmm1, %zmm30
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm2
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm26
; AVX512-NEXT: vmovdqa64 %zmm6, %zmm17 {%k2}
; AVX512-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm6, %zmm1, %zmm26
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm2
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm25
; AVX512-NEXT: vmovdqa64 %zmm7, %zmm19 {%k2}
; AVX512-NEXT: vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm7, %zmm1, %zmm25
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm2
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm19
; AVX512-NEXT: vmovdqa64 %zmm9, %zmm18 {%k2}
; AVX512-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-NEXT: vpermt2q %zmm9, %zmm1, %zmm19
-; AVX512-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm16
@@ -13885,32 +13885,32 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm14 {%k1}
-; AVX512-NEXT: vmovdqa 128(%rdx), %xmm6
+; AVX512-NEXT: vmovdqa64 128(%rdx), %xmm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm15 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rdx), %xmm6
+; AVX512-NEXT: vmovdqa64 192(%rdx), %xmm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k1}
-; AVX512-NEXT: vmovdqa 256(%rdx), %xmm6
+; AVX512-NEXT: vmovdqa64 256(%rdx), %xmm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm12 {%k1}
-; AVX512-NEXT: vmovdqa 320(%rdx), %xmm6
+; AVX512-NEXT: vmovdqa64 320(%rdx), %xmm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm2 {%k1}
-; AVX512-NEXT: vmovdqa 384(%rdx), %xmm6
+; AVX512-NEXT: vmovdqa64 384(%rdx), %xmm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm9 {%k1}
-; AVX512-NEXT: vmovdqa 448(%rdx), %xmm6
+; AVX512-NEXT: vmovdqa64 448(%rdx), %xmm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -14223,27 +14223,27 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm17
; AVX512-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 128(%rdx), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdx), %xmm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm18
; AVX512-FCP-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 192(%rdx), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdx), %xmm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm21
; AVX512-FCP-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 256(%rdx), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdx), %xmm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm26
; AVX512-FCP-NEXT: vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 320(%rdx), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdx), %xmm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm25
; AVX512-FCP-NEXT: vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 384(%rdx), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdx), %xmm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm6
; AVX512-FCP-NEXT: vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 448(%rdx), %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdx), %xmm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm4
; AVX512-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -14484,7 +14484,7 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm13 = [10,0,2,3,4,5,11,0]
; AVX512-FCP-NEXT: vpermt2q %zmm2, %zmm13, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
@@ -14492,35 +14492,35 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm4, %zmm13, %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm22
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm20 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm5, %zmm13, %zmm22
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm1
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm26
; AVX512-FCP-NEXT: vmovdqa64 %zmm6, %zmm21 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm6, %zmm13, %zmm26
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm1
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm29
; AVX512-FCP-NEXT: vmovdqa64 %zmm7, %zmm24 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm7, %zmm13, %zmm29
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm0
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm30
; AVX512-FCP-NEXT: vmovdqa64 %zmm8, %zmm25 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %zmm8, %zmm13, %zmm30
-; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %ymm0
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm20
@@ -15145,42 +15145,42 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [10,0,2,3,4,5,11,0]
; AVX512DQ-NEXT: vpermt2q %zmm2, %zmm1, %zmm5
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm5, %zmm31
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm21 {%k2}
; AVX512DQ-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm0, %zmm1, %zmm31
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm30
; AVX512DQ-NEXT: vmovdqa64 %zmm4, %zmm16 {%k2}
; AVX512DQ-NEXT: vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm4, %zmm1, %zmm30
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm2
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm26
; AVX512DQ-NEXT: vmovdqa64 %zmm6, %zmm17 {%k2}
; AVX512DQ-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm6, %zmm1, %zmm26
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm2
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm25
; AVX512DQ-NEXT: vmovdqa64 %zmm7, %zmm19 {%k2}
; AVX512DQ-NEXT: vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm7, %zmm1, %zmm25
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm2
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm19
; AVX512DQ-NEXT: vmovdqa64 %zmm9, %zmm18 {%k2}
; AVX512DQ-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-NEXT: vpermt2q %zmm9, %zmm1, %zmm19
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm16
@@ -15274,32 +15274,32 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm14 {%k1}
-; AVX512DQ-NEXT: vmovdqa 128(%rdx), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rdx), %xmm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm15 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rdx), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 192(%rdx), %xmm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k1}
-; AVX512DQ-NEXT: vmovdqa 256(%rdx), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 256(%rdx), %xmm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm12 {%k1}
-; AVX512DQ-NEXT: vmovdqa 320(%rdx), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 320(%rdx), %xmm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm2 {%k1}
-; AVX512DQ-NEXT: vmovdqa 384(%rdx), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 384(%rdx), %xmm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm9 {%k1}
-; AVX512DQ-NEXT: vmovdqa 448(%rdx), %xmm6
+; AVX512DQ-NEXT: vmovdqa64 448(%rdx), %xmm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -15612,27 +15612,27 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm17
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdx), %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm18
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdx), %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm21
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdx), %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm26
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdx), %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm25
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdx), %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm6
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdx), %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -15873,7 +15873,7 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm13 = [10,0,2,3,4,5,11,0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm2, %zmm13, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
@@ -15881,35 +15881,35 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm4, %zmm13, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm22
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm20 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm5, %zmm13, %zmm22
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm26
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, %zmm21 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm6, %zmm13, %zmm26
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm1
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm29
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm7, %zmm24 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm7, %zmm13, %zmm29
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm30
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, %zmm25 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm8, %zmm13, %zmm30
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm20
@@ -16534,42 +16534,42 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [10,0,2,3,4,5,11,0]
; AVX512BW-NEXT: vpermt2q %zmm2, %zmm1, %zmm5
; AVX512BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm5, %zmm31
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm21 {%k2}
; AVX512BW-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm31
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm30
; AVX512BW-NEXT: vmovdqa64 %zmm4, %zmm16 {%k2}
; AVX512BW-NEXT: vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpermt2q %zmm4, %zmm1, %zmm30
-; AVX512BW-NEXT: vmovdqa 256(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 256(%rdi), %ymm2
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm26
; AVX512BW-NEXT: vmovdqa64 %zmm6, %zmm17 {%k2}
; AVX512BW-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpermt2q %zmm6, %zmm1, %zmm26
-; AVX512BW-NEXT: vmovdqa 320(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 320(%rdi), %ymm2
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm25
; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm19 {%k2}
; AVX512BW-NEXT: vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpermt2q %zmm7, %zmm1, %zmm25
-; AVX512BW-NEXT: vmovdqa 384(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 384(%rdi), %ymm2
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm19
; AVX512BW-NEXT: vmovdqa64 %zmm9, %zmm18 {%k2}
; AVX512BW-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-NEXT: vpermt2q %zmm9, %zmm1, %zmm19
-; AVX512BW-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512BW-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm16
@@ -16663,32 +16663,32 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm14 {%k1}
-; AVX512BW-NEXT: vmovdqa 128(%rdx), %xmm6
+; AVX512BW-NEXT: vmovdqa64 128(%rdx), %xmm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm15 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rdx), %xmm6
+; AVX512BW-NEXT: vmovdqa64 192(%rdx), %xmm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k1}
-; AVX512BW-NEXT: vmovdqa 256(%rdx), %xmm6
+; AVX512BW-NEXT: vmovdqa64 256(%rdx), %xmm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm12 {%k1}
-; AVX512BW-NEXT: vmovdqa 320(%rdx), %xmm6
+; AVX512BW-NEXT: vmovdqa64 320(%rdx), %xmm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT: vmovdqa 384(%rdx), %xmm6
+; AVX512BW-NEXT: vmovdqa64 384(%rdx), %xmm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm9 {%k1}
-; AVX512BW-NEXT: vmovdqa 448(%rdx), %xmm6
+; AVX512BW-NEXT: vmovdqa64 448(%rdx), %xmm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -17001,27 +17001,27 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm17
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdx), %xmm1
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdx), %xmm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm18
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdx), %xmm1
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdx), %xmm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm21
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rdx), %xmm1
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdx), %xmm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm26
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rdx), %xmm1
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdx), %xmm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm25
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rdx), %xmm1
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdx), %xmm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm6
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 448(%rdx), %xmm1
+; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdx), %xmm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm4
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17262,7 +17262,7 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm13 = [10,0,2,3,4,5,11,0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm2, %zmm13, %zmm0
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
@@ -17270,35 +17270,35 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm4, %zmm13, %zmm0
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm22
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm20 {%k1}
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm5, %zmm13, %zmm22
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm26
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, %zmm21 {%k1}
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm6, %zmm13, %zmm26
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdi), %ymm1
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm29
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm24 {%k1}
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm7, %zmm13, %zmm29
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm30
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm25 {%k1}
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %zmm8, %zmm13, %zmm30
-; AVX512BW-FCP-NEXT: vmovdqa 448(%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdi), %ymm0
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm20
@@ -17923,42 +17923,42 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [10,0,2,3,4,5,11,0]
; AVX512DQ-BW-NEXT: vpermt2q %zmm2, %zmm1, %zmm5
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm5, %zmm31
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm21 {%k2}
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm31
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm30
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, %zmm16 {%k2}
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermt2q %zmm4, %zmm1, %zmm30
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm26
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, %zmm17 {%k2}
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermt2q %zmm6, %zmm1, %zmm26
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm25
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm19 {%k2}
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermt2q %zmm7, %zmm1, %zmm25
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm19
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, %zmm18 {%k2}
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-NEXT: vpermt2q %zmm9, %zmm1, %zmm19
-; AVX512DQ-BW-NEXT: vmovdqa 448(%rdi), %ymm2
+; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdi), %ymm2
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm16
@@ -18052,32 +18052,32 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm14 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdx), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdx), %xmm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm15 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdx), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdx), %xmm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rdx), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdx), %xmm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm12 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rdx), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdx), %xmm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdx), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdx), %xmm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm9 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 448(%rdx), %xmm6
+; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdx), %xmm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -18390,27 +18390,27 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm17
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdx), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdx), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm18
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdx), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdx), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm21
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rdx), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdx), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm26
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rdx), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdx), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm25
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rdx), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdx), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm6
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rdx), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdx), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm3, %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18651,7 +18651,7 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm13 = [10,0,2,3,4,5,11,0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm2, %zmm13, %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm0
@@ -18659,35 +18659,35 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm4, %zmm13, %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm22
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm20 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm5, %zmm13, %zmm22
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm26
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, %zmm21 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm6, %zmm13, %zmm26
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rdi), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdi), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm1, %zmm0, %zmm29
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm24 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm7, %zmm13, %zmm29
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm30
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm25 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm8, %zmm13, %zmm30
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdi), %ymm0
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm1, %zmm20
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-7.ll
index 18103e958b532..628f08b37e301 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-7.ll
@@ -655,7 +655,7 @@ define void @store_i64_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vmovdqa64 %zmm7, 64(%rax)
; AVX512-NEXT: vmovdqa64 %zmm6, (%rax)
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -700,7 +700,7 @@ define void @store_i64_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vmovdqa64 %zmm7, 64(%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm6, (%rax)
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -745,7 +745,7 @@ define void @store_i64_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vmovdqa64 %zmm7, 64(%rax)
; AVX512DQ-NEXT: vmovdqa64 %zmm6, (%rax)
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -790,7 +790,7 @@ define void @store_i64_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm7, 64(%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, (%rax)
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
@@ -835,7 +835,7 @@ define void @store_i64_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: vmovdqa64 %zmm7, 64(%rax)
; AVX512BW-NEXT: vmovdqa64 %zmm6, (%rax)
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512BW-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -880,7 +880,7 @@ define void @store_i64_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, 64(%rax)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, (%rax)
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512BW-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
@@ -925,7 +925,7 @@ define void @store_i64_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, 64(%rax)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, (%rax)
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-BW-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -970,7 +970,7 @@ define void @store_i64_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, 64(%rax)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, (%rax)
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, 192(%rax)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm0, 192(%rax)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%in.vec0 = load <4 x i64>, ptr %in.vecptr0, align 64
@@ -8087,9 +8087,9 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2q %zmm28, %zmm21, %zmm5
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm5 {%k1}
; AVX512-NEXT: vmovdqa64 128(%rax), %zmm30
-; AVX512-NEXT: vmovdqa 128(%r9), %ymm13
+; AVX512-NEXT: vmovdqa64 128(%r9), %ymm13
; AVX512-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 128(%r8), %ymm1
+; AVX512-NEXT: vmovdqa64 128(%r8), %ymm1
; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm13[0],ymm1[2],ymm13[2]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm30[2,3,2,3]
@@ -8238,11 +8238,11 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm23 {%k4}
-; AVX512-NEXT: vmovdqa 128(%rdx), %xmm8
+; AVX512-NEXT: vmovdqa64 128(%rdx), %xmm8
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm20 {%k4}
-; AVX512-NEXT: vmovdqa 192(%rdx), %xmm8
+; AVX512-NEXT: vmovdqa64 192(%rdx), %xmm8
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm13 {%k4}
@@ -8337,7 +8337,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpmovsxbq {{.*#+}} zmm0 = [0,1,11,0,4,5,6,7]
; AVX512-NEXT: vpermi2q %zmm1, %zmm9, %zmm0
; AVX512-NEXT: vmovdqa64 %zmm7, %zmm10 {%k1}
-; AVX512-NEXT: vmovdqa 192(%r8), %ymm7
+; AVX512-NEXT: vmovdqa64 192(%r8), %ymm7
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm7[0],mem[0],ymm7[2],mem[2]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm7[2,3,2,3],zmm6[2,3,2,3]
; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
@@ -8437,7 +8437,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q %zmm26, %zmm5, %zmm4
; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm0
; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %xmm1
-; AVX512-FCP-NEXT: vmovdqa 128(%rdx), %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdx), %xmm2
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,1,8,9,4,5,6,7]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm3, %zmm4
@@ -8453,7 +8453,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm3, %zmm1
; AVX512-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermi2q %zmm15, %zmm11, %zmm5
-; AVX512-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm3, %zmm5
; AVX512-FCP-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -8477,10 +8477,10 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm4
; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm3
; AVX512-FCP-NEXT: vmovdqa 64(%r9), %ymm2
-; AVX512-FCP-NEXT: vmovdqa 128(%r9), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 128(%r9), %ymm0
; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm12
; AVX512-FCP-NEXT: vmovdqa 64(%r8), %ymm14
-; AVX512-FCP-NEXT: vmovdqa 128(%r8), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 128(%r8), %ymm5
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm12[0],ymm3[0],ymm12[2],ymm3[2]
; AVX512-FCP-NEXT: movb $28, %sil
; AVX512-FCP-NEXT: kmovw %esi, %k2
@@ -8789,7 +8789,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm11 {%k3}
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm10 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 192(%r8), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 192(%r8), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2]
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm1[2,3,2,3],zmm3[2,3,2,3]
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,12,0,3,4,5,6,7]
@@ -8958,9 +8958,9 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2q %zmm31, %zmm15, %zmm0
; AVX512DQ-NEXT: vmovdqa64 %zmm5, %zmm0 {%k1}
; AVX512DQ-NEXT: vmovdqa64 128(%rax), %zmm15
-; AVX512DQ-NEXT: vmovdqa 128(%r9), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 128(%r9), %ymm14
; AVX512DQ-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 128(%r8), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 128(%r8), %ymm5
; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm5[0],ymm14[0],ymm5[2],ymm14[2]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm5[2,3,2,3],zmm15[2,3,2,3]
@@ -9153,11 +9153,11 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm23 {%k4}
-; AVX512DQ-NEXT: vmovdqa 128(%rdx), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 128(%rdx), %xmm2
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm18 {%k4}
-; AVX512DQ-NEXT: vmovdqa 192(%rdx), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 192(%rdx), %xmm2
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm13 {%k4}
@@ -9204,7 +9204,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm5 = [0,1,11,0,4,5,6,7]
; AVX512DQ-NEXT: vpermi2q %zmm2, %zmm10, %zmm5
; AVX512DQ-NEXT: vmovdqa64 %zmm8, %zmm12 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%r8), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 192(%r8), %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm6[0],mem[0],ymm6[2],mem[2]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm12 {%k2} = zmm6[2,3,2,3],zmm0[2,3,2,3]
; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
@@ -9320,10 +9320,10 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r9), %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%r9), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%r9), %ymm4
; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm10
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r8), %ymm13
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%r8), %ymm12
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%r8), %ymm12
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm10[0],ymm0[0],ymm10[2],ymm0[2]
; AVX512DQ-FCP-NEXT: movb $28, %r10b
; AVX512DQ-FCP-NEXT: kmovw %r10d, %k2
@@ -9439,14 +9439,14 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm27, %zmm0
; AVX512DQ-FCP-NEXT: vpermt2q %zmm24, %zmm23, %zmm0
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdx), %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm2, %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %zmm15
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rsi), %zmm8
; AVX512DQ-FCP-NEXT: vpermi2q %zmm8, %zmm15, %zmm23
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdx), %xmm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm2, %zmm23
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdx), %zmm9
@@ -9653,7 +9653,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, %zmm13 {%k4}
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm10 {%k3}
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm15, %zmm9 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%r8), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%r8), %ymm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2]
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm0[2,3,2,3],zmm2[2,3,2,3]
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm0 = [0,12,0,3,4,5,6,7]
@@ -9820,9 +9820,9 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2q %zmm28, %zmm21, %zmm5
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm5 {%k1}
; AVX512BW-NEXT: vmovdqa64 128(%rax), %zmm30
-; AVX512BW-NEXT: vmovdqa 128(%r9), %ymm13
+; AVX512BW-NEXT: vmovdqa64 128(%r9), %ymm13
; AVX512BW-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 128(%r8), %ymm1
+; AVX512BW-NEXT: vmovdqa64 128(%r8), %ymm1
; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm13[0],ymm1[2],ymm13[2]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm30[2,3,2,3]
@@ -9971,11 +9971,11 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm23 {%k4}
-; AVX512BW-NEXT: vmovdqa 128(%rdx), %xmm8
+; AVX512BW-NEXT: vmovdqa64 128(%rdx), %xmm8
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm20 {%k4}
-; AVX512BW-NEXT: vmovdqa 192(%rdx), %xmm8
+; AVX512BW-NEXT: vmovdqa64 192(%rdx), %xmm8
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm8
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm13 {%k4}
@@ -10070,7 +10070,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm0 = [0,1,11,0,4,5,6,7]
; AVX512BW-NEXT: vpermi2q %zmm1, %zmm9, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm10 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%r8), %ymm7
+; AVX512BW-NEXT: vmovdqa64 192(%r8), %ymm7
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm7[0],mem[0],ymm7[2],mem[2]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm7[2,3,2,3],zmm6[2,3,2,3]
; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
@@ -10170,7 +10170,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vpermt2q %zmm26, %zmm5, %zmm4
; AVX512BW-FCP-NEXT: vmovdqa (%rdx), %xmm0
; AVX512BW-FCP-NEXT: vmovdqa 64(%rdx), %xmm1
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdx), %xmm2
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdx), %xmm2
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,1,8,9,4,5,6,7]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm3, %zmm4
@@ -10186,7 +10186,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm3, %zmm1
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermi2q %zmm15, %zmm11, %zmm5
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm3, %zmm5
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -10210,10 +10210,10 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm4
; AVX512BW-FCP-NEXT: vmovdqa (%r9), %ymm3
; AVX512BW-FCP-NEXT: vmovdqa 64(%r9), %ymm2
-; AVX512BW-FCP-NEXT: vmovdqa 128(%r9), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%r9), %ymm0
; AVX512BW-FCP-NEXT: vmovdqa (%r8), %ymm12
; AVX512BW-FCP-NEXT: vmovdqa 64(%r8), %ymm14
-; AVX512BW-FCP-NEXT: vmovdqa 128(%r8), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%r8), %ymm5
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm12[0],ymm3[0],ymm12[2],ymm3[2]
; AVX512BW-FCP-NEXT: movb $28, %sil
; AVX512BW-FCP-NEXT: kmovd %esi, %k2
@@ -10522,7 +10522,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm11 {%k3}
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm10 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 192(%r8), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%r8), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2]
; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm1[2,3,2,3],zmm3[2,3,2,3]
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,12,0,3,4,5,6,7]
@@ -10691,9 +10691,9 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2q %zmm31, %zmm15, %zmm0
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm0 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa64 128(%rax), %zmm15
-; AVX512DQ-BW-NEXT: vmovdqa 128(%r9), %ymm14
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%r9), %ymm14
; AVX512DQ-BW-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 128(%r8), %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%r8), %ymm5
; AVX512DQ-BW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm5[0],ymm14[0],ymm5[2],ymm14[2]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm5[2,3,2,3],zmm15[2,3,2,3]
@@ -10886,11 +10886,11 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm23 {%k4}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdx), %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdx), %xmm2
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm18 {%k4}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdx), %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdx), %xmm2
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm13 {%k4}
@@ -10937,7 +10937,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} zmm5 = [0,1,11,0,4,5,6,7]
; AVX512DQ-BW-NEXT: vpermi2q %zmm2, %zmm10, %zmm5
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, %zmm12 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%r8), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%r8), %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm6[0],mem[0],ymm6[2],mem[2]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm12 {%k2} = zmm6[2,3,2,3],zmm0[2,3,2,3]
; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
@@ -11053,10 +11053,10 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm5
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%r9), %ymm0
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%r9), %ymm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%r9), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%r9), %ymm4
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%r8), %ymm10
; AVX512DQ-BW-FCP-NEXT: vmovdqa 64(%r8), %ymm13
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%r8), %ymm12
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%r8), %ymm12
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm10[0],ymm0[0],ymm10[2],ymm0[2]
; AVX512DQ-BW-FCP-NEXT: movb $28, %r10b
; AVX512DQ-BW-FCP-NEXT: kmovd %r10d, %k2
@@ -11172,14 +11172,14 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm27, %zmm0
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm24, %zmm23, %zmm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdx), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdx), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm2, %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm15
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rsi), %zmm8
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm8, %zmm15, %zmm23
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdx), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdx), %xmm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm2, %zmm23
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdx), %zmm9
@@ -11386,7 +11386,7 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm13 {%k4}
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm10 {%k3}
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm15, %zmm9 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%r8), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%r8), %ymm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2]
; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm0[2,3,2,3],zmm2[2,3,2,3]
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm0 = [0,12,0,3,4,5,6,7]
@@ -16899,13 +16899,13 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2q %zmm4, %zmm31, %zmm2
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm2 {%k1}
; AVX512-NEXT: vmovdqa (%r9), %ymm7
-; AVX512-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa 64(%r9), %ymm3
-; AVX512-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa (%r8), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa 64(%r8), %ymm4
-; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
; AVX512-NEXT: movb $28, %r10b
; AVX512-NEXT: kmovw %r10d, %k2
@@ -16973,10 +16973,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2q %zmm0, %zmm31, %zmm3
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512-NEXT: vmovdqa64 128(%rax), %zmm5
-; AVX512-NEXT: vmovdqa 128(%r9), %ymm4
-; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 128(%r8), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 128(%r9), %ymm4
+; AVX512-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 128(%r8), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17011,10 +17011,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2q %zmm0, %zmm31, %zmm3
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512-NEXT: vmovdqa64 192(%rax), %zmm0
-; AVX512-NEXT: vmovdqa 192(%r9), %ymm5
-; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 192(%r8), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 192(%r9), %ymm5
+; AVX512-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 192(%r8), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[2],ymm5[2]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
; AVX512-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17049,10 +17049,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2q %zmm23, %zmm31, %zmm5
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm5 {%k1}
; AVX512-NEXT: vmovdqa64 256(%rax), %zmm0
-; AVX512-NEXT: vmovdqa 256(%r9), %ymm7
-; AVX512-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 256(%r8), %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 256(%r9), %ymm7
+; AVX512-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 256(%r8), %ymm1
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm7[0],ymm1[2],ymm7[2]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
; AVX512-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17089,10 +17089,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2q %zmm17, %zmm31, %zmm14
; AVX512-NEXT: vmovdqa64 %zmm5, %zmm14 {%k1}
; AVX512-NEXT: vmovdqa64 320(%rax), %zmm1
-; AVX512-NEXT: vmovdqa 320(%r9), %ymm5
-; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 320(%r8), %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 320(%r9), %ymm5
+; AVX512-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 320(%r8), %ymm0
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm5[2,3,2,3],zmm1[2,3,2,3]
; AVX512-NEXT: vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -17415,7 +17415,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2q %zmm0, %zmm1, %zmm17
; AVX512-NEXT: movb $12, %sil
; AVX512-NEXT: kmovw %esi, %k5
-; AVX512-NEXT: vmovdqa 448(%rdx), %xmm2
+; AVX512-NEXT: vmovdqa64 448(%rdx), %xmm2
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm4 {%k5}
@@ -17430,10 +17430,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpermt2q %zmm0, %zmm1, %zmm9
; AVX512-NEXT: vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm25, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa 384(%r9), %ymm12
-; AVX512-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-NEXT: vmovdqa 384(%r8), %ymm8
-; AVX512-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 384(%r9), %ymm12
+; AVX512-NEXT: vmovdqu64 %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqa64 384(%r8), %ymm8
+; AVX512-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[2],ymm12[2]
; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm8[2,3,2,3],zmm0[2,3,2,3]
; AVX512-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
@@ -17508,25 +17508,25 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k5}
-; AVX512-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
-; AVX512-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k5}
-; AVX512-NEXT: vmovdqa 256(%rdx), %xmm0
+; AVX512-NEXT: vmovdqa64 256(%rdx), %xmm0
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
-; AVX512-NEXT: vmovdqa 320(%rdx), %xmm0
+; AVX512-NEXT: vmovdqa64 320(%rdx), %xmm0
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
-; AVX512-NEXT: vmovdqa 384(%rdx), %xmm0
+; AVX512-NEXT: vmovdqa64 384(%rdx), %xmm0
; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
@@ -17670,7 +17670,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 64-byte Folded Reload
; AVX512-NEXT: # zmm7 = zmm10[0,1,2,3],mem[4,5,6,7]
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
@@ -17679,42 +17679,42 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: kmovw %ecx, %k1
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm10 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm18 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm9 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm3 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm2 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
; AVX512-NEXT: # ymm8 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
@@ -17853,9 +17853,9 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q %zmm7, %zmm18, %zmm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa 64(%r9), %ymm15
-; AVX512-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm7
; AVX512-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 64(%r8), %ymm30
@@ -17929,8 +17929,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm18, %zmm12
; AVX512-FCP-NEXT: vmovdqa64 %zmm2, %zmm8 {%k1}
; AVX512-FCP-NEXT: vmovdqa64 128(%rax), %zmm25
-; AVX512-FCP-NEXT: vmovdqa 128(%r9), %ymm0
-; AVX512-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 128(%r9), %ymm0
+; AVX512-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 128(%r8), %ymm22
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm22[0],ymm0[0],ymm22[2],ymm0[2]
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 {%k2} = zmm4[2,3,2,3],zmm25[2,3,2,3]
@@ -17969,7 +17969,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm4, %zmm7 {%k1}
; AVX512-FCP-NEXT: vmovdqa64 192(%rax), %zmm29
; AVX512-FCP-NEXT: vmovdqa64 192(%r9), %ymm25
-; AVX512-FCP-NEXT: vmovdqa 192(%r8), %ymm15
+; AVX512-FCP-NEXT: vmovdqa64 192(%r8), %ymm15
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm25[0],ymm15[2],ymm25[2]
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm5[2,3,2,3],zmm29[2,3,2,3]
; AVX512-FCP-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18006,8 +18006,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm12, %zmm9
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm9 {%k1}
; AVX512-FCP-NEXT: vmovdqa64 256(%rax), %zmm1
-; AVX512-FCP-NEXT: vmovdqa 256(%r9), %ymm14
-; AVX512-FCP-NEXT: vmovdqa 256(%r8), %ymm7
+; AVX512-FCP-NEXT: vmovdqa64 256(%r9), %ymm14
+; AVX512-FCP-NEXT: vmovdqa64 256(%r8), %ymm7
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm7[0],ymm14[0],ymm7[2],ymm14[2]
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm6[2,3,2,3],zmm1[2,3,2,3]
; AVX512-FCP-NEXT: vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18048,8 +18048,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm4, %zmm19
; AVX512-FCP-NEXT: vmovdqa64 %zmm9, %zmm11 {%k1}
; AVX512-FCP-NEXT: vmovdqa64 320(%rax), %zmm1
-; AVX512-FCP-NEXT: vmovdqa 320(%r9), %ymm9
-; AVX512-FCP-NEXT: vmovdqa 320(%r8), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 320(%r9), %ymm9
+; AVX512-FCP-NEXT: vmovdqa64 320(%r8), %ymm2
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm2[0],ymm9[0],ymm2[2],ymm9[2]
; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm11 {%k2} = zmm10[2,3,2,3],zmm1[2,3,2,3]
; AVX512-FCP-NEXT: vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18080,17 +18080,17 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm22 # 32-byte Folded Reload
; AVX512-FCP-NEXT: vmovdqu64 %ymm22, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermt2q %ymm25, %ymm0, %ymm15
-; AVX512-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermt2q %ymm14, %ymm0, %ymm7
-; AVX512-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vpermt2q %ymm9, %ymm0, %ymm2
-; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT: vmovdqa 384(%r9), %ymm2
-; AVX512-FCP-NEXT: vmovdqa 384(%r8), %ymm1
+; AVX512-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqa64 384(%r9), %ymm2
+; AVX512-FCP-NEXT: vmovdqa64 384(%r8), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
; AVX512-FCP-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpermt2q %ymm2, %ymm0, %ymm1
-; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 384(%rdx), %zmm0
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 384(%rcx), %zmm22
@@ -18157,7 +18157,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm30, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
; AVX512-FCP-NEXT: vpermt2q %zmm18, %zmm8, %zmm2
-; AVX512-FCP-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18165,13 +18165,13 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm28, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
; AVX512-FCP-NEXT: vpermt2q %zmm21, %zmm8, %zmm2
-; AVX512-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 %zmm29, %zmm2
; AVX512-FCP-NEXT: vpermt2q %zmm20, %zmm8, %zmm2
-; AVX512-FCP-NEXT: vmovdqa 256(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18179,20 +18179,20 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm12, %zmm2
; AVX512-FCP-NEXT: vmovdqa64 %zmm26, %zmm27
; AVX512-FCP-NEXT: vpermt2q %zmm26, %zmm8, %zmm2
-; AVX512-FCP-NEXT: vmovdqa 320(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 %zmm4, %zmm2
; AVX512-FCP-NEXT: vpermt2q %zmm17, %zmm8, %zmm2
-; AVX512-FCP-NEXT: vmovdqa 384(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %zmm9
; AVX512-FCP-NEXT: vmovdqa64 448(%rsi), %zmm4
; AVX512-FCP-NEXT: vpermi2q %zmm4, %zmm9, %zmm8
-; AVX512-FCP-NEXT: vmovdqa 448(%rdx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdx), %xmm0
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm8
; AVX512-FCP-NEXT: vmovdqa64 448(%rdx), %zmm31
@@ -18597,27 +18597,27 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: kmovw %ecx, %k2
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm16 {%k2}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm14 {%k2}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm13 {%k2}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm12 {%k2}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm4 {%k2}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm9 {%k2}
-; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm19 {%k2}
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -18823,13 +18823,13 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2q %zmm6, %zmm31, %zmm2
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm2 {%k1}
; AVX512DQ-NEXT: vmovdqa (%r9), %ymm7
-; AVX512DQ-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa 64(%r9), %ymm5
-; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa (%r8), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa 64(%r8), %ymm6
-; AVX512DQ-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
; AVX512DQ-NEXT: movb $28, %r10b
; AVX512DQ-NEXT: kmovw %r10d, %k2
@@ -18896,10 +18896,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2q %zmm0, %zmm31, %zmm3
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512DQ-NEXT: vmovdqa64 128(%rax), %zmm8
-; AVX512DQ-NEXT: vmovdqa 128(%r9), %ymm4
-; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 128(%r8), %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 128(%r9), %ymm4
+; AVX512DQ-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 128(%r8), %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm8[2,3,2,3]
; AVX512DQ-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18935,10 +18935,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2q %zmm0, %zmm31, %zmm5
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm5 {%k1}
; AVX512DQ-NEXT: vmovdqa64 192(%rax), %zmm0
-; AVX512DQ-NEXT: vmovdqa 192(%r9), %ymm8
-; AVX512DQ-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 192(%r8), %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 192(%r9), %ymm8
+; AVX512DQ-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 192(%r8), %ymm1
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[2],ymm8[2]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -18973,10 +18973,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2q %zmm0, %zmm31, %zmm1
; AVX512DQ-NEXT: vmovdqa64 %zmm4, %zmm1 {%k1}
; AVX512DQ-NEXT: vmovdqa64 256(%rax), %zmm0
-; AVX512DQ-NEXT: vmovdqa 256(%r9), %ymm12
-; AVX512DQ-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 256(%r8), %ymm4
-; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 256(%r9), %ymm12
+; AVX512DQ-NEXT: vmovdqu64 %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 256(%r8), %ymm4
+; AVX512DQ-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm12[0],ymm4[2],ymm12[2]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm4[2,3,2,3],zmm0[2,3,2,3]
; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -19012,10 +19012,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2q %zmm29, %zmm31, %zmm5
; AVX512DQ-NEXT: vmovdqa64 %zmm7, %zmm5 {%k1}
; AVX512DQ-NEXT: vmovdqa64 320(%rax), %zmm1
-; AVX512DQ-NEXT: vmovdqa 320(%r9), %ymm7
-; AVX512DQ-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 320(%r8), %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 320(%r9), %ymm7
+; AVX512DQ-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 320(%r8), %ymm0
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm7[2,3,2,3],zmm1[2,3,2,3]
; AVX512DQ-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -19337,7 +19337,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2q %zmm0, %zmm1, %zmm13
; AVX512DQ-NEXT: movb $12, %sil
; AVX512DQ-NEXT: kmovw %esi, %k5
-; AVX512DQ-NEXT: vmovdqa 448(%rdx), %xmm2
+; AVX512DQ-NEXT: vmovdqa64 448(%rdx), %xmm2
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm4 {%k5}
@@ -19352,10 +19352,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpermt2q %zmm0, %zmm8, %zmm10
; AVX512DQ-NEXT: vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm25, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqa 384(%r9), %ymm12
-; AVX512DQ-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-NEXT: vmovdqa 384(%r8), %ymm8
-; AVX512DQ-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 384(%r9), %ymm12
+; AVX512DQ-NEXT: vmovdqu64 %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqa64 384(%r8), %ymm8
+; AVX512DQ-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[2],ymm12[2]
; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm8[2,3,2,3],zmm0[2,3,2,3]
; AVX512DQ-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
@@ -19432,25 +19432,25 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
-; AVX512DQ-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k5}
-; AVX512DQ-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k5}
-; AVX512DQ-NEXT: vmovdqa 256(%rdx), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 256(%rdx), %xmm0
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
-; AVX512DQ-NEXT: vmovdqa 320(%rdx), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 320(%rdx), %xmm0
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
-; AVX512DQ-NEXT: vmovdqa 384(%rdx), %xmm0
+; AVX512DQ-NEXT: vmovdqa64 384(%rdx), %xmm0
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
@@ -19588,7 +19588,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: # zmm7 = zmm9[0,1,2,3],mem[4,5,6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
@@ -19597,42 +19597,42 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: kmovw %ecx, %k1
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm9 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm17 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm18 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm3 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm2 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm1 {%k1}
-; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
; AVX512DQ-NEXT: # ymm8 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
@@ -19771,7 +19771,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q %zmm7, %zmm31, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm2 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 64(%r9), %ymm16
; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm5
@@ -19845,8 +19845,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm31, %zmm6
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm6 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rax), %zmm9
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%r9), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%r9), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%r8), %ymm26
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm26[0],ymm0[0],ymm26[2],ymm0[2]
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm1[2,3,2,3],zmm9[2,3,2,3]
@@ -19884,7 +19884,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm6 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rax), %zmm29
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%r9), %ymm22
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%r8), %ymm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%r8), %ymm14
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm22[0],ymm14[2],ymm22[2]
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm1[2,3,2,3],zmm29[2,3,2,3]
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -19922,8 +19922,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm31, %zmm8
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, %zmm8 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rax), %zmm25
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%r9), %ymm13
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%r8), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%r9), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%r8), %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm13[0],ymm6[2],ymm13[2]
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 {%k2} = zmm7[2,3,2,3],zmm25[2,3,2,3]
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -19963,7 +19963,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q %zmm5, %zmm31, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, %zmm2 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rax), %zmm25
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%r9), %ymm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%r9), %ymm10
; AVX512DQ-FCP-NEXT: vmovdqa64 320(%r8), %ymm21
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm21[0],ymm10[0],ymm21[2],ymm10[2]
; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm8[2,3,2,3],zmm25[2,3,2,3]
@@ -19995,17 +19995,17 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm26 # 32-byte Folded Reload
; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm26, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %ymm22, %ymm0, %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %ymm13, %ymm0, %ymm6
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %ymm10, %ymm0, %ymm21
; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm21, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%r9), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%r8), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%r9), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%r8), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %ymm2, %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdx), %zmm0
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rcx), %zmm21
@@ -20072,7 +20072,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm30, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vpermt2q %zmm18, %zmm4, %zmm2
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -20080,32 +20080,32 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm23, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vpermt2q %zmm19, %zmm4, %zmm2
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm29, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vpermt2q %zmm29, %zmm4, %zmm2
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpermt2q %zmm24, %zmm4, %zmm20
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm20
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm2
; AVX512DQ-FCP-NEXT: vpermt2q %zmm14, %zmm4, %zmm2
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %zmm9
; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rsi), %zmm3
; AVX512DQ-FCP-NEXT: vpermi2q %zmm3, %zmm9, %zmm4
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdx), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdx), %zmm5
@@ -20495,25 +20495,25 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: kmovw %ecx, %k2
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm13 {%k2}
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm0 {%k2}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm16 {%k2}
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm31 {%k2}
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm20 {%k2}
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm19 {%k2}
-; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k2}
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -20718,13 +20718,13 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2q %zmm4, %zmm31, %zmm2
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm2 {%k1}
; AVX512BW-NEXT: vmovdqa (%r9), %ymm7
-; AVX512BW-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vmovdqa 64(%r9), %ymm3
-; AVX512BW-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqu64 %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vmovdqa (%r8), %ymm0
-; AVX512BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vmovdqa 64(%r8), %ymm4
-; AVX512BW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
; AVX512BW-NEXT: movb $28, %r10b
; AVX512BW-NEXT: kmovd %r10d, %k2
@@ -20792,10 +20792,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2q %zmm0, %zmm31, %zmm3
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512BW-NEXT: vmovdqa64 128(%rax), %zmm5
-; AVX512BW-NEXT: vmovdqa 128(%r9), %ymm4
-; AVX512BW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 128(%r8), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 128(%r9), %ymm4
+; AVX512BW-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 128(%r8), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
; AVX512BW-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -20830,10 +20830,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2q %zmm0, %zmm31, %zmm3
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512BW-NEXT: vmovdqa64 192(%rax), %zmm0
-; AVX512BW-NEXT: vmovdqa 192(%r9), %ymm5
-; AVX512BW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 192(%r8), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 192(%r9), %ymm5
+; AVX512BW-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 192(%r8), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[2],ymm5[2]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
; AVX512BW-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -20868,10 +20868,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2q %zmm23, %zmm31, %zmm5
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm5 {%k1}
; AVX512BW-NEXT: vmovdqa64 256(%rax), %zmm0
-; AVX512BW-NEXT: vmovdqa 256(%r9), %ymm7
-; AVX512BW-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 256(%r8), %ymm1
-; AVX512BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 256(%r9), %ymm7
+; AVX512BW-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 256(%r8), %ymm1
+; AVX512BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm7[0],ymm1[2],ymm7[2]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
; AVX512BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -20908,10 +20908,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2q %zmm17, %zmm31, %zmm14
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm14 {%k1}
; AVX512BW-NEXT: vmovdqa64 320(%rax), %zmm1
-; AVX512BW-NEXT: vmovdqa 320(%r9), %ymm5
-; AVX512BW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 320(%r8), %ymm0
-; AVX512BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 320(%r9), %ymm5
+; AVX512BW-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 320(%r8), %ymm0
+; AVX512BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm5[2,3,2,3],zmm1[2,3,2,3]
; AVX512BW-NEXT: vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21234,7 +21234,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm17
; AVX512BW-NEXT: movb $12, %sil
; AVX512BW-NEXT: kmovd %esi, %k5
-; AVX512BW-NEXT: vmovdqa 448(%rdx), %xmm2
+; AVX512BW-NEXT: vmovdqa64 448(%rdx), %xmm2
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm4 {%k5}
@@ -21249,10 +21249,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm9
; AVX512BW-NEXT: vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm25, %zmm1 {%k1}
-; AVX512BW-NEXT: vmovdqa 384(%r9), %ymm12
-; AVX512BW-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT: vmovdqa 384(%r8), %ymm8
-; AVX512BW-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 384(%r9), %ymm12
+; AVX512BW-NEXT: vmovdqu64 %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT: vmovdqa64 384(%r8), %ymm8
+; AVX512BW-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[2],ymm12[2]
; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm8[2,3,2,3],zmm0[2,3,2,3]
; AVX512BW-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
@@ -21327,25 +21327,25 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k5}
-; AVX512BW-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512BW-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
-; AVX512BW-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512BW-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k5}
-; AVX512BW-NEXT: vmovdqa 256(%rdx), %xmm0
+; AVX512BW-NEXT: vmovdqa64 256(%rdx), %xmm0
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
-; AVX512BW-NEXT: vmovdqa 320(%rdx), %xmm0
+; AVX512BW-NEXT: vmovdqa64 320(%rdx), %xmm0
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
-; AVX512BW-NEXT: vmovdqa 384(%rdx), %xmm0
+; AVX512BW-NEXT: vmovdqa64 384(%rdx), %xmm0
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
@@ -21489,7 +21489,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 64-byte Folded Reload
; AVX512BW-NEXT: # zmm7 = zmm10[0,1,2,3],mem[4,5,6,7]
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
@@ -21498,42 +21498,42 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: kmovd %ecx, %k1
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm10 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm18 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm9 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm3 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
; AVX512BW-NEXT: # ymm8 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
@@ -21672,9 +21672,9 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vpermt2q %zmm7, %zmm18, %zmm1
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa (%r9), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa 64(%r9), %ymm15
-; AVX512BW-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa (%r8), %ymm7
; AVX512BW-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 64(%r8), %ymm30
@@ -21748,8 +21748,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm18, %zmm12
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm8 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa64 128(%rax), %zmm25
-; AVX512BW-FCP-NEXT: vmovdqa 128(%r9), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%r9), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 128(%r8), %ymm22
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm22[0],ymm0[0],ymm22[2],ymm0[2]
; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 {%k2} = zmm4[2,3,2,3],zmm25[2,3,2,3]
@@ -21788,7 +21788,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm7 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa64 192(%rax), %zmm29
; AVX512BW-FCP-NEXT: vmovdqa64 192(%r9), %ymm25
-; AVX512BW-FCP-NEXT: vmovdqa 192(%r8), %ymm15
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%r8), %ymm15
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm25[0],ymm15[2],ymm25[2]
; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm5[2,3,2,3],zmm29[2,3,2,3]
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21825,8 +21825,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm12, %zmm9
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm9 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa64 256(%rax), %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 256(%r9), %ymm14
-; AVX512BW-FCP-NEXT: vmovdqa 256(%r8), %ymm7
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%r9), %ymm14
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%r8), %ymm7
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm7[0],ymm14[0],ymm7[2],ymm14[2]
; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm6[2,3,2,3],zmm1[2,3,2,3]
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21867,8 +21867,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm19
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, %zmm11 {%k1}
; AVX512BW-FCP-NEXT: vmovdqa64 320(%rax), %zmm1
-; AVX512BW-FCP-NEXT: vmovdqa 320(%r9), %ymm9
-; AVX512BW-FCP-NEXT: vmovdqa 320(%r8), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%r9), %ymm9
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%r8), %ymm2
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm10 = ymm2[0],ymm9[0],ymm2[2],ymm9[2]
; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm11 {%k2} = zmm10[2,3,2,3],zmm1[2,3,2,3]
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21899,17 +21899,17 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm22 # 32-byte Folded Reload
; AVX512BW-FCP-NEXT: vmovdqu64 %ymm22, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm25, %ymm0, %ymm15
-; AVX512BW-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm14, %ymm0, %ymm7
-; AVX512BW-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm9, %ymm0, %ymm2
-; AVX512BW-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-FCP-NEXT: vmovdqa 384(%r9), %ymm2
-; AVX512BW-FCP-NEXT: vmovdqa 384(%r8), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%r9), %ymm2
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%r8), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vpermt2q %ymm2, %ymm0, %ymm1
-; AVX512BW-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdx), %zmm0
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 384(%rcx), %zmm22
@@ -21976,7 +21976,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm30, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
; AVX512BW-FCP-NEXT: vpermt2q %zmm18, %zmm8, %zmm2
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21984,13 +21984,13 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm28, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
; AVX512BW-FCP-NEXT: vpermt2q %zmm21, %zmm8, %zmm2
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm29, %zmm2
; AVX512BW-FCP-NEXT: vpermt2q %zmm20, %zmm8, %zmm2
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -21998,20 +21998,20 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm12, %zmm2
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm26, %zmm27
; AVX512BW-FCP-NEXT: vpermt2q %zmm26, %zmm8, %zmm2
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, %zmm2
; AVX512BW-FCP-NEXT: vpermt2q %zmm17, %zmm8, %zmm2
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdi), %zmm9
; AVX512BW-FCP-NEXT: vmovdqa64 448(%rsi), %zmm4
; AVX512BW-FCP-NEXT: vpermi2q %zmm4, %zmm9, %zmm8
-; AVX512BW-FCP-NEXT: vmovdqa 448(%rdx), %xmm0
+; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdx), %xmm0
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm8
; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdx), %zmm31
@@ -22416,27 +22416,27 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: kmovd %ecx, %k2
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm16 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm14 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm13 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm12 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm4 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm9 {%k2}
-; AVX512BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512BW-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm5, %zmm0, %zmm19 {%k2}
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -22642,13 +22642,13 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2q %zmm6, %zmm31, %zmm2
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm2 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa (%r9), %ymm7
-; AVX512DQ-BW-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa 64(%r9), %ymm5
-; AVX512DQ-BW-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa (%r8), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vmovdqa 64(%r8), %ymm6
-; AVX512DQ-BW-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
; AVX512DQ-BW-NEXT: movb $28, %r10b
; AVX512DQ-BW-NEXT: kmovd %r10d, %k2
@@ -22715,10 +22715,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2q %zmm0, %zmm31, %zmm3
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm3 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa64 128(%rax), %zmm8
-; AVX512DQ-BW-NEXT: vmovdqa 128(%r9), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 128(%r8), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%r9), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%r8), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm8[2,3,2,3]
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -22754,10 +22754,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2q %zmm0, %zmm31, %zmm5
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm5 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa64 192(%rax), %zmm0
-; AVX512DQ-BW-NEXT: vmovdqa 192(%r9), %ymm8
-; AVX512DQ-BW-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 192(%r8), %ymm1
-; AVX512DQ-BW-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%r9), %ymm8
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%r8), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[2],ymm8[2]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -22792,10 +22792,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2q %zmm0, %zmm31, %zmm1
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, %zmm1 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa64 256(%rax), %zmm0
-; AVX512DQ-BW-NEXT: vmovdqa 256(%r9), %ymm12
-; AVX512DQ-BW-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 256(%r8), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%r9), %ymm12
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%r8), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm12[0],ymm4[2],ymm12[2]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm4[2,3,2,3],zmm0[2,3,2,3]
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -22831,10 +22831,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2q %zmm29, %zmm31, %zmm5
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm5 {%k1}
; AVX512DQ-BW-NEXT: vmovdqa64 320(%rax), %zmm1
-; AVX512DQ-BW-NEXT: vmovdqa 320(%r9), %ymm7
-; AVX512DQ-BW-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 320(%r8), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%r9), %ymm7
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%r8), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm7[2,3,2,3],zmm1[2,3,2,3]
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -23156,7 +23156,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm13
; AVX512DQ-BW-NEXT: movb $12, %sil
; AVX512DQ-BW-NEXT: kmovd %esi, %k5
-; AVX512DQ-BW-NEXT: vmovdqa 448(%rdx), %xmm2
+; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdx), %xmm2
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm0, %zmm4 {%k5}
@@ -23171,10 +23171,10 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpermt2q %zmm0, %zmm8, %zmm10
; AVX512DQ-BW-NEXT: vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm25, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 384(%r9), %ymm12
-; AVX512DQ-BW-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-NEXT: vmovdqa 384(%r8), %ymm8
-; AVX512DQ-BW-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%r9), %ymm12
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%r8), %ymm8
+; AVX512DQ-BW-NEXT: vmovdqu64 %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[2],ymm12[2]
; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm8[2,3,2,3],zmm0[2,3,2,3]
; AVX512DQ-BW-NEXT: vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
@@ -23251,25 +23251,25 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k5}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k5}
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rdx), %xmm0
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdx), %xmm0
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rdx), %xmm0
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdx), %xmm0
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdx), %xmm0
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdx), %xmm0
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
@@ -23407,7 +23407,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: # zmm7 = zmm9[0,1,2,3],mem[4,5,6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
@@ -23416,42 +23416,42 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: kmovd %ecx, %k1
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm9 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm17 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm18 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm3 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm8, %zmm0, %zmm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-BW-NEXT: vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
; AVX512DQ-BW-NEXT: # ymm8 = ymm0[1],mem[1],ymm0[3],mem[3]
; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
@@ -23590,7 +23590,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm7, %zmm31, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm2 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%r9), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%r9), %ymm16
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%r8), %ymm5
@@ -23664,8 +23664,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm31, %zmm6
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm6 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rax), %zmm9
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%r9), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%r9), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%r8), %ymm26
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm26[0],ymm0[0],ymm26[2],ymm0[2]
; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm1[2,3,2,3],zmm9[2,3,2,3]
@@ -23703,7 +23703,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm6 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rax), %zmm29
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%r9), %ymm22
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%r8), %ymm14
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%r8), %ymm14
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm22[0],ymm14[2],ymm22[2]
; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm1[2,3,2,3],zmm29[2,3,2,3]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -23741,8 +23741,8 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm31, %zmm8
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, %zmm8 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rax), %zmm25
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%r9), %ymm13
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%r8), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%r9), %ymm13
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%r8), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm13[0],ymm6[2],ymm13[2]
; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 {%k2} = zmm7[2,3,2,3],zmm25[2,3,2,3]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -23782,7 +23782,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm5, %zmm31, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, %zmm2 {%k1}
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rax), %zmm25
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%r9), %ymm10
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%r9), %ymm10
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%r8), %ymm21
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm8 = ymm21[0],ymm10[0],ymm21[2],ymm10[2]
; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm8[2,3,2,3],zmm25[2,3,2,3]
@@ -23814,17 +23814,17 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm26 # 32-byte Folded Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm26, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm22, %ymm0, %ymm14
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm13, %ymm0, %ymm6
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm10, %ymm0, %ymm21
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm21, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%r9), %ymm2
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%r8), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%r9), %ymm2
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%r8), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %ymm2, %ymm0, %ymm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdx), %zmm0
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rcx), %zmm21
@@ -23891,7 +23891,7 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm30, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm18, %zmm4, %zmm2
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdx), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -23899,32 +23899,32 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm23, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm19, %zmm4, %zmm2
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdx), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm29, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm29, %zmm4, %zmm2
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rdx), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm24, %zmm4, %zmm20
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rdx), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm20
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm2
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm14, %zmm4, %zmm2
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rdx), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdi), %zmm9
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rsi), %zmm3
; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm3, %zmm9, %zmm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rdx), %xmm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdx), %xmm0
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdx), %zmm5
@@ -24314,25 +24314,25 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: kmovd %ecx, %k2
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm13 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm0 {%k2}
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm16 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm31 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm20 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm19 {%k2}
-; AVX512DQ-BW-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512DQ-BW-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],mem[6,7]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k2}
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-8.ll
index 71f96a25a0941..b742cf4feffb0 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-8.ll
@@ -8881,10 +8881,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
; AVX512-NEXT: vinserti64x4 $0, %ymm3, %zmm11, %zmm26
; AVX512-NEXT: vmovdqa64 %zmm28, %zmm4 {%k1}
-; AVX512-NEXT: vmovdqa 128(%rsi), %xmm3
-; AVX512-NEXT: vinserti128 $1, 128(%rcx), %ymm3, %ymm3
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512-NEXT: vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512-NEXT: vmovdqa64 128(%rsi), %xmm3
+; AVX512-NEXT: vinserti32x4 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512-NEXT: vinserti32x4 $1, 128(%rdx), %ymm5, %ymm5
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm28
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -8896,10 +8896,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
; AVX512-NEXT: vinserti64x4 $0, %ymm4, %zmm10, %zmm16
; AVX512-NEXT: vmovdqa64 %zmm12, %zmm30 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rsi), %xmm4
-; AVX512-NEXT: vinserti128 $1, 192(%rcx), %ymm4, %ymm5
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm4
-; AVX512-NEXT: vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512-NEXT: vmovdqa64 192(%rsi), %xmm4
+; AVX512-NEXT: vinserti32x4 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm4
+; AVX512-NEXT: vinserti32x4 $1, 192(%rdx), %ymm4, %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm4, %zmm30, %zmm30
; AVX512-NEXT: vmovdqa64 %zmm15, %zmm31 {%k1}
@@ -8960,8 +8960,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti64x4 $0, %ymm13, %zmm4, %zmm13
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm3, %zmm17 {%k1}
-; AVX512-NEXT: vmovdqa 128(%rcx), %ymm14
-; AVX512-NEXT: vmovdqa 128(%rdx), %ymm15
+; AVX512-NEXT: vmovdqa64 128(%rcx), %ymm14
+; AVX512-NEXT: vmovdqa64 128(%rdx), %ymm15
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512-NEXT: vmovdqa64 128(%rsi), %ymm21
; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm22
@@ -8975,8 +8975,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm4
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm5, %zmm23 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rcx), %ymm14
-; AVX512-NEXT: vmovdqa 192(%rdx), %ymm15
+; AVX512-NEXT: vmovdqa64 192(%rcx), %ymm14
+; AVX512-NEXT: vmovdqa64 192(%rdx), %ymm15
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512-NEXT: vmovdqa64 192(%rsi), %ymm17
; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -9354,10 +9354,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm11, %zmm26
; AVX512-FCP-NEXT: vmovdqa64 %zmm28, %zmm4 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 128(%rsi), %xmm3
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rcx), %ymm3, %ymm3
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 128(%rsi), %xmm3
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rdx), %ymm5, %ymm5
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm28
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -9369,10 +9369,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm10, %zmm16
; AVX512-FCP-NEXT: vmovdqa64 %zmm12, %zmm30 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 192(%rsi), %xmm4
-; AVX512-FCP-NEXT: vinserti128 $1, 192(%rcx), %ymm4, %ymm5
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %xmm4
-; AVX512-FCP-NEXT: vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 192(%rsi), %xmm4
+; AVX512-FCP-NEXT: vinserti32x4 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm4
+; AVX512-FCP-NEXT: vinserti32x4 $1, 192(%rdx), %ymm4, %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm30, %zmm30
; AVX512-FCP-NEXT: vmovdqa64 %zmm15, %zmm31 {%k1}
@@ -9433,8 +9433,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm13, %zmm4, %zmm13
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm3, %zmm17 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 128(%rcx), %ymm14
-; AVX512-FCP-NEXT: vmovdqa 128(%rdx), %ymm15
+; AVX512-FCP-NEXT: vmovdqa64 128(%rcx), %ymm14
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdx), %ymm15
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512-FCP-NEXT: vmovdqa64 128(%rsi), %ymm21
; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm22
@@ -9448,8 +9448,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm4
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm23 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 192(%rcx), %ymm14
-; AVX512-FCP-NEXT: vmovdqa 192(%rdx), %ymm15
+; AVX512-FCP-NEXT: vmovdqa64 192(%rcx), %ymm14
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdx), %ymm15
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512-FCP-NEXT: vmovdqa64 192(%rsi), %ymm17
; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -9827,10 +9827,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm3, %zmm11, %zmm26
; AVX512DQ-NEXT: vmovdqa64 %zmm28, %zmm4 {%k1}
-; AVX512DQ-NEXT: vmovdqa 128(%rsi), %xmm3
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rcx), %ymm3, %ymm3
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512DQ-NEXT: vmovdqa64 128(%rsi), %xmm3
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rdx), %ymm5, %ymm5
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm28
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -9842,10 +9842,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm4, %zmm10, %zmm16
; AVX512DQ-NEXT: vmovdqa64 %zmm12, %zmm30 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rsi), %xmm4
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rcx), %ymm4, %ymm5
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm4
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512DQ-NEXT: vmovdqa64 192(%rsi), %xmm4
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm4
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdx), %ymm4, %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm4, %zmm30, %zmm30
; AVX512DQ-NEXT: vmovdqa64 %zmm15, %zmm31 {%k1}
@@ -9906,8 +9906,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm13, %zmm4, %zmm13
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm3, %zmm17 {%k1}
-; AVX512DQ-NEXT: vmovdqa 128(%rcx), %ymm14
-; AVX512DQ-NEXT: vmovdqa 128(%rdx), %ymm15
+; AVX512DQ-NEXT: vmovdqa64 128(%rcx), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 128(%rdx), %ymm15
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-NEXT: vmovdqa64 128(%rsi), %ymm21
; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm22
@@ -9921,8 +9921,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm4
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm5, %zmm23 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rcx), %ymm14
-; AVX512DQ-NEXT: vmovdqa 192(%rdx), %ymm15
+; AVX512DQ-NEXT: vmovdqa64 192(%rcx), %ymm14
+; AVX512DQ-NEXT: vmovdqa64 192(%rdx), %ymm15
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-NEXT: vmovdqa64 192(%rsi), %ymm17
; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -10300,10 +10300,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm11, %zmm26
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm28, %zmm4 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rsi), %xmm3
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rcx), %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rsi), %xmm3
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rdx), %ymm5, %ymm5
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm28
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -10315,10 +10315,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm10, %zmm16
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm12, %zmm30 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rsi), %xmm4
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 192(%rcx), %ymm4, %ymm5
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rsi), %xmm4
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 192(%rdx), %ymm4, %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm30, %zmm30
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm15, %zmm31 {%k1}
@@ -10379,8 +10379,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm13, %zmm4, %zmm13
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, %zmm17 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rcx), %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdx), %ymm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rcx), %ymm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdx), %ymm15
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rsi), %ymm21
; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm22
@@ -10394,8 +10394,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm23 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rcx), %ymm14
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdx), %ymm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rcx), %ymm14
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdx), %ymm15
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rsi), %ymm17
; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -10773,10 +10773,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm3, %zmm11, %zmm26
; AVX512BW-NEXT: vmovdqa64 %zmm28, %zmm4 {%k1}
-; AVX512BW-NEXT: vmovdqa 128(%rsi), %xmm3
-; AVX512BW-NEXT: vinserti128 $1, 128(%rcx), %ymm3, %ymm3
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512BW-NEXT: vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512BW-NEXT: vmovdqa64 128(%rsi), %xmm3
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rdx), %ymm5, %ymm5
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm28
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -10788,10 +10788,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm4, %zmm10, %zmm16
; AVX512BW-NEXT: vmovdqa64 %zmm12, %zmm30 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rsi), %xmm4
-; AVX512BW-NEXT: vinserti128 $1, 192(%rcx), %ymm4, %ymm5
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm4
-; AVX512BW-NEXT: vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512BW-NEXT: vmovdqa64 192(%rsi), %xmm4
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %xmm4
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdx), %ymm4, %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm4, %zmm30, %zmm30
; AVX512BW-NEXT: vmovdqa64 %zmm15, %zmm31 {%k1}
@@ -10852,8 +10852,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti64x4 $0, %ymm13, %zmm4, %zmm13
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm3, %zmm17 {%k1}
-; AVX512BW-NEXT: vmovdqa 128(%rcx), %ymm14
-; AVX512BW-NEXT: vmovdqa 128(%rdx), %ymm15
+; AVX512BW-NEXT: vmovdqa64 128(%rcx), %ymm14
+; AVX512BW-NEXT: vmovdqa64 128(%rdx), %ymm15
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512BW-NEXT: vmovdqa64 128(%rsi), %ymm21
; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm22
@@ -10867,8 +10867,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm4
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm23 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rcx), %ymm14
-; AVX512BW-NEXT: vmovdqa 192(%rdx), %ymm15
+; AVX512BW-NEXT: vmovdqa64 192(%rcx), %ymm14
+; AVX512BW-NEXT: vmovdqa64 192(%rdx), %ymm15
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512BW-NEXT: vmovdqa64 192(%rsi), %ymm17
; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -11246,10 +11246,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm11, %zmm26
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm28, %zmm4 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rsi), %xmm3
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rcx), %ymm3, %ymm3
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rsi), %xmm3
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rdx), %ymm5, %ymm5
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm28
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -11261,10 +11261,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm10, %zmm16
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm12, %zmm30 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rsi), %xmm4
-; AVX512BW-FCP-NEXT: vinserti128 $1, 192(%rcx), %ymm4, %ymm5
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm4
-; AVX512BW-FCP-NEXT: vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rsi), %xmm4
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm4
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 192(%rdx), %ymm4, %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm30, %zmm30
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm15, %zmm31 {%k1}
@@ -11325,8 +11325,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm13, %zmm4, %zmm13
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm17 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rcx), %ymm14
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdx), %ymm15
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rcx), %ymm14
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdx), %ymm15
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512BW-FCP-NEXT: vmovdqa64 128(%rsi), %ymm21
; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm22
@@ -11340,8 +11340,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm4
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm23 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rcx), %ymm14
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdx), %ymm15
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rcx), %ymm14
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdx), %ymm15
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512BW-FCP-NEXT: vmovdqa64 192(%rsi), %ymm17
; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -11719,10 +11719,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm3, %zmm11, %zmm26
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm28, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rsi), %xmm3
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rcx), %ymm3, %ymm3
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rsi), %xmm3
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rdx), %ymm5, %ymm5
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm28
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -11734,10 +11734,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm4, %zmm10, %zmm16
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm12, %zmm30 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rsi), %xmm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rcx), %ymm4, %ymm5
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rsi), %xmm4
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %xmm4
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdx), %ymm4, %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm4, %zmm30, %zmm30
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm15, %zmm31 {%k1}
@@ -11798,8 +11798,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm13, %zmm4, %zmm13
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, %zmm17 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rcx), %ymm14
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdx), %ymm15
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rcx), %ymm14
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdx), %ymm15
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-BW-NEXT: vmovdqa64 128(%rsi), %ymm21
; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm22
@@ -11813,8 +11813,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm4
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm23 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rcx), %ymm14
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdx), %ymm15
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rcx), %ymm14
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdx), %ymm15
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-BW-NEXT: vmovdqa64 192(%rsi), %ymm17
; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -12192,10 +12192,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm3, %zmm11, %zmm26
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm28, %zmm4 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rsi), %xmm3
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rcx), %ymm3, %ymm3
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm5
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rsi), %xmm3
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm5
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rdx), %ymm5, %ymm5
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm4, %zmm28
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -12207,10 +12207,10 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm10, %zmm16
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm12, %zmm30 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rsi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 192(%rcx), %ymm4, %ymm5
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rsi), %xmm4
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm4
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 192(%rdx), %ymm4, %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm30, %zmm30
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm15, %zmm31 {%k1}
@@ -12271,8 +12271,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm13, %zmm4, %zmm13
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm17 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rcx), %ymm14
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdx), %ymm15
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rcx), %ymm14
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdx), %ymm15
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rsi), %ymm21
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm22
@@ -12286,8 +12286,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm20, %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm23 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rcx), %ymm14
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdx), %ymm15
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rcx), %ymm14
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdx), %ymm15
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rsi), %ymm17
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm19
@@ -19257,11 +19257,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm8
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512-NEXT: vmovdqa 128(%rcx), %ymm0
-; AVX512-NEXT: vmovdqa 128(%rdx), %ymm1
+; AVX512-NEXT: vmovdqa64 128(%rcx), %ymm0
+; AVX512-NEXT: vmovdqa64 128(%rdx), %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-NEXT: vmovdqa 128(%rsi), %ymm4
-; AVX512-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 128(%rsi), %ymm4
+; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm27, %zmm23
@@ -19274,11 +19274,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm25
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm20 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rcx), %ymm0
-; AVX512-NEXT: vmovdqa 192(%rdx), %ymm1
+; AVX512-NEXT: vmovdqa64 192(%rcx), %ymm0
+; AVX512-NEXT: vmovdqa64 192(%rdx), %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-NEXT: vmovdqa 192(%rsi), %ymm4
-; AVX512-NEXT: vmovdqa 192(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 192(%rsi), %ymm4
+; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm20, %zmm21
@@ -19291,11 +19291,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm19
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm28 {%k1}
-; AVX512-NEXT: vmovdqa 256(%rcx), %ymm0
-; AVX512-NEXT: vmovdqa 256(%rdx), %ymm1
+; AVX512-NEXT: vmovdqa64 256(%rcx), %ymm0
+; AVX512-NEXT: vmovdqa64 256(%rdx), %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-NEXT: vmovdqa 256(%rsi), %ymm4
-; AVX512-NEXT: vmovdqa 256(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 256(%rsi), %ymm4
+; AVX512-NEXT: vmovdqa64 256(%rdi), %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm28, %zmm15
@@ -19308,11 +19308,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm28
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512-NEXT: vmovdqa 320(%rcx), %ymm0
-; AVX512-NEXT: vmovdqa 320(%rdx), %ymm1
+; AVX512-NEXT: vmovdqa64 320(%rcx), %ymm0
+; AVX512-NEXT: vmovdqa64 320(%rdx), %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-NEXT: vmovdqa 320(%rsi), %ymm4
-; AVX512-NEXT: vmovdqa 320(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 320(%rsi), %ymm4
+; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm22, %zmm22
@@ -19323,11 +19323,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm29, %zmm27
; AVX512-NEXT: vmovdqa64 %zmm30, %zmm17 {%k1}
-; AVX512-NEXT: vmovdqa 384(%rcx), %ymm0
-; AVX512-NEXT: vmovdqa 384(%rdx), %ymm1
+; AVX512-NEXT: vmovdqa64 384(%rcx), %ymm0
+; AVX512-NEXT: vmovdqa64 384(%rdx), %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-NEXT: vmovdqa 384(%rsi), %ymm4
-; AVX512-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512-NEXT: vmovdqa64 384(%rsi), %ymm4
+; AVX512-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm17, %zmm14
@@ -19337,11 +19337,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm13
; AVX512-NEXT: vmovdqa64 %zmm18, %zmm10 {%k1}
-; AVX512-NEXT: vmovdqa 448(%rcx), %ymm0
-; AVX512-NEXT: vmovdqa 448(%rdx), %ymm1
+; AVX512-NEXT: vmovdqa64 448(%rcx), %ymm0
+; AVX512-NEXT: vmovdqa64 448(%rdx), %ymm1
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-NEXT: vmovdqa 448(%rsi), %ymm4
-; AVX512-NEXT: vmovdqa 448(%rdi), %ymm5
+; AVX512-NEXT: vmovdqa64 448(%rsi), %ymm4
+; AVX512-NEXT: vmovdqa64 448(%rdi), %ymm5
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
; AVX512-NEXT: vinserti64x4 $0, %ymm2, %zmm10, %zmm10
@@ -19381,10 +19381,10 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512-NEXT: vmovdqa 128(%rsi), %xmm1
-; AVX512-NEXT: vinserti128 $1, 128(%rcx), %ymm1, %ymm1
-; AVX512-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512-NEXT: vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512-NEXT: vmovdqa64 128(%rsi), %xmm1
+; AVX512-NEXT: vinserti32x4 $1, 128(%rcx), %ymm1, %ymm1
+; AVX512-NEXT: vmovdqa64 128(%rdi), %xmm4
+; AVX512-NEXT: vinserti32x4 $1, 128(%rdx), %ymm4, %ymm6
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm4, %zmm7, %zmm4
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -19395,9 +19395,9 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512-NEXT: vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512-NEXT: vmovdqa 192(%rsi), %xmm1
-; AVX512-NEXT: vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512-NEXT: vmovdqa64 192(%rsi), %xmm1
+; AVX512-NEXT: vinserti32x4 $1, 192(%rcx), %ymm1, %ymm1
+; AVX512-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512-NEXT: vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
; AVX512-NEXT: vinserti64x4 $0, %ymm7, %zmm17, %zmm7
@@ -20204,11 +20204,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm8
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 128(%rcx), %ymm0
-; AVX512-FCP-NEXT: vmovdqa 128(%rdx), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 128(%rcx), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdx), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-FCP-NEXT: vmovdqa 128(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 128(%rsi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm27, %zmm23
@@ -20221,11 +20221,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm25
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm20 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 192(%rcx), %ymm0
-; AVX512-FCP-NEXT: vmovdqa 192(%rdx), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 192(%rcx), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdx), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-FCP-NEXT: vmovdqa 192(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 192(%rsi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm20, %zmm21
@@ -20238,11 +20238,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm19
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm28 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 256(%rcx), %ymm0
-; AVX512-FCP-NEXT: vmovdqa 256(%rdx), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 256(%rcx), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdx), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-FCP-NEXT: vmovdqa 256(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 256(%rsi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm28, %zmm15
@@ -20255,11 +20255,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm28
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 320(%rcx), %ymm0
-; AVX512-FCP-NEXT: vmovdqa 320(%rdx), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 320(%rcx), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdx), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-FCP-NEXT: vmovdqa 320(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 320(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 320(%rsi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm22, %zmm22
@@ -20270,11 +20270,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm29, %zmm27
; AVX512-FCP-NEXT: vmovdqa64 %zmm30, %zmm17 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 384(%rcx), %ymm0
-; AVX512-FCP-NEXT: vmovdqa 384(%rdx), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 384(%rcx), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdx), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-FCP-NEXT: vmovdqa 384(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 384(%rsi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm17, %zmm14
@@ -20284,11 +20284,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm13
; AVX512-FCP-NEXT: vmovdqa64 %zmm18, %zmm10 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 448(%rcx), %ymm0
-; AVX512-FCP-NEXT: vmovdqa 448(%rdx), %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 448(%rcx), %ymm0
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdx), %ymm1
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512-FCP-NEXT: vmovdqa 448(%rsi), %ymm4
-; AVX512-FCP-NEXT: vmovdqa 448(%rdi), %ymm5
+; AVX512-FCP-NEXT: vmovdqa64 448(%rsi), %ymm4
+; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %ymm5
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm10, %zmm10
@@ -20328,10 +20328,10 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 128(%rsi), %xmm1
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rcx), %ymm1, %ymm1
-; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512-FCP-NEXT: vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512-FCP-NEXT: vmovdqa64 128(%rsi), %xmm1
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rcx), %ymm1, %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %xmm4
+; AVX512-FCP-NEXT: vinserti32x4 $1, 128(%rdx), %ymm4, %ymm6
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm7, %zmm4
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -20342,9 +20342,9 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 192(%rsi), %xmm1
-; AVX512-FCP-NEXT: vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512-FCP-NEXT: vmovdqa64 192(%rsi), %xmm1
+; AVX512-FCP-NEXT: vinserti32x4 $1, 192(%rcx), %ymm1, %ymm1
+; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512-FCP-NEXT: vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
; AVX512-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm17, %zmm7
@@ -21151,11 +21151,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm8
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512DQ-NEXT: vmovdqa 128(%rcx), %ymm0
-; AVX512DQ-NEXT: vmovdqa 128(%rdx), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 128(%rcx), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 128(%rdx), %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-NEXT: vmovdqa 128(%rsi), %ymm4
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rsi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm27, %zmm23
@@ -21168,11 +21168,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm25
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm20 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rcx), %ymm0
-; AVX512DQ-NEXT: vmovdqa 192(%rdx), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 192(%rcx), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 192(%rdx), %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-NEXT: vmovdqa 192(%rsi), %ymm4
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 192(%rsi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm20, %zmm21
@@ -21185,11 +21185,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm19
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm28 {%k1}
-; AVX512DQ-NEXT: vmovdqa 256(%rcx), %ymm0
-; AVX512DQ-NEXT: vmovdqa 256(%rdx), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 256(%rcx), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 256(%rdx), %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-NEXT: vmovdqa 256(%rsi), %ymm4
-; AVX512DQ-NEXT: vmovdqa 256(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 256(%rsi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm28, %zmm15
@@ -21202,11 +21202,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm28
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512DQ-NEXT: vmovdqa 320(%rcx), %ymm0
-; AVX512DQ-NEXT: vmovdqa 320(%rdx), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 320(%rcx), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 320(%rdx), %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-NEXT: vmovdqa 320(%rsi), %ymm4
-; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 320(%rsi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm22, %zmm22
@@ -21217,11 +21217,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm29, %zmm27
; AVX512DQ-NEXT: vmovdqa64 %zmm30, %zmm17 {%k1}
-; AVX512DQ-NEXT: vmovdqa 384(%rcx), %ymm0
-; AVX512DQ-NEXT: vmovdqa 384(%rdx), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 384(%rcx), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 384(%rdx), %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-NEXT: vmovdqa 384(%rsi), %ymm4
-; AVX512DQ-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512DQ-NEXT: vmovdqa64 384(%rsi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm17, %zmm14
@@ -21231,11 +21231,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm13
; AVX512DQ-NEXT: vmovdqa64 %zmm18, %zmm10 {%k1}
-; AVX512DQ-NEXT: vmovdqa 448(%rcx), %ymm0
-; AVX512DQ-NEXT: vmovdqa 448(%rdx), %ymm1
+; AVX512DQ-NEXT: vmovdqa64 448(%rcx), %ymm0
+; AVX512DQ-NEXT: vmovdqa64 448(%rdx), %ymm1
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-NEXT: vmovdqa 448(%rsi), %ymm4
-; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm5
+; AVX512DQ-NEXT: vmovdqa64 448(%rsi), %ymm4
+; AVX512DQ-NEXT: vmovdqa64 448(%rdi), %ymm5
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm2, %zmm10, %zmm10
@@ -21275,10 +21275,10 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512DQ-NEXT: vmovdqa 128(%rsi), %xmm1
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rcx), %ymm1, %ymm1
-; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512DQ-NEXT: vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512DQ-NEXT: vmovdqa64 128(%rsi), %xmm1
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rcx), %ymm1, %ymm1
+; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %xmm4
+; AVX512DQ-NEXT: vinserti32x4 $1, 128(%rdx), %ymm4, %ymm6
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm4, %zmm7, %zmm4
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -21289,9 +21289,9 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512DQ-NEXT: vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512DQ-NEXT: vmovdqa 192(%rsi), %xmm1
-; AVX512DQ-NEXT: vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512DQ-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512DQ-NEXT: vmovdqa64 192(%rsi), %xmm1
+; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rcx), %ymm1, %ymm1
+; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512DQ-NEXT: vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
; AVX512DQ-NEXT: vinserti64x4 $0, %ymm7, %zmm17, %zmm7
@@ -22098,11 +22098,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm8
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rcx), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdx), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rcx), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdx), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rsi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm27, %zmm23
@@ -22115,11 +22115,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm25
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm20 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rcx), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdx), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rcx), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdx), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rsi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm20, %zmm21
@@ -22132,11 +22132,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm19
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm28 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rcx), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdx), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rcx), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdx), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rsi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm28, %zmm15
@@ -22149,11 +22149,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm28
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rcx), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdx), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rcx), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdx), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 320(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rsi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm22, %zmm22
@@ -22164,11 +22164,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm29, %zmm27
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm30, %zmm17 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rcx), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdx), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rcx), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdx), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rsi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm17, %zmm14
@@ -22178,11 +22178,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm13
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm18, %zmm10 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rcx), %ymm0
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdx), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rcx), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdx), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rsi), %ymm4
-; AVX512DQ-FCP-NEXT: vmovdqa 448(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rsi), %ymm4
+; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %ymm5
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm10, %zmm10
@@ -22222,10 +22222,10 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rsi), %xmm1
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rcx), %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rsi), %xmm1
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rcx), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %xmm4
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 128(%rdx), %ymm4, %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm7, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -22236,9 +22236,9 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rsi), %xmm1
-; AVX512DQ-FCP-NEXT: vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rsi), %xmm1
+; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 192(%rcx), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512DQ-FCP-NEXT: vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
; AVX512DQ-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm17, %zmm7
@@ -23045,11 +23045,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm8
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512BW-NEXT: vmovdqa 128(%rcx), %ymm0
-; AVX512BW-NEXT: vmovdqa 128(%rdx), %ymm1
+; AVX512BW-NEXT: vmovdqa64 128(%rcx), %ymm0
+; AVX512BW-NEXT: vmovdqa64 128(%rdx), %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT: vmovdqa 128(%rsi), %ymm4
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 128(%rsi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm27, %zmm23
@@ -23062,11 +23062,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm25
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm20 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rcx), %ymm0
-; AVX512BW-NEXT: vmovdqa 192(%rdx), %ymm1
+; AVX512BW-NEXT: vmovdqa64 192(%rcx), %ymm0
+; AVX512BW-NEXT: vmovdqa64 192(%rdx), %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT: vmovdqa 192(%rsi), %ymm4
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 192(%rsi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm20, %zmm21
@@ -23079,11 +23079,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm19
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm28 {%k1}
-; AVX512BW-NEXT: vmovdqa 256(%rcx), %ymm0
-; AVX512BW-NEXT: vmovdqa 256(%rdx), %ymm1
+; AVX512BW-NEXT: vmovdqa64 256(%rcx), %ymm0
+; AVX512BW-NEXT: vmovdqa64 256(%rdx), %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT: vmovdqa 256(%rsi), %ymm4
-; AVX512BW-NEXT: vmovdqa 256(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 256(%rsi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 256(%rdi), %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm28, %zmm15
@@ -23096,11 +23096,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm28
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512BW-NEXT: vmovdqa 320(%rcx), %ymm0
-; AVX512BW-NEXT: vmovdqa 320(%rdx), %ymm1
+; AVX512BW-NEXT: vmovdqa64 320(%rcx), %ymm0
+; AVX512BW-NEXT: vmovdqa64 320(%rdx), %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT: vmovdqa 320(%rsi), %ymm4
-; AVX512BW-NEXT: vmovdqa 320(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 320(%rsi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 320(%rdi), %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm22, %zmm22
@@ -23111,11 +23111,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm29, %zmm27
; AVX512BW-NEXT: vmovdqa64 %zmm30, %zmm17 {%k1}
-; AVX512BW-NEXT: vmovdqa 384(%rcx), %ymm0
-; AVX512BW-NEXT: vmovdqa 384(%rdx), %ymm1
+; AVX512BW-NEXT: vmovdqa64 384(%rcx), %ymm0
+; AVX512BW-NEXT: vmovdqa64 384(%rdx), %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT: vmovdqa 384(%rsi), %ymm4
-; AVX512BW-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512BW-NEXT: vmovdqa64 384(%rsi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm17, %zmm14
@@ -23125,11 +23125,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm13
; AVX512BW-NEXT: vmovdqa64 %zmm18, %zmm10 {%k1}
-; AVX512BW-NEXT: vmovdqa 448(%rcx), %ymm0
-; AVX512BW-NEXT: vmovdqa 448(%rdx), %ymm1
+; AVX512BW-NEXT: vmovdqa64 448(%rcx), %ymm0
+; AVX512BW-NEXT: vmovdqa64 448(%rdx), %ymm1
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT: vmovdqa 448(%rsi), %ymm4
-; AVX512BW-NEXT: vmovdqa 448(%rdi), %ymm5
+; AVX512BW-NEXT: vmovdqa64 448(%rsi), %ymm4
+; AVX512BW-NEXT: vmovdqa64 448(%rdi), %ymm5
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm2, %zmm10, %zmm10
@@ -23169,10 +23169,10 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512BW-NEXT: vmovdqa 128(%rsi), %xmm1
-; AVX512BW-NEXT: vinserti128 $1, 128(%rcx), %ymm1, %ymm1
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512BW-NEXT: vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512BW-NEXT: vmovdqa64 128(%rsi), %xmm1
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rcx), %ymm1, %ymm1
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %xmm4
+; AVX512BW-NEXT: vinserti32x4 $1, 128(%rdx), %ymm4, %ymm6
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm4, %zmm7, %zmm4
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -23183,9 +23183,9 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512BW-NEXT: vmovdqa 192(%rsi), %xmm1
-; AVX512BW-NEXT: vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512BW-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512BW-NEXT: vmovdqa64 192(%rsi), %xmm1
+; AVX512BW-NEXT: vinserti32x4 $1, 192(%rcx), %ymm1, %ymm1
+; AVX512BW-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512BW-NEXT: vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
; AVX512BW-NEXT: vinserti64x4 $0, %ymm7, %zmm17, %zmm7
@@ -23992,11 +23992,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm8
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rcx), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdx), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rcx), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdx), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rsi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rsi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm27, %zmm23
@@ -24009,11 +24009,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm25
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm20 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rcx), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdx), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rcx), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdx), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rsi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rsi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm20, %zmm21
@@ -24026,11 +24026,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm19
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm28 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rcx), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rdx), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rcx), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdx), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rsi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rsi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm28, %zmm15
@@ -24043,11 +24043,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm28
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rcx), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rdx), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rcx), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdx), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rsi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 320(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rsi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm22, %zmm22
@@ -24058,11 +24058,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm29, %zmm27
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm30, %zmm17 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rcx), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rdx), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rcx), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdx), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rsi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rsi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm17, %zmm14
@@ -24072,11 +24072,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm13
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm18, %zmm10 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 448(%rcx), %ymm0
-; AVX512BW-FCP-NEXT: vmovdqa 448(%rdx), %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 448(%rcx), %ymm0
+; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdx), %ymm1
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-FCP-NEXT: vmovdqa 448(%rsi), %ymm4
-; AVX512BW-FCP-NEXT: vmovdqa 448(%rdi), %ymm5
+; AVX512BW-FCP-NEXT: vmovdqa64 448(%rsi), %ymm4
+; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdi), %ymm5
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
; AVX512BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm10, %zmm10
@@ -24116,10 +24116,10 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rsi), %xmm1
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rcx), %ymm1, %ymm1
-; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512BW-FCP-NEXT: vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rsi), %xmm1
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rcx), %ymm1, %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm4
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 128(%rdx), %ymm4, %ymm6
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm7, %zmm4
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -24130,9 +24130,9 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rsi), %xmm1
-; AVX512BW-FCP-NEXT: vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rsi), %xmm1
+; AVX512BW-FCP-NEXT: vinserti32x4 $1, 192(%rcx), %ymm1, %ymm1
+; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512BW-FCP-NEXT: vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
; AVX512BW-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm17, %zmm7
@@ -24939,11 +24939,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm8
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rcx), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdx), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rcx), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdx), %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rsi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rsi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm27, %zmm23
@@ -24956,11 +24956,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm25
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm20 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rcx), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdx), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rcx), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdx), %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rsi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rsi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm20, %zmm21
@@ -24973,11 +24973,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm19
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm28 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rcx), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rdx), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rcx), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdx), %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rsi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 256(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rsi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm28, %zmm15
@@ -24990,11 +24990,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm28
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rcx), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rdx), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rcx), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdx), %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rsi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 320(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rsi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm22, %zmm22
@@ -25005,11 +25005,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm29, %zmm27
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm30, %zmm17 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rcx), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdx), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rcx), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdx), %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rsi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rsi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm17, %zmm14
@@ -25019,11 +25019,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm13
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm18, %zmm10 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 448(%rcx), %ymm0
-; AVX512DQ-BW-NEXT: vmovdqa 448(%rdx), %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 448(%rcx), %ymm0
+; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdx), %ymm1
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-NEXT: vmovdqa 448(%rsi), %ymm4
-; AVX512DQ-BW-NEXT: vmovdqa 448(%rdi), %ymm5
+; AVX512DQ-BW-NEXT: vmovdqa64 448(%rsi), %ymm4
+; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdi), %ymm5
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
; AVX512DQ-BW-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm2, %zmm10, %zmm10
@@ -25063,10 +25063,10 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rsi), %xmm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rcx), %ymm1, %ymm1
-; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512DQ-BW-NEXT: vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rsi), %xmm1
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rcx), %ymm1, %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %xmm4
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 128(%rdx), %ymm4, %ymm6
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm4, %zmm7, %zmm4
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -25077,9 +25077,9 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rsi), %xmm1
-; AVX512DQ-BW-NEXT: vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512DQ-BW-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rsi), %xmm1
+; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rcx), %ymm1, %ymm1
+; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512DQ-BW-NEXT: vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
; AVX512DQ-BW-NEXT: vinserti64x4 $0, %ymm7, %zmm17, %zmm7
@@ -25886,11 +25886,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm8
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rcx), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdx), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rcx), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdx), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rsi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rsi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm27, %zmm23
@@ -25903,11 +25903,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm25
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm20 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rcx), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdx), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rcx), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdx), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rsi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rsi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm20, %zmm21
@@ -25920,11 +25920,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm19
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm28 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rcx), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rdx), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rcx), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdx), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rsi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 256(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rsi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm28, %zmm15
@@ -25937,11 +25937,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm7, %zmm28
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rcx), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rdx), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rcx), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdx), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rsi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 320(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rsi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm22, %zmm22
@@ -25952,11 +25952,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm29, %zmm27
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm30, %zmm17 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rcx), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rdx), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rcx), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdx), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rsi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 384(%rdi), %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rsi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdi), %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm17, %zmm14
@@ -25966,11 +25966,11 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm0, %zmm5, %zmm13
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm18, %zmm10 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rcx), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rdx), %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rcx), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdx), %ymm1
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rsi), %ymm4
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 448(%rdi), %ymm5
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rsi), %ymm4
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdi), %ymm5
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
; AVX512DQ-BW-FCP-NEXT: vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm2, %zmm10, %zmm10
@@ -26010,10 +26010,10 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rsi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rcx), %ymm1, %ymm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm4
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rsi), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rcx), %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %xmm4
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 128(%rdx), %ymm4, %ymm6
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm4, %zmm7, %zmm4
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
@@ -26024,9 +26024,9 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rsi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512DQ-BW-FCP-NEXT: vmovdqa 192(%rdi), %xmm7
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rsi), %xmm1
+; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 192(%rcx), %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %xmm7
; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $0, %ymm7, %zmm17, %zmm7
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
index b8abf759b8c2c..3a2431c70ea61 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
@@ -1967,8 +1967,8 @@ define void @store_i8_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vpshufb %ymm7, %ymm3, %ymm3
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm4[2,3]
; AVX512-NEXT: vpshufb %ymm7, %ymm1, %ymm1
-; AVX512-NEXT: vmovdqa %ymm3, 128(%rcx)
-; AVX512-NEXT: vmovdqa %ymm1, 160(%rcx)
+; AVX512-NEXT: vmovdqa64 %ymm3, 128(%rcx)
+; AVX512-NEXT: vmovdqa64 %ymm1, 160(%rcx)
; AVX512-NEXT: vmovdqa %ymm0, 64(%rcx)
; AVX512-NEXT: vmovdqa %ymm6, (%rcx)
; AVX512-NEXT: vmovdqa %ymm2, 32(%rcx)
@@ -2014,8 +2014,8 @@ define void @store_i8_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm3, %ymm3
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm4[2,3]
; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm1
-; AVX512-FCP-NEXT: vmovdqa %ymm3, 128(%rcx)
-; AVX512-FCP-NEXT: vmovdqa %ymm1, 160(%rcx)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm3, 128(%rcx)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm1, 160(%rcx)
; AVX512-FCP-NEXT: vmovdqa %ymm0, 64(%rcx)
; AVX512-FCP-NEXT: vmovdqa %ymm6, (%rcx)
; AVX512-FCP-NEXT: vmovdqa %ymm2, 32(%rcx)
@@ -2061,8 +2061,8 @@ define void @store_i8_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vpshufb %ymm7, %ymm3, %ymm3
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm4[2,3]
; AVX512DQ-NEXT: vpshufb %ymm7, %ymm1, %ymm1
-; AVX512DQ-NEXT: vmovdqa %ymm3, 128(%rcx)
-; AVX512DQ-NEXT: vmovdqa %ymm1, 160(%rcx)
+; AVX512DQ-NEXT: vmovdqa64 %ymm3, 128(%rcx)
+; AVX512DQ-NEXT: vmovdqa64 %ymm1, 160(%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm0, 64(%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm6, (%rcx)
; AVX512DQ-NEXT: vmovdqa %ymm2, 32(%rcx)
@@ -2108,8 +2108,8 @@ define void @store_i8_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm3, %ymm3
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm4[2,3]
; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm3, 128(%rcx)
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, 160(%rcx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm3, 128(%rcx)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, 160(%rcx)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 64(%rcx)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm6, (%rcx)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, 32(%rcx)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-4.ll
index 0a82488c7da4a..faf9815cc6bea 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-4.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-4.ll
@@ -1523,10 +1523,10 @@ define void @store_i8_stride4_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm8, %ymm9
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm8[2,3],ymm1[2,3]
-; AVX512-NEXT: vmovdqa %ymm5, 192(%r8)
-; AVX512-NEXT: vmovdqa %ymm1, 224(%r8)
-; AVX512-NEXT: vmovdqa %ymm7, 128(%r8)
-; AVX512-NEXT: vmovdqa %ymm9, 160(%r8)
+; AVX512-NEXT: vmovdqa64 %ymm5, 192(%r8)
+; AVX512-NEXT: vmovdqa64 %ymm1, 224(%r8)
+; AVX512-NEXT: vmovdqa64 %ymm7, 128(%r8)
+; AVX512-NEXT: vmovdqa64 %ymm9, 160(%r8)
; AVX512-NEXT: vmovdqa %ymm4, 64(%r8)
; AVX512-NEXT: vmovdqa %ymm0, 96(%r8)
; AVX512-NEXT: vmovdqa %ymm2, (%r8)
@@ -1568,10 +1568,10 @@ define void @store_i8_stride4_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm8, %ymm9
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512-FCP-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm8[2,3],ymm1[2,3]
-; AVX512-FCP-NEXT: vmovdqa %ymm5, 192(%r8)
-; AVX512-FCP-NEXT: vmovdqa %ymm1, 224(%r8)
-; AVX512-FCP-NEXT: vmovdqa %ymm7, 128(%r8)
-; AVX512-FCP-NEXT: vmovdqa %ymm9, 160(%r8)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm5, 192(%r8)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm1, 224(%r8)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm7, 128(%r8)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm9, 160(%r8)
; AVX512-FCP-NEXT: vmovdqa %ymm4, 64(%r8)
; AVX512-FCP-NEXT: vmovdqa %ymm0, 96(%r8)
; AVX512-FCP-NEXT: vmovdqa %ymm2, (%r8)
@@ -1613,10 +1613,10 @@ define void @store_i8_stride4_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm8, %ymm9
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512DQ-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm8[2,3],ymm1[2,3]
-; AVX512DQ-NEXT: vmovdqa %ymm5, 192(%r8)
-; AVX512DQ-NEXT: vmovdqa %ymm1, 224(%r8)
-; AVX512DQ-NEXT: vmovdqa %ymm7, 128(%r8)
-; AVX512DQ-NEXT: vmovdqa %ymm9, 160(%r8)
+; AVX512DQ-NEXT: vmovdqa64 %ymm5, 192(%r8)
+; AVX512DQ-NEXT: vmovdqa64 %ymm1, 224(%r8)
+; AVX512DQ-NEXT: vmovdqa64 %ymm7, 128(%r8)
+; AVX512DQ-NEXT: vmovdqa64 %ymm9, 160(%r8)
; AVX512DQ-NEXT: vmovdqa %ymm4, 64(%r8)
; AVX512DQ-NEXT: vmovdqa %ymm0, 96(%r8)
; AVX512DQ-NEXT: vmovdqa %ymm2, (%r8)
@@ -1658,10 +1658,10 @@ define void @store_i8_stride4_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm8, %ymm9
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
; AVX512DQ-FCP-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm8[2,3],ymm1[2,3]
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm5, 192(%r8)
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, 224(%r8)
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm7, 128(%r8)
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm9, 160(%r8)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm5, 192(%r8)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm1, 224(%r8)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, 128(%r8)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm9, 160(%r8)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, 64(%r8)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 96(%r8)
; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, (%r8)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index 480cfef7fda1e..f8d338009d6b0 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -2432,7 +2432,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,2,3,3,6,6,7,7]
; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm7))
-; AVX512-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512-NEXT: vmovdqa64 %zmm6, (%r9)
; AVX512-NEXT: vzeroupper
@@ -2503,7 +2503,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [6,6,6,0,7,7,7,7]
; AVX512-FCP-NEXT: vpermd %ymm4, %ymm0, %ymm0
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm7))
-; AVX512-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512-FCP-NEXT: vmovdqa64 %zmm6, 64(%r9)
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512-FCP-NEXT: vzeroupper
@@ -2575,7 +2575,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,2,3,3,6,6,7,7]
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm7))
-; AVX512DQ-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm5, 64(%r9)
; AVX512DQ-NEXT: vmovdqa64 %zmm6, (%r9)
; AVX512DQ-NEXT: vzeroupper
@@ -2646,7 +2646,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [6,6,6,0,7,7,7,7]
; AVX512DQ-FCP-NEXT: vpermd %ymm4, %ymm0, %ymm0
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm7))
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, 64(%r9)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, (%r9)
; AVX512DQ-FCP-NEXT: vzeroupper
@@ -2734,7 +2734,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: movl $-2078209982, %eax # imm = 0x84210842
; AVX512BW-NEXT: kmovd %eax, %k1
; AVX512BW-NEXT: vmovdqu8 %ymm0, %ymm1 {%k1}
-; AVX512BW-NEXT: vmovdqa %ymm1, 128(%r9)
+; AVX512BW-NEXT: vmovdqa64 %ymm1, 128(%r9)
; AVX512BW-NEXT: vmovdqa64 %zmm7, 64(%r9)
; AVX512BW-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512BW-NEXT: vzeroupper
@@ -2815,7 +2815,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-FCP-NEXT: movl $-2078209982, %eax # imm = 0x84210842
; AVX512BW-FCP-NEXT: kmovd %eax, %k1
; AVX512BW-FCP-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512BW-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, 64(%r9)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, (%r9)
; AVX512BW-FCP-NEXT: vzeroupper
@@ -2903,7 +2903,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: movl $-2078209982, %eax # imm = 0x84210842
; AVX512DQ-BW-NEXT: kmovd %eax, %k1
; AVX512DQ-BW-NEXT: vmovdqu8 %ymm0, %ymm1 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa %ymm1, 128(%r9)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm1, 128(%r9)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, 64(%r9)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%r9)
; AVX512DQ-BW-NEXT: vzeroupper
@@ -2984,7 +2984,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-FCP-NEXT: movl $-2078209982, %eax # imm = 0x84210842
; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1
; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, 128(%r9)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm0, 128(%r9)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, 64(%r9)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, (%r9)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index ec4c2b0ca1cf5..800d8f3c9a6c5 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -4196,7 +4196,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3]
; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm0))
; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm1))
-; AVX512-NEXT: vmovdqa %ymm3, 192(%rax)
+; AVX512-NEXT: vmovdqa64 %ymm3, 192(%rax)
; AVX512-NEXT: vmovdqa64 %zmm15, 128(%rax)
; AVX512-NEXT: vmovdqa64 %zmm17, (%rax)
; AVX512-NEXT: vmovdqa64 %zmm2, 64(%rax)
@@ -4326,7 +4326,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3]
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (mem & (ymm2 ^ ymm0))
; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (mem & (ymm2 ^ ymm1))
-; AVX512-FCP-NEXT: vmovdqa %ymm2, 192(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %ymm2, 192(%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm8, (%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm7, 128(%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm9, 64(%rax)
@@ -4461,7 +4461,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3]
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm0))
; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm3 ^ (mem & (ymm3 ^ ymm1))
-; AVX512DQ-NEXT: vmovdqa %ymm3, 192(%rax)
+; AVX512DQ-NEXT: vmovdqa64 %ymm3, 192(%rax)
; AVX512DQ-NEXT: vmovdqa64 %zmm15, 128(%rax)
; AVX512DQ-NEXT: vmovdqa64 %zmm17, (%rax)
; AVX512DQ-NEXT: vmovdqa64 %zmm2, 64(%rax)
@@ -4591,7 +4591,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3]
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (mem & (ymm2 ^ ymm0))
; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm2 ^ (mem & (ymm2 ^ ymm1))
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, 192(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm2, 192(%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, (%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm7, 128(%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, 64(%rax)
@@ -4734,7 +4734,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-NEXT: movl $-507279602, %ecx # imm = 0xE1C3870E
; AVX512BW-NEXT: kmovd %ecx, %k1
; AVX512BW-NEXT: vmovdqu8 %ymm1, %ymm6 {%k1}
-; AVX512BW-NEXT: vmovdqa %ymm6, 192(%rax)
+; AVX512BW-NEXT: vmovdqa64 %ymm6, 192(%rax)
; AVX512BW-NEXT: vmovdqa64 %zmm14, 128(%rax)
; AVX512BW-NEXT: vmovdqa64 %zmm0, 64(%rax)
; AVX512BW-NEXT: vzeroupper
@@ -4870,7 +4870,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512BW-FCP-NEXT: movl $-507279602, %ecx # imm = 0xE1C3870E
; AVX512BW-FCP-NEXT: kmovd %ecx, %k1
; AVX512BW-FCP-NEXT: vmovdqu8 %ymm3, %ymm1 {%k1}
-; AVX512BW-FCP-NEXT: vmovdqa %ymm1, 192(%rax)
+; AVX512BW-FCP-NEXT: vmovdqa64 %ymm1, 192(%rax)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm10, 128(%rax)
; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, 64(%rax)
; AVX512BW-FCP-NEXT: vzeroupper
@@ -5012,7 +5012,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-NEXT: movl $-507279602, %ecx # imm = 0xE1C3870E
; AVX512DQ-BW-NEXT: kmovd %ecx, %k1
; AVX512DQ-BW-NEXT: vmovdqu8 %ymm1, %ymm6 {%k1}
-; AVX512DQ-BW-NEXT: vmovdqa %ymm6, 192(%rax)
+; AVX512DQ-BW-NEXT: vmovdqa64 %ymm6, 192(%rax)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm14, 128(%rax)
; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, 64(%rax)
; AVX512DQ-BW-NEXT: vzeroupper
@@ -5148,7 +5148,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-BW-FCP-NEXT: movl $-507279602, %ecx # imm = 0xE1C3870E
; AVX512DQ-BW-FCP-NEXT: kmovd %ecx, %k1
; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %ymm3, %ymm1 {%k1}
-; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, 192(%rax)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %ymm1, 192(%rax)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm10, 128(%rax)
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, 64(%rax)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
@@ -8380,7 +8380,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vpshufb %ymm2, %ymm4, %ymm1
; AVX512-NEXT: vmovdqa64 %ymm4, %ymm17
; AVX512-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa (%rcx), %ymm1
; AVX512-NEXT: vmovdqa {{.*#+}} ymm7 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
; AVX512-NEXT: vpshufb %ymm7, %ymm1, %ymm0
@@ -9112,7 +9112,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vpshufb %ymm2, %ymm4, %ymm1
; AVX512DQ-NEXT: vmovdqa64 %ymm4, %ymm17
; AVX512DQ-NEXT: vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa (%rcx), %ymm1
; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
; AVX512DQ-NEXT: vpshufb %ymm7, %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
index 2abc201705929..74e1b93049172 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-8.ll
@@ -6669,7 +6669,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm2[0,1,2,3,4,4,6,5]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa (%r10), %xmm5
; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 48(%r10), %xmm3
@@ -6680,11 +6680,11 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,0,2,1,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm2[0,2,2,3,4,5,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm5
-; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,4,6,5]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vmovdqa (%r9), %xmm5
; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512-NEXT: vmovdqa 48(%r9), %xmm6
@@ -6695,25 +6695,25 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,1,1,3,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,1,3,3,4,5,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm5
-; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,5,5,7]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,7,7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,0,2,1,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm2[0,2,2,3,4,5,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm5, %ymm5
-; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,4,6,5]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,0,2,1,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm2[0,2,2,3,4,5,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm5, %ymm5
-; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,4,6,5]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2
@@ -6777,25 +6777,25 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[0,0,2,1,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm5[0,2,2,3,4,5,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm6, %ymm4
-; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} xmm6 = xmm5[0,1,2,3,4,4,6,5]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm6, %ymm4
-; AVX512-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,0,2,1,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm0[0,2,2,3,4,5,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm2, %ymm2
-; AVX512-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,6,5]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
-; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,1,1,3,4,5,6,7]
; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[2,1,3,3,4,5,6,7]
; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,5,7]
; AVX512-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,7,7]
; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
@@ -7353,7 +7353,7 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm2[0,1,2,3,4,4,6,5]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa (%r10), %xmm5
; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa 48(%r10), %xmm3
@@ -7364,11 +7364,11 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,0,2,1,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm2[0,2,2,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm5
-; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,4,6,5]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vmovdqa (%r9), %xmm5
; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX512DQ-NEXT: vmovdqa 48(%r9), %xmm6
@@ -7379,25 +7379,25 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,1,1,3,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,1,3,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm5
-; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,5,5,7]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,7,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,0,2,1,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm2[0,2,2,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm5, %ymm5
-; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,4,6,5]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,0,2,1,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm2[0,2,2,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm5, %ymm5
-; AVX512DQ-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,4,6,5]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm5, %ymm2
@@ -7461,25 +7461,25 @@ define void @store_i8_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[0,0,2,1,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm5[0,2,2,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm6, %ymm4
-; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm6 = xmm5[0,1,2,3,4,4,6,5]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm6, %ymm4
-; AVX512DQ-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,0,2,1,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm0[0,2,2,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm2, %ymm2
-; AVX512DQ-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,6,5]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
-; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,1,1,3,4,5,6,7]
; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[2,1,3,3,4,5,6,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT: vmovdqu64 %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,5,7]
; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,7,7]
; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll b/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll
index c8d401c0f7b7b..fef8ff812dace 100644
--- a/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll
+++ b/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll
@@ -2030,7 +2030,7 @@ define void @mask_replication_factor5_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512F-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm1 {%k1} {z}
; AVX512F-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm2 {%k3} {z}
; AVX512F-ONLY-NEXT: vmovdqa64 %zmm2, 64(%rdx)
-; AVX512F-ONLY-NEXT: vmovdqa %ymm1, 128(%rdx)
+; AVX512F-ONLY-NEXT: vmovdqa64 %ymm1, 128(%rdx)
; AVX512F-ONLY-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512F-ONLY-NEXT: vzeroupper
; AVX512F-ONLY-NEXT: retq
@@ -2058,7 +2058,7 @@ define void @mask_replication_factor5_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512DQ-NEXT: vmovdqa32 128(%rsi), %zmm1 {%k3} {z}
; AVX512DQ-NEXT: vmovdqa32 64(%rsi), %zmm2 {%k2} {z}
; AVX512DQ-NEXT: vmovdqa64 %zmm2, 64(%rdx)
-; AVX512DQ-NEXT: vmovdqa %ymm1, 128(%rdx)
+; AVX512DQ-NEXT: vmovdqa64 %ymm1, 128(%rdx)
; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
@@ -2080,7 +2080,7 @@ define void @mask_replication_factor5_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm2 {%k1} {z}
; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm2, 64(%rdx)
; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm1, (%rdx)
-; AVX512BW-ONLY-NEXT: vmovdqa %ymm0, 128(%rdx)
+; AVX512BW-ONLY-NEXT: vmovdqa64 %ymm0, 128(%rdx)
; AVX512BW-ONLY-NEXT: vzeroupper
; AVX512BW-ONLY-NEXT: retq
;
@@ -2101,7 +2101,7 @@ define void @mask_replication_factor5_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm2 {%k1} {z}
; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm2, 64(%rdx)
; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm1, (%rdx)
-; AVX512VBMI-ONLY-NEXT: vmovdqa %ymm0, 128(%rdx)
+; AVX512VBMI-ONLY-NEXT: vmovdqa64 %ymm0, 128(%rdx)
; AVX512VBMI-ONLY-NEXT: vzeroupper
; AVX512VBMI-ONLY-NEXT: retq
%src.mask.padded = load <64 x i1>, ptr %in.maskvec, align 64
@@ -4080,7 +4080,7 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512F-SLOW-NEXT: vmovdqa64 %zmm3, 64(%rdx)
; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, 128(%rdx)
; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512F-SLOW-NEXT: vmovdqa %ymm1, 192(%rdx)
+; AVX512F-SLOW-NEXT: vmovdqa64 %ymm1, 192(%rdx)
; AVX512F-SLOW-NEXT: vzeroupper
; AVX512F-SLOW-NEXT: retq
;
@@ -4113,7 +4113,7 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512F-FAST-NEXT: vmovdqa32 64(%rsi), %zmm3 {%k3} {z}
; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 64(%rdx)
; AVX512F-FAST-NEXT: vmovdqa64 %zmm2, 128(%rdx)
-; AVX512F-FAST-NEXT: vmovdqa %ymm1, 192(%rdx)
+; AVX512F-FAST-NEXT: vmovdqa64 %ymm1, 192(%rdx)
; AVX512F-FAST-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512F-FAST-NEXT: vzeroupper
; AVX512F-FAST-NEXT: retq
@@ -4146,7 +4146,7 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512DQ-SLOW-NEXT: vmovdqa32 64(%rsi), %zmm3 {%k2} {z}
; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm3, 64(%rdx)
; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm2, 128(%rdx)
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm1, 192(%rdx)
+; AVX512DQ-SLOW-NEXT: vmovdqa64 %ymm1, 192(%rdx)
; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512DQ-SLOW-NEXT: vzeroupper
; AVX512DQ-SLOW-NEXT: retq
@@ -4179,7 +4179,7 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512DQ-FAST-NEXT: vmovdqa32 64(%rsi), %zmm3 {%k2} {z}
; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm3, 64(%rdx)
; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm2, 128(%rdx)
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm1, 192(%rdx)
+; AVX512DQ-FAST-NEXT: vmovdqa64 %ymm1, 192(%rdx)
; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512DQ-FAST-NEXT: vzeroupper
; AVX512DQ-FAST-NEXT: retq
@@ -4204,7 +4204,7 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm3, 64(%rdx)
; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm2, 128(%rdx)
; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm1, (%rdx)
-; AVX512BW-ONLY-NEXT: vmovdqa %ymm0, 192(%rdx)
+; AVX512BW-ONLY-NEXT: vmovdqa64 %ymm0, 192(%rdx)
; AVX512BW-ONLY-NEXT: vzeroupper
; AVX512BW-ONLY-NEXT: retq
;
@@ -4228,7 +4228,7 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out
; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm3, 64(%rdx)
; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm2, 128(%rdx)
; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm1, (%rdx)
-; AVX512VBMI-ONLY-NEXT: vmovdqa %ymm0, 192(%rdx)
+; AVX512VBMI-ONLY-NEXT: vmovdqa64 %ymm0, 192(%rdx)
; AVX512VBMI-ONLY-NEXT: vzeroupper
; AVX512VBMI-ONLY-NEXT: retq
%src.mask.padded = load <64 x i1>, ptr %in.maskvec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
index 5e3e74da1edf9..eaa053b1f58e3 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
@@ -553,10 +553,10 @@ define void @test_demandedelts_pshufb_v32i8_v16i8(ptr %src, ptr %dst) {
; X86-AVX512-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-AVX512-SLOW-NEXT: vpbroadcastd 44(%ecx), %xmm0
; X86-AVX512-SLOW-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-AVX512-SLOW-NEXT: vmovdqa %ymm0, 672(%eax)
+; X86-AVX512-SLOW-NEXT: vmovdqa64 %ymm0, 672(%eax)
; X86-AVX512-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = mem[1,0,2,3]
; X86-AVX512-SLOW-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-AVX512-SLOW-NEXT: vmovdqa %ymm0, 832(%eax)
+; X86-AVX512-SLOW-NEXT: vmovdqa64 %ymm0, 832(%eax)
; X86-AVX512-SLOW-NEXT: vzeroupper
; X86-AVX512-SLOW-NEXT: retl
;
@@ -564,10 +564,10 @@ define void @test_demandedelts_pshufb_v32i8_v16i8(ptr %src, ptr %dst) {
; X64-AVX512-SLOW: # %bb.0:
; X64-AVX512-SLOW-NEXT: vpbroadcastd 44(%rdi), %xmm0
; X64-AVX512-SLOW-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; X64-AVX512-SLOW-NEXT: vmovdqa %ymm0, 672(%rsi)
+; X64-AVX512-SLOW-NEXT: vmovdqa64 %ymm0, 672(%rsi)
; X64-AVX512-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = mem[1,0,2,3]
; X64-AVX512-SLOW-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; X64-AVX512-SLOW-NEXT: vmovdqa %ymm0, 832(%rsi)
+; X64-AVX512-SLOW-NEXT: vmovdqa64 %ymm0, 832(%rsi)
; X64-AVX512-SLOW-NEXT: vzeroupper
; X64-AVX512-SLOW-NEXT: retq
;
@@ -577,10 +577,10 @@ define void @test_demandedelts_pshufb_v32i8_v16i8(ptr %src, ptr %dst) {
; X86-AVX512-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-AVX512-FAST-NEXT: vpbroadcastd 44(%ecx), %xmm0
; X86-AVX512-FAST-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-AVX512-FAST-NEXT: vmovdqa %ymm0, 672(%eax)
-; X86-AVX512-FAST-NEXT: vmovdqa 208(%ecx), %xmm0
+; X86-AVX512-FAST-NEXT: vmovdqa64 %ymm0, 672(%eax)
+; X86-AVX512-FAST-NEXT: vmovdqa64 208(%ecx), %xmm0
; X86-AVX512-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7,0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero
-; X86-AVX512-FAST-NEXT: vmovdqa %ymm0, 832(%eax)
+; X86-AVX512-FAST-NEXT: vmovdqa64 %ymm0, 832(%eax)
; X86-AVX512-FAST-NEXT: vzeroupper
; X86-AVX512-FAST-NEXT: retl
;
@@ -588,10 +588,10 @@ define void @test_demandedelts_pshufb_v32i8_v16i8(ptr %src, ptr %dst) {
; X64-AVX512-FAST: # %bb.0:
; X64-AVX512-FAST-NEXT: vpbroadcastd 44(%rdi), %xmm0
; X64-AVX512-FAST-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; X64-AVX512-FAST-NEXT: vmovdqa %ymm0, 672(%rsi)
-; X64-AVX512-FAST-NEXT: vmovdqa 208(%rdi), %xmm0
+; X64-AVX512-FAST-NEXT: vmovdqa64 %ymm0, 672(%rsi)
+; X64-AVX512-FAST-NEXT: vmovdqa64 208(%rdi), %xmm0
; X64-AVX512-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7,0,1,2,3],zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX512-FAST-NEXT: vmovdqa %ymm0, 832(%rsi)
+; X64-AVX512-FAST-NEXT: vmovdqa64 %ymm0, 832(%rsi)
; X64-AVX512-FAST-NEXT: vzeroupper
; X64-AVX512-FAST-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v192.ll b/llvm/test/CodeGen/X86/vector-shuffle-v192.ll
index 43af01d07c77d..0fa49cae68364 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v192.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v192.ll
@@ -57,15 +57,15 @@ define <64 x i8> @f1(ptr %p0) {
; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [1,3,7,9,13,15,128,128,128,128,128,u,u,u,u,u]
; AVX512F-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT: vmovdqa 176(%rdi), %xmm2
+; AVX512F-NEXT: vmovdqa64 176(%rdi), %xmm2
; AVX512F-NEXT: vmovdqa {{.*#+}} xmm4 = [u,u,u,u,u,u,128,128,128,128,128,1,5,7,11,13]
; AVX512F-NEXT: vpshufb %xmm4, %xmm2, %xmm2
-; AVX512F-NEXT: vmovdqa 160(%rdi), %xmm5
+; AVX512F-NEXT: vmovdqa64 160(%rdi), %xmm5
; AVX512F-NEXT: vmovdqa {{.*#+}} xmm6 = [u,u,u,u,u,u,3,5,9,11,15,128,128,128,128,128]
; AVX512F-NEXT: vpshufb %xmm6, %xmm5, %xmm5
; AVX512F-NEXT: vpor %xmm2, %xmm5, %xmm2
; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-NEXT: vmovdqa 128(%rdi), %ymm5
+; AVX512F-NEXT: vmovdqa64 128(%rdi), %ymm5
; AVX512F-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,u,u,u,1,5,7,11,13,17,19,23,25,29,31,u,u,u,u,u,u,u,u,u,u]
; AVX512F-NEXT: vpblendw {{.*#+}} ymm2 = ymm5[0,1,2],ymm2[3,4,5,6,7],ymm5[8,9,10],ymm2[11,12,13,14,15]
; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
@@ -98,15 +98,15 @@ define <64 x i8> @f1(ptr %p0) {
; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [1,3,7,9,13,15,128,128,128,128,128,u,u,u,u,u]
; AVX512BW-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512BW-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm2
+; AVX512BW-NEXT: vmovdqa64 176(%rdi), %xmm2
; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm4 = [u,u,u,u,u,u,128,128,128,128,128,1,5,7,11,13]
; AVX512BW-NEXT: vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm5
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm5
; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm6 = [u,u,u,u,u,u,3,5,9,11,15,128,128,128,128,128]
; AVX512BW-NEXT: vpshufb %xmm6, %xmm5, %xmm5
; AVX512BW-NEXT: vpor %xmm2, %xmm5, %xmm2
; AVX512BW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm5
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm5
; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm7 = [1,3,7,9,13,15,0,0,0,0,0,1,5,7,11,13,1,3,7,9,13,15,0,0,0,0,0,1,5,7,11,13]
; AVX512BW-NEXT: # ymm7 = mem[0,1,0,1]
; AVX512BW-NEXT: vpshufb %ymm7, %ymm5, %ymm5
@@ -208,9 +208,9 @@ define <64 x i8> @f2(ptr %p0) {
; AVX512F-NEXT: vpshufb %xmm6, %xmm5, %xmm5
; AVX512F-NEXT: vpor %xmm2, %xmm5, %xmm2
; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm0[0,1,0,1]
-; AVX512F-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX512F-NEXT: vmovdqa64 176(%rdi), %xmm0
; AVX512F-NEXT: vpshufb %xmm4, %xmm0, %xmm0
-; AVX512F-NEXT: vmovdqa 160(%rdi), %xmm4
+; AVX512F-NEXT: vmovdqa64 160(%rdi), %xmm4
; AVX512F-NEXT: vpshufb %xmm6, %xmm4, %xmm4
; AVX512F-NEXT: vpor %xmm0, %xmm4, %xmm0
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
@@ -222,7 +222,7 @@ define <64 x i8> @f2(ptr %p0) {
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[3,5,9,11,15,17,21,23,27,29,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512F-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512F-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,3,5,9,11,15,17,21,23,27,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
@@ -231,10 +231,10 @@ define <64 x i8> @f2(ptr %p0) {
;
; AVX512BW-LABEL: f2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX512BW-NEXT: vmovdqa64 176(%rdi), %xmm0
; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm1 = [u,u,u,u,u,128,128,128,128,128,128,3,5,9,11,15]
; AVX512BW-NEXT: vpshufb %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm2
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm2
; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,1,3,7,9,13,15,128,128,128,128,128]
; AVX512BW-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512BW-NEXT: vpor %xmm0, %xmm2, %xmm0
@@ -327,15 +327,15 @@ define <64 x i8> @f3(ptr %p0) {
;
; AVX512F-LABEL: f3:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa 160(%rdi), %xmm0
+; AVX512F-NEXT: vmovdqa64 160(%rdi), %xmm0
; AVX512F-NEXT: vmovdqa {{.*#+}} xmm1 = [u,u,u,u,u,0,4,6,10,12,128,128,128,128,128,128]
; AVX512F-NEXT: vpshufb %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vmovdqa 176(%rdi), %xmm2
+; AVX512F-NEXT: vmovdqa64 176(%rdi), %xmm2
; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,128,128,128,128,128,0,2,6,8,12,14]
; AVX512F-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-NEXT: vmovdqa 128(%rdi), %ymm2
+; AVX512F-NEXT: vmovdqa64 128(%rdi), %ymm2
; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [2,4,8,10,14,0,0,0,0,0,0,2,6,8,12,14,2,4,8,10,14,0,0,0,0,0,0,2,6,8,12,14]
; AVX512F-NEXT: # ymm4 = mem[0,1,0,1]
; AVX512F-NEXT: vpshufb %ymm4, %ymm2, %ymm2
@@ -400,13 +400,13 @@ define <64 x i8> @f3(ptr %p0) {
; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm4
; AVX512BW-NEXT: vpshufb %xmm6, %xmm4, %xmm4
; AVX512BW-NEXT: vpor %xmm0, %xmm4, %xmm0
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm4
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm4
; AVX512BW-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm4
+; AVX512BW-NEXT: vmovdqa64 176(%rdi), %xmm4
; AVX512BW-NEXT: vpshufb %xmm3, %xmm4, %xmm3
; AVX512BW-NEXT: vpor %xmm1, %xmm3, %xmm1
; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512BW-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512BW-NEXT: vpshufb %ymm7, %ymm3, %ymm3
; AVX512BW-NEXT: vmovdqu8 %ymm1, %ymm3 {%k1}
; AVX512BW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5,6,7]
@@ -487,9 +487,9 @@ define <64 x i8> @f4(ptr %p0) {
; AVX512F-NEXT: vpshufb %xmm6, %xmm5, %xmm5
; AVX512F-NEXT: vpor %xmm2, %xmm5, %xmm2
; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,0,1],zmm0[0,1,0,1]
-; AVX512F-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX512F-NEXT: vmovdqa64 176(%rdi), %xmm0
; AVX512F-NEXT: vpshufb %xmm4, %xmm0, %xmm0
-; AVX512F-NEXT: vmovdqa 160(%rdi), %xmm4
+; AVX512F-NEXT: vmovdqa64 160(%rdi), %xmm4
; AVX512F-NEXT: vpshufb %xmm6, %xmm4, %xmm4
; AVX512F-NEXT: vpor %xmm0, %xmm4, %xmm0
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
@@ -501,7 +501,7 @@ define <64 x i8> @f4(ptr %p0) {
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm1
; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm0
; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[2,4,8,10,14,16,20,22,26,28,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT: vmovdqa 128(%rdi), %ymm3
+; AVX512F-NEXT: vmovdqa64 128(%rdi), %ymm3
; AVX512F-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,2,4,8,10,14,16,20,22,26,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
@@ -510,10 +510,10 @@ define <64 x i8> @f4(ptr %p0) {
;
; AVX512BW-LABEL: f4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX512BW-NEXT: vmovdqa64 176(%rdi), %xmm0
; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm1 = [u,u,u,u,u,128,128,128,128,128,128,2,4,8,10,14]
; AVX512BW-NEXT: vpshufb %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovdqa 160(%rdi), %xmm2
+; AVX512BW-NEXT: vmovdqa64 160(%rdi), %xmm2
; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [u,u,u,u,u,0,2,6,8,12,14,128,128,128,128,128]
; AVX512BW-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512BW-NEXT: vpor %xmm0, %xmm2, %xmm0
More information about the llvm-commits
mailing list