[llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 27 04:58:34 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/180573
>From 0e984e7cab098f884dcf317a714962ab73918f9d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 9 Feb 2026 17:46:25 +0000
Subject: [PATCH] [DAG] SelectionDAGBuilder::visitShuffleVector - split
shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns
If an unary shuffle sources are twice the width of the result, then split the lower/upper subvectors and perform a binary shuffle instead of scalarizing
Fixes #88030
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 29 +
llvm/test/CodeGen/AArch64/addp-shuffle.ll | 12 +-
.../complex-deinterleaving-mixed-cases.ll | 146 +-
.../complex-deinterleaving-multiuses.ll | 142 +-
.../complex-deinterleaving-uniform-cases.ll | 34 +-
.../CodeGen/AArch64/highextractbitcast.ll | 12 +-
.../sve-fixed-length-extract-subvector.ll | 7 +-
...ffer-fat-pointers-contents-legalization.ll | 44 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 288 ++--
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 186 +--
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 239 ++--
.../AMDGPU/load-local-redundant-copies.ll | 33 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr.ll | 54 +-
.../AMDGPU/shufflevector.v2f32.v4f32.ll | 16 +-
.../AMDGPU/shufflevector.v2i32.v4i32.ll | 16 +-
.../CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll | 16 +-
llvm/test/CodeGen/ARM/bf16-shuffle.ll | 3 +-
llvm/test/CodeGen/ARM/vext.ll | 5 +-
llvm/test/CodeGen/ARM/vpadd.ll | 27 +-
llvm/test/CodeGen/ARM/vuzp.ll | 10 +-
llvm/test/CodeGen/ARM/vzip.ll | 5 +-
.../fixed-vectors-shuffle-changes-length.ll | 114 +-
.../rvv/fixed-vectors-shuffle-deinterleave.ll | 159 +--
.../fixed-vectors-shuffle-deinterleave2.ll | 152 ++-
llvm/test/CodeGen/Thumb2/mve-shuffle.ll | 8 +-
.../any_extend_vector_inreg_of_broadcast.ll | 268 ++--
...d_vector_inreg_of_broadcast_from_memory.ll | 174 +--
.../X86/avx512-shuffles/partial_permute.ll | 1182 +++++++----------
llvm/test/CodeGen/X86/madd.ll | 377 +++++-
llvm/test/CodeGen/X86/pclmulqdq.ll | 26 +-
llvm/test/CodeGen/X86/pmaddubsw.ll | 352 ++++-
llvm/test/CodeGen/X86/pmul.ll | 6 +-
.../test/CodeGen/X86/setcc-non-simple-type.ll | 35 +-
.../X86/shuffle-strided-with-offset-256.ll | 180 ++-
.../X86/shuffle-strided-with-offset-512.ll | 120 +-
llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll | 541 ++------
llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll | 149 ++-
llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll | 17 +-
llvm/test/CodeGen/X86/sse2.ll | 42 +-
.../vector-interleaved-load-i16-stride-2.ll | 195 ++-
.../vector-interleaved-load-i32-stride-2.ll | 82 +-
.../vector-interleaved-load-i64-stride-2.ll | 210 ++-
.../vector-interleaved-load-i8-stride-2.ll | 452 +++----
llvm/test/CodeGen/X86/vector-narrow-binop.ll | 4 +-
.../CodeGen/X86/vector-shuffle-512-v16.ll | 19 +-
llvm/test/CodeGen/X86/vpdpwssd.ll | 85 +-
llvm/test/CodeGen/X86/vselect-avx.ll | 10 +-
47 files changed, 3097 insertions(+), 3186 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index bacea075da00c..c7f4647e0da9f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4280,6 +4280,35 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
assert(SrcNumElts > MaskNumElts);
+ // See if we can recreate a binary shuffle by splitting or widening an unary
+ // shuffle with sources twice the destination size.
+ if (SrcNumElts == (MaskNumElts * 2) && Src2.isUndef() &&
+ TLI.isTypeLegal(VT)) {
+ SmallVector<int, 16> SplitMask(Mask.begin(), Mask.end());
+ for (int &M : SplitMask)
+ M = M >= (int)SrcNumElts ? -1 : M;
+ bool LoOnly = all_of(SplitMask,
+ [MaskNumElts](int M) { return M < (int)MaskNumElts; });
+ bool HiOnly = all_of(SplitMask, [MaskNumElts](int M) {
+ return M < 0 || M >= (int)MaskNumElts;
+ });
+ if (LoOnly || HiOnly || Src1.getOpcode() == ISD::CONCAT_VECTORS) {
+ SDValue LHS = DAG.getExtractSubvector(DL, VT, Src1, 0);
+ SDValue RHS = DAG.getExtractSubvector(DL, VT, Src1, MaskNumElts);
+ SDValue Result = DAG.getVectorShuffle(VT, DL, LHS, RHS, SplitMask);
+ setValue(&I, Result);
+ return;
+ }
+ if (TLI.isTypeLegal(SrcVT) || isa<LoadSDNode>(Src1)) {
+ SmallVector<int, 16> WideMask(Mask.begin(), Mask.end());
+ WideMask.append(SrcNumElts - MaskNumElts, -1);
+ SDValue Wide = DAG.getVectorShuffle(SrcVT, DL, Src1, Src2, WideMask);
+ SDValue Result = DAG.getExtractSubvector(DL, VT, Wide, 0);
+ setValue(&I, Result);
+ return;
+ }
+ }
+
// Analyze the access pattern of the vector to see if we can extract
// two subvectors and do the shuffle.
int StartIdx[2] = {-1, -1}; // StartIdx to extract from
diff --git a/llvm/test/CodeGen/AArch64/addp-shuffle.ll b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
index cf02a090b8e3c..70949bb64180d 100644
--- a/llvm/test/CodeGen/AArch64/addp-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
@@ -27,8 +27,9 @@ define <4 x i32> @deinterleave_shuffle_v8i32_c(<8 x i32> %a) {
define <2 x i32> @deinterleave_shuffle_v4i32(<4 x i32> %a) {
; CHECK-LABEL: deinterleave_shuffle_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: addp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: xtn v1.2s, v0.2d
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: add v0.2s, v1.2s, v0.2s
; CHECK-NEXT: ret
%r0 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
%r1 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
@@ -387,10 +388,9 @@ define <4 x i32> @or_deinterleave_shuffle_v8i32_c(<8 x i32> %a) {
define <2 x i32> @or_deinterleave_shuffle_v4i32(<4 x i32> %a) {
; CHECK-LABEL: or_deinterleave_shuffle_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-NEXT: xtn v1.2s, v0.2d
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: orr v0.8b, v1.8b, v0.8b
; CHECK-NEXT: ret
%r0 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
%r1 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
index 1ed9cf2db24f7..5d89a51597bf1 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
@@ -44,16 +44,13 @@ define <4 x float> @add_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: fsub v0.4s, v1.4s, v0.4s
; CHECK-NEXT: fsub v1.4s, v1.4s, v2.4s
-; CHECK-NEXT: ext v3.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v5.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip2 v4.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: fmul v5.2s, v4.2s, v0.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fneg v4.2s, v5.2s
+; CHECK-NEXT: uzp2 v3.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v4.2s, v3.2s, v0.2s
+; CHECK-NEXT: uzp1 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmul v3.2s, v1.2s, v3.2s
+; CHECK-NEXT: fneg v4.2s, v4.2s
; CHECK-NEXT: fmla v3.2s, v0.2s, v2.2s
; CHECK-NEXT: fmla v4.2s, v1.2s, v2.2s
; CHECK-NEXT: zip1 v0.4s, v4.4s, v3.4s
@@ -79,24 +76,21 @@ entry:
define <4 x float> @mul_mul270_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: mul_mul270_mul:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v3.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v5.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: fmul v7.2s, v6.2s, v5.2s
-; CHECK-NEXT: fneg v4.2s, v7.2s
-; CHECK-NEXT: zip2 v7.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
-; CHECK-NEXT: fmla v4.2s, v2.2s, v1.2s
-; CHECK-NEXT: fmul v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v4.2s, v7.2s
-; CHECK-NEXT: fmla v1.2s, v2.2s, v6.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v0.2s
+; CHECK-NEXT: uzp1 v3.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v6.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v5.2s, v4.2s, v3.2s
+; CHECK-NEXT: fneg v5.2s, v5.2s
+; CHECK-NEXT: fmla v5.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmul v1.2s, v1.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v5.2s, v6.2s
+; CHECK-NEXT: fmla v1.2s, v2.2s, v4.2s
+; CHECK-NEXT: fmul v2.2s, v5.2s, v0.2s
; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v2.2s, v7.2s, v1.2s
+; CHECK-NEXT: fmla v2.2s, v6.2s, v1.2s
; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
; CHECK-NEXT: zip1 v0.4s, v3.4s, v2.4s
; CHECK-NEXT: ret
@@ -258,25 +252,22 @@ entry:
define <4 x float> @mul_triangle_addmul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: mul_triangle_addmul:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v5.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
-; CHECK-NEXT: ext v4.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v3.2s
-; CHECK-NEXT: fmul v7.2s, v6.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v4.2s
-; CHECK-NEXT: fmov d4, d7
-; CHECK-NEXT: fmov d16, d5
-; CHECK-NEXT: fmls v7.2s, v0.2s, v2.2s
-; CHECK-NEXT: fmla v5.2s, v0.2s, v3.2s
-; CHECK-NEXT: fmls v4.2s, v0.2s, v1.2s
-; CHECK-NEXT: fmla v16.2s, v0.2s, v6.2s
-; CHECK-NEXT: fsub v0.2s, v7.2s, v16.2s
-; CHECK-NEXT: fadd v1.2s, v5.2s, v4.2s
+; CHECK-NEXT: uzp1 v3.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v5.2s, v4.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v1.2s, v3.2s
+; CHECK-NEXT: uzp1 v6.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmov d7, d5
+; CHECK-NEXT: fmov d16, d3
+; CHECK-NEXT: fmls v5.2s, v0.2s, v2.2s
+; CHECK-NEXT: fmla v3.2s, v0.2s, v6.2s
+; CHECK-NEXT: fmls v7.2s, v0.2s, v1.2s
+; CHECK-NEXT: fmla v16.2s, v0.2s, v4.2s
+; CHECK-NEXT: fsub v0.2s, v5.2s, v16.2s
+; CHECK-NEXT: fadd v1.2s, v3.2s, v7.2s
; CHECK-NEXT: zip1 v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
entry:
@@ -311,24 +302,22 @@ entry:
define <4 x float> @mul_triangle_multiuses(<4 x float> %a, <4 x float> %b, ptr %p) {
; CHECK-LABEL: mul_triangle_multiuses:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v2.2s, v0.2s
-; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
-; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: mov v1.d[1], v2.d[0]
-; CHECK-NEXT: zip1 v0.4s, v3.4s, v5.4s
+; CHECK-NEXT: uzp2 v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v4.2s, v2.2s, v3.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v2.2s
+; CHECK-NEXT: fmla v4.2s, v0.2s, v1.2s
+; CHECK-NEXT: fneg v1.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v4.2s, v2.2s
+; CHECK-NEXT: fmla v1.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v4.2s, v0.2s
+; CHECK-NEXT: fneg v5.2s, v5.2s
+; CHECK-NEXT: fmla v3.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
+; CHECK-NEXT: mov v1.d[1], v4.d[0]
+; CHECK-NEXT: zip1 v0.4s, v5.4s, v3.4s
; CHECK-NEXT: str q1, [x0]
; CHECK-NEXT: ret
entry:
@@ -442,23 +431,20 @@ entry:
define <4 x float> @mul_divequal(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: mul_divequal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip2 v5.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip2 v6.2s, v1.2s, v4.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v4.2s
-; CHECK-NEXT: ext v3.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: fmul v7.2s, v5.2s, v6.2s
-; CHECK-NEXT: fneg v4.2s, v7.2s
-; CHECK-NEXT: zip1 v7.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: fmla v4.2s, v0.2s, v1.2s
-; CHECK-NEXT: fmul v0.2s, v6.2s, v0.2s
-; CHECK-NEXT: fmla v0.2s, v5.2s, v1.2s
-; CHECK-NEXT: fdiv v4.2s, v4.2s, v7.2s
+; CHECK-NEXT: uzp2 v3.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v4.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v5.2s, v3.2s, v4.2s
+; CHECK-NEXT: uzp1 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v6.4s, v2.4s, v0.4s
+; CHECK-NEXT: fneg v5.2s, v5.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
+; CHECK-NEXT: fmul v0.2s, v4.2s, v0.2s
+; CHECK-NEXT: uzp2 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmla v0.2s, v3.2s, v1.2s
+; CHECK-NEXT: fdiv v5.2s, v5.2s, v6.2s
; CHECK-NEXT: fdiv v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v0.4s, v4.4s, v0.4s
+; CHECK-NEXT: zip1 v0.4s, v5.4s, v0.4s
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
index 039025dafa0d6..5654feb5a4356 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
@@ -46,24 +46,22 @@ entry:
define <4 x float> @mul_triangle_external_use(<4 x float> %a, <4 x float> %b, ptr %p) {
; CHECK-LABEL: mul_triangle_external_use:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: str d2, [x0]
-; CHECK-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v2.2s, v0.2s
-; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
-; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: zip1 v0.4s, v3.4s, v5.4s
+; CHECK-NEXT: uzp2 v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v4.2s, v2.2s, v3.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v2.2s
+; CHECK-NEXT: fmla v4.2s, v0.2s, v1.2s
+; CHECK-NEXT: fneg v1.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v4.2s, v2.2s
+; CHECK-NEXT: str d4, [x0]
+; CHECK-NEXT: fmla v1.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v4.2s, v0.2s
+; CHECK-NEXT: fneg v5.2s, v5.2s
+; CHECK-NEXT: fmla v3.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
+; CHECK-NEXT: zip1 v0.4s, v5.4s, v3.4s
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -94,29 +92,26 @@ entry:
define <4 x float> @multiple_muls_shuffle_external(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
; CHECK-LABEL: multiple_muls_shuffle_external:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v5.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v6.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: ext v4.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: zip2 v7.2s, v0.2s, v5.2s
-; CHECK-NEXT: zip1 v16.2s, v1.2s, v6.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v6.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v16.2s, v7.2s
-; CHECK-NEXT: fmul v6.2s, v1.2s, v7.2s
-; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v6.2s
-; CHECK-NEXT: zip1 v6.2s, v2.2s, v4.2s
-; CHECK-NEXT: zip2 v4.2s, v2.2s, v4.2s
-; CHECK-NEXT: fmla v1.2s, v0.2s, v16.2s
-; CHECK-NEXT: fmul v17.2s, v6.2s, v5.2s
+; CHECK-NEXT: uzp2 v5.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v6.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v4.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v7.2s, v6.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v5.2s
+; CHECK-NEXT: fmla v7.2s, v0.2s, v1.2s
+; CHECK-NEXT: fneg v1.2s, v5.2s
+; CHECK-NEXT: uzp2 v5.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmla v1.2s, v0.2s, v6.2s
+; CHECK-NEXT: fmul v17.2s, v4.2s, v7.2s
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: fmul v5.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmla v17.2s, v1.2s, v4.2s
+; CHECK-NEXT: fmul v6.2s, v5.2s, v7.2s
+; CHECK-NEXT: fmla v17.2s, v1.2s, v5.2s
; CHECK-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
; CHECK-NEXT: str d1, [x0]
-; CHECK-NEXT: fneg v16.2s, v5.2s
+; CHECK-NEXT: fneg v16.2s, v6.2s
; CHECK-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-NEXT: fmla v16.2s, v1.2s, v6.2s
+; CHECK-NEXT: fmla v16.2s, v1.2s, v4.2s
; CHECK-NEXT: st2 { v16.2s, v17.2s }, [x1]
; CHECK-NEXT: ret
entry:
@@ -162,25 +157,24 @@ define <4 x float> @multiple_muls_shuffle_external_with_loads(ptr %ptr_a, ptr %p
; CHECK-NEXT: ld2 { v0.2s, v1.2s }, [x0]
; CHECK-NEXT: ld2 { v2.2s, v3.2s }, [x1]
; CHECK-NEXT: fmul v4.2s, v3.2s, v1.2s
-; CHECK-NEXT: fmul v6.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmul v5.2s, v2.2s, v1.2s
; CHECK-NEXT: fneg v4.2s, v4.2s
-; CHECK-NEXT: fmla v6.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v3.2s
; CHECK-NEXT: fmla v4.2s, v0.2s, v2.2s
; CHECK-NEXT: str d4, [x4]
-; CHECK-NEXT: ldr q5, [x2]
-; CHECK-NEXT: ext v7.16b, v5.16b, v5.16b, #8
-; CHECK-NEXT: zip1 v0.2s, v5.2s, v7.2s
-; CHECK-NEXT: zip2 v1.2s, v5.2s, v7.2s
-; CHECK-NEXT: fmul v3.2s, v0.2s, v6.2s
-; CHECK-NEXT: fmul v6.2s, v1.2s, v6.2s
+; CHECK-NEXT: ldr q6, [x2]
+; CHECK-NEXT: uzp1 v0.4s, v6.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v6.4s, v0.4s
+; CHECK-NEXT: fmul v3.2s, v0.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v5.2s
; CHECK-NEXT: fmla v3.2s, v4.2s, v1.2s
-; CHECK-NEXT: fneg v2.2s, v6.2s
+; CHECK-NEXT: fneg v2.2s, v5.2s
; CHECK-NEXT: fmla v2.2s, v4.2s, v0.2s
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: st2 { v2.2s, v3.2s }, [x5]
; CHECK-NEXT: ldr q1, [x3]
-; CHECK-NEXT: fcmla v0.4s, v5.4s, v1.4s, #0
-; CHECK-NEXT: fcmla v0.4s, v5.4s, v1.4s, #90
+; CHECK-NEXT: fcmla v0.4s, v6.4s, v1.4s, #0
+; CHECK-NEXT: fcmla v0.4s, v6.4s, v1.4s, #90
; CHECK-NEXT: ret
entry:
%a = load <4 x float>, ptr %ptr_a
@@ -227,36 +221,32 @@ entry:
define <4 x float> @multiple_muls_mul_external(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
; CHECK-LABEL: multiple_muls_mul_external:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v5.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: ext v16.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: ext v17.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT: zip2 v6.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip2 v7.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v19.2s, v2.2s, v16.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v16.2s
-; CHECK-NEXT: zip2 v16.2s, v3.2s, v17.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v3.2s, v3.2s, v17.2s
-; CHECK-NEXT: fmul v18.2s, v6.2s, v7.2s
-; CHECK-NEXT: fmul v5.2s, v19.2s, v16.2s
+; CHECK-NEXT: uzp2 v4.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v5.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v7.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v16.4s, v3.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v6.2s, v4.2s, v5.2s
+; CHECK-NEXT: uzp1 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v3.4s, v0.4s
+; CHECK-NEXT: fmul v17.2s, v7.2s, v16.2s
; CHECK-NEXT: fmul v16.2s, v2.2s, v16.2s
-; CHECK-NEXT: fmul v7.2s, v0.2s, v7.2s
-; CHECK-NEXT: fneg v4.2s, v18.2s
-; CHECK-NEXT: fmla v5.2s, v3.2s, v2.2s
+; CHECK-NEXT: fmul v5.2s, v0.2s, v5.2s
+; CHECK-NEXT: fneg v6.2s, v6.2s
+; CHECK-NEXT: fmla v17.2s, v3.2s, v2.2s
; CHECK-NEXT: fneg v2.2s, v16.2s
-; CHECK-NEXT: fmla v7.2s, v1.2s, v6.2s
-; CHECK-NEXT: fmla v4.2s, v1.2s, v0.2s
-; CHECK-NEXT: fmla v2.2s, v3.2s, v19.2s
-; CHECK-NEXT: fmul v0.2s, v7.2s, v5.2s
-; CHECK-NEXT: fmul v17.2s, v4.2s, v5.2s
-; CHECK-NEXT: str d4, [x0]
-; CHECK-NEXT: fmla v17.2s, v2.2s, v7.2s
-; CHECK-NEXT: fneg v16.2s, v0.2s
-; CHECK-NEXT: zip1 v0.4s, v2.4s, v5.4s
-; CHECK-NEXT: fmla v16.2s, v2.2s, v4.2s
-; CHECK-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-NEXT: fmla v5.2s, v1.2s, v4.2s
+; CHECK-NEXT: fmla v6.2s, v1.2s, v0.2s
+; CHECK-NEXT: fmla v2.2s, v3.2s, v7.2s
+; CHECK-NEXT: fmul v0.2s, v5.2s, v17.2s
+; CHECK-NEXT: fmul v4.2s, v6.2s, v17.2s
+; CHECK-NEXT: str d6, [x0]
+; CHECK-NEXT: fmla v4.2s, v2.2s, v5.2s
+; CHECK-NEXT: fneg v3.2s, v0.2s
+; CHECK-NEXT: zip1 v0.4s, v2.4s, v17.4s
+; CHECK-NEXT: fmla v3.2s, v2.2s, v6.2s
+; CHECK-NEXT: st2 { v3.2s, v4.2s }, [x1]
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
index 13434fabefa78..6ec1517cc1177 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
@@ -31,18 +31,16 @@ entry:
define <4 x float> @simple_mul_no_contract(<4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: simple_mul_no_contract:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v2.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmul v4.2s, v2.2s, v4.2s
+; CHECK-NEXT: uzp1 v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v4.2s, v1.2s, v2.2s
+; CHECK-NEXT: fmul v2.2s, v3.2s, v2.2s
; CHECK-NEXT: fmul v1.2s, v0.2s, v1.2s
-; CHECK-NEXT: fmla v3.2s, v0.2s, v2.2s
-; CHECK-NEXT: fsub v0.2s, v4.2s, v1.2s
-; CHECK-NEXT: zip1 v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: fmla v4.2s, v0.2s, v3.2s
+; CHECK-NEXT: fsub v0.2s, v2.2s, v1.2s
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v4.4s
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -149,14 +147,12 @@ entry:
define <4 x float> @add_external_use(<4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: add_external_use:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v2.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fadd v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: fsub v1.2s, v4.2s, v1.2s
+; CHECK-NEXT: uzp1 v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fsub v1.2s, v2.2s, v1.2s
+; CHECK-NEXT: fadd v0.2s, v0.2s, v3.2s
; CHECK-NEXT: zip1 v0.4s, v1.4s, v0.4s
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/highextractbitcast.ll b/llvm/test/CodeGen/AArch64/highextractbitcast.ll
index de2911b8a5127..97e878e7d3ebd 100644
--- a/llvm/test/CodeGen/AArch64/highextractbitcast.ll
+++ b/llvm/test/CodeGen/AArch64/highextractbitcast.ll
@@ -140,9 +140,8 @@ entry:
define <4 x i32> @test_smull_high_s16_bitcasta1_wrongindex(<2 x i64> %aa, <8 x i16> %b) #0 {
; CHECK-LE-LABEL: test_smull_high_s16_bitcasta1_wrongindex:
; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-LE-NEXT: ext v0.16b, v0.16b, v0.16b, #4
; CHECK-LE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-LE-NEXT: ext v0.8b, v0.8b, v2.8b, #4
; CHECK-LE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-LE-NEXT: ret
;
@@ -176,9 +175,8 @@ entry:
define <4 x i32> @test_smull_high_s16_bitcastb1_wrongindex(<8 x i16> %a, <16 x i8> %bb) #0 {
; CHECK-LE-LABEL: test_smull_high_s16_bitcastb1_wrongindex:
; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
; CHECK-LE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-LE-NEXT: ext v1.8b, v1.8b, v2.8b, #6
+; CHECK-LE-NEXT: ext v1.16b, v1.16b, v0.16b, #6
; CHECK-LE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-LE-NEXT: ret
;
@@ -189,7 +187,7 @@ define <4 x i32> @test_smull_high_s16_bitcastb1_wrongindex(<8 x i16> %a, <16 x i
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #6
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v0.16b, #6
; CHECK-BE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
@@ -249,7 +247,7 @@ define <4 x i32> @test_smull_high_s16_bitcastb2_wrongindex(<8 x i16> %a, <16 x i
; CHECK-LE-LABEL: test_smull_high_s16_bitcastb2_wrongindex:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-LE-NEXT: ext v1.16b, v1.16b, v1.16b, #4
+; CHECK-LE-NEXT: ext v1.16b, v1.16b, v0.16b, #4
; CHECK-LE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-LE-NEXT: ret
;
@@ -259,7 +257,7 @@ define <4 x i32> @test_smull_high_s16_bitcastb2_wrongindex(<8 x i16> %a, <16 x i
; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #4
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v0.16b, #4
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: rev16 v1.8b, v1.8b
; CHECK-BE-NEXT: smull v0.4s, v0.4h, v1.4h
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
index 6b082b1762cc8..dad67750f4e1d 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
@@ -67,10 +67,11 @@ entry:
define void @extract_v32i8_half_unaligned(ptr %in, ptr %out) #0 vscale_range(2,2) {
; CHECK-LABEL: extract_v32i8_half_unaligned:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: adrp x8, .LCPI4_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI4_0
; CHECK-NEXT: ldr z0, [x0]
-; CHECK-NEXT: movprfx z1, z0
-; CHECK-NEXT: ext z1.b, z1.b, z0.b, #16
-; CHECK-NEXT: ext v0.16b, v0.16b, v1.16b, #4
+; CHECK-NEXT: ldr z1, [x8]
+; CHECK-NEXT: tbl z0.s, { z0.s }, z1.s
; CHECK-NEXT: str q0, [x1]
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index 021b15ef09bee..8d18c6dbd9339 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2872,34 +2872,34 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: s_mov_b32 s4, 0xc0c0004
-; SDAG-NEXT: v_perm_b32 v8, v8, v9, s4
-; SDAG-NEXT: v_perm_b32 v9, v10, v11, s4
-; SDAG-NEXT: buffer_load_ubyte v10, off, s[0:3], s32
+; SDAG-NEXT: v_perm_b32 v20, v20, v21, s4
+; SDAG-NEXT: v_perm_b32 v21, v22, v23, s4
+; SDAG-NEXT: buffer_load_ubyte v22, off, s[0:3], s32
; SDAG-NEXT: v_perm_b32 v12, v12, v13, s4
; SDAG-NEXT: v_perm_b32 v13, v14, v15, s4
+; SDAG-NEXT: v_perm_b32 v8, v8, v9, s4
+; SDAG-NEXT: v_perm_b32 v9, v10, v11, s4
; SDAG-NEXT: v_perm_b32 v4, v4, v5, s4
; SDAG-NEXT: v_perm_b32 v5, v6, v7, s4
-; SDAG-NEXT: v_perm_b32 v0, v0, v1, s4
-; SDAG-NEXT: v_perm_b32 v6, v2, v3, s4
-; SDAG-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; SDAG-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; SDAG-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; SDAG-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; SDAG-NEXT: v_perm_b32 v7, v28, v29, s4
-; SDAG-NEXT: v_perm_b32 v11, v24, v25, s4
-; SDAG-NEXT: v_perm_b32 v14, v26, v27, s4
-; SDAG-NEXT: v_perm_b32 v15, v20, v21, s4
-; SDAG-NEXT: v_perm_b32 v20, v22, v23, s4
+; SDAG-NEXT: v_perm_b32 v10, v0, v1, s4
+; SDAG-NEXT: v_perm_b32 v3, v2, v3, s4
+; SDAG-NEXT: v_perm_b32 v24, v24, v25, s4
+; SDAG-NEXT: v_perm_b32 v25, v26, v27, s4
+; SDAG-NEXT: v_perm_b32 v23, v28, v29, s4
; SDAG-NEXT: v_perm_b32 v16, v16, v17, s4
; SDAG-NEXT: v_perm_b32 v17, v18, v19, s4
-; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; SDAG-NEXT: v_lshl_or_b32 v5, v14, 16, v11
-; SDAG-NEXT: v_lshl_or_b32 v4, v20, 16, v15
-; SDAG-NEXT: v_lshl_or_b32 v3, v17, 16, v16
-; SDAG-NEXT: s_waitcnt vmcnt(1)
-; SDAG-NEXT: v_perm_b32 v0, v30, v10, s4
-; SDAG-NEXT: v_lshl_or_b32 v6, v0, 16, v7
-; SDAG-NEXT: buffer_store_dwordx4 v[3:6], off, s[16:19], 0 offset:16
+; SDAG-NEXT: v_lshl_or_b32 v7, v13, 16, v12
+; SDAG-NEXT: v_lshl_or_b32 v6, v9, 16, v8
+; SDAG-NEXT: v_lshl_or_b32 v5, v5, 16, v4
+; SDAG-NEXT: v_lshl_or_b32 v4, v3, 16, v10
+; SDAG-NEXT: v_lshl_or_b32 v2, v25, 16, v24
+; SDAG-NEXT: v_lshl_or_b32 v1, v21, 16, v20
+; SDAG-NEXT: v_lshl_or_b32 v0, v17, 16, v16
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_perm_b32 v3, v30, v22, s4
+; SDAG-NEXT: v_lshl_or_b32 v3, v3, 16, v23
+; SDAG-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0
+; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index a1fe463de1c54..4927b8110e5cc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -2859,15 +2859,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v36, s24
-; SDAG-NEXT: v_mov_b32_e32 v37, s25
-; SDAG-NEXT: v_mov_b32_e32 v38, s26
-; SDAG-NEXT: v_mov_b32_e32 v39, s27
+; SDAG-NEXT: v_mov_b32_e32 v40, s24
+; SDAG-NEXT: v_mov_b32_e32 v41, s25
+; SDAG-NEXT: v_mov_b32_e32 v42, s26
+; SDAG-NEXT: v_mov_b32_e32 v43, s27
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b32_e32 v40, s28
-; SDAG-NEXT: v_mov_b32_e32 v41, s29
-; SDAG-NEXT: v_mov_b32_e32 v42, s30
-; SDAG-NEXT: v_mov_b32_e32 v43, s31
+; SDAG-NEXT: v_mov_b32_e32 v36, s28
+; SDAG-NEXT: v_mov_b32_e32 v37, s29
+; SDAG-NEXT: v_mov_b32_e32 v38, s30
+; SDAG-NEXT: v_mov_b32_e32 v39, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -2876,7 +2876,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[40:43], v[36:39], v[16:31]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -2978,15 +2978,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v4, s24
-; HEURRC-NEXT: v_mov_b32_e32 v5, s25
-; HEURRC-NEXT: v_mov_b32_e32 v6, s26
-; HEURRC-NEXT: v_mov_b32_e32 v7, s27
+; HEURRC-NEXT: v_mov_b32_e32 v8, s24
+; HEURRC-NEXT: v_mov_b32_e32 v9, s25
+; HEURRC-NEXT: v_mov_b32_e32 v10, s26
+; HEURRC-NEXT: v_mov_b32_e32 v11, s27
; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
-; HEURRC-NEXT: v_mov_b32_e32 v8, s28
-; HEURRC-NEXT: v_mov_b32_e32 v9, s29
-; HEURRC-NEXT: v_mov_b32_e32 v10, s30
-; HEURRC-NEXT: v_mov_b32_e32 v11, s31
+; HEURRC-NEXT: v_mov_b32_e32 v4, s28
+; HEURRC-NEXT: v_mov_b32_e32 v5, s29
+; HEURRC-NEXT: v_mov_b32_e32 v6, s30
+; HEURRC-NEXT: v_mov_b32_e32 v7, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
@@ -3003,7 +3003,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[8:11], v[4:7], a[16:31]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
@@ -3049,15 +3049,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3066,7 +3066,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[40:43], v[36:39], v[16:31]
; VGPRRC-NEXT: s_nop 11
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
@@ -3260,15 +3260,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v36, s24
-; SDAG-NEXT: v_mov_b32_e32 v37, s25
-; SDAG-NEXT: v_mov_b32_e32 v38, s26
-; SDAG-NEXT: v_mov_b32_e32 v39, s27
+; SDAG-NEXT: v_mov_b32_e32 v40, s24
+; SDAG-NEXT: v_mov_b32_e32 v41, s25
+; SDAG-NEXT: v_mov_b32_e32 v42, s26
+; SDAG-NEXT: v_mov_b32_e32 v43, s27
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b32_e32 v40, s28
-; SDAG-NEXT: v_mov_b32_e32 v41, s29
-; SDAG-NEXT: v_mov_b32_e32 v42, s30
-; SDAG-NEXT: v_mov_b32_e32 v43, s31
+; SDAG-NEXT: v_mov_b32_e32 v36, s28
+; SDAG-NEXT: v_mov_b32_e32 v37, s29
+; SDAG-NEXT: v_mov_b32_e32 v38, s30
+; SDAG-NEXT: v_mov_b32_e32 v39, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3277,7 +3277,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[40:43], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -3379,15 +3379,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v4, s24
-; HEURRC-NEXT: v_mov_b32_e32 v5, s25
-; HEURRC-NEXT: v_mov_b32_e32 v6, s26
-; HEURRC-NEXT: v_mov_b32_e32 v7, s27
+; HEURRC-NEXT: v_mov_b32_e32 v8, s24
+; HEURRC-NEXT: v_mov_b32_e32 v9, s25
+; HEURRC-NEXT: v_mov_b32_e32 v10, s26
+; HEURRC-NEXT: v_mov_b32_e32 v11, s27
; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
-; HEURRC-NEXT: v_mov_b32_e32 v8, s28
-; HEURRC-NEXT: v_mov_b32_e32 v9, s29
-; HEURRC-NEXT: v_mov_b32_e32 v10, s30
-; HEURRC-NEXT: v_mov_b32_e32 v11, s31
+; HEURRC-NEXT: v_mov_b32_e32 v4, s28
+; HEURRC-NEXT: v_mov_b32_e32 v5, s29
+; HEURRC-NEXT: v_mov_b32_e32 v6, s30
+; HEURRC-NEXT: v_mov_b32_e32 v7, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
@@ -3404,7 +3404,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[8:11], v[4:7], a[16:31] cbsz:2 abid:3 blgp:1
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
@@ -3450,15 +3450,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3467,7 +3467,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[40:43], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
; VGPRRC-NEXT: s_nop 11
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
@@ -3962,21 +3962,20 @@ define <16 x i32> @test_mfma_i32_32x32x32_i8__mac__flags(<4 x i32> %arg0, <4 x i
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s20
-; SDAG-NEXT: v_mov_b32_e32 v33, s21
-; SDAG-NEXT: v_mov_b32_e32 v34, s22
-; SDAG-NEXT: v_mov_b32_e32 v35, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v36, s24
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b32_e32 v32, s28
+; SDAG-NEXT: v_mov_b32_e32 v33, s29
+; SDAG-NEXT: v_mov_b32_e32 v34, s30
+; SDAG-NEXT: v_mov_b32_e32 v35, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3985,7 +3984,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -4072,21 +4071,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v32, s20
-; HEURRC-NEXT: v_mov_b32_e32 v33, s21
-; HEURRC-NEXT: v_mov_b32_e32 v34, s22
-; HEURRC-NEXT: v_mov_b32_e32 v35, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v36, s24
; HEURRC-NEXT: v_mov_b32_e32 v37, s25
; HEURRC-NEXT: v_mov_b32_e32 v38, s26
; HEURRC-NEXT: v_mov_b32_e32 v39, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; HEURRC-NEXT: v_mov_b32_e32 v32, s28
+; HEURRC-NEXT: v_mov_b32_e32 v33, s29
+; HEURRC-NEXT: v_mov_b32_e32 v34, s30
+; HEURRC-NEXT: v_mov_b32_e32 v35, s31
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4095,7 +4093,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4136,21 +4134,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4159,7 +4156,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4339,21 +4336,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s20
-; SDAG-NEXT: v_mov_b32_e32 v33, s21
-; SDAG-NEXT: v_mov_b32_e32 v34, s22
-; SDAG-NEXT: v_mov_b32_e32 v35, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v36, s24
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b32_e32 v32, s28
+; SDAG-NEXT: v_mov_b32_e32 v33, s29
+; SDAG-NEXT: v_mov_b32_e32 v34, s30
+; SDAG-NEXT: v_mov_b32_e32 v35, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4362,7 +4358,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -4449,21 +4445,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v32, s20
-; HEURRC-NEXT: v_mov_b32_e32 v33, s21
-; HEURRC-NEXT: v_mov_b32_e32 v34, s22
-; HEURRC-NEXT: v_mov_b32_e32 v35, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v36, s24
; HEURRC-NEXT: v_mov_b32_e32 v37, s25
; HEURRC-NEXT: v_mov_b32_e32 v38, s26
; HEURRC-NEXT: v_mov_b32_e32 v39, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; HEURRC-NEXT: v_mov_b32_e32 v32, s28
+; HEURRC-NEXT: v_mov_b32_e32 v33, s29
+; HEURRC-NEXT: v_mov_b32_e32 v34, s30
+; HEURRC-NEXT: v_mov_b32_e32 v35, s31
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4472,7 +4467,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4513,21 +4508,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4536,7 +4530,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4716,20 +4710,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v20, s24
; SDAG-NEXT: v_mov_b32_e32 v21, s25
; SDAG-NEXT: v_mov_b32_e32 v22, s26
; SDAG-NEXT: v_mov_b32_e32 v23, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v16, s28
+; SDAG-NEXT: v_mov_b32_e32 v17, s29
+; SDAG-NEXT: v_mov_b32_e32 v18, s30
+; SDAG-NEXT: v_mov_b32_e32 v19, s31
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4738,7 +4731,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4777,20 +4770,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v20, s24
; HEURRC-NEXT: v_mov_b32_e32 v21, s25
; HEURRC-NEXT: v_mov_b32_e32 v22, s26
; HEURRC-NEXT: v_mov_b32_e32 v23, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b32_e32 v16, s28
+; HEURRC-NEXT: v_mov_b32_e32 v17, s29
+; HEURRC-NEXT: v_mov_b32_e32 v18, s30
+; HEURRC-NEXT: v_mov_b32_e32 v19, s31
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4799,7 +4791,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 10
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4810,20 +4802,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4832,7 +4823,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 10
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4922,20 +4913,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v20, s24
; SDAG-NEXT: v_mov_b32_e32 v21, s25
; SDAG-NEXT: v_mov_b32_e32 v22, s26
; SDAG-NEXT: v_mov_b32_e32 v23, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v16, s28
+; SDAG-NEXT: v_mov_b32_e32 v17, s29
+; SDAG-NEXT: v_mov_b32_e32 v18, s30
+; SDAG-NEXT: v_mov_b32_e32 v19, s31
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4944,7 +4934,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4983,20 +4973,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v20, s24
; HEURRC-NEXT: v_mov_b32_e32 v21, s25
; HEURRC-NEXT: v_mov_b32_e32 v22, s26
; HEURRC-NEXT: v_mov_b32_e32 v23, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b32_e32 v16, s28
+; HEURRC-NEXT: v_mov_b32_e32 v17, s29
+; HEURRC-NEXT: v_mov_b32_e32 v18, s30
+; HEURRC-NEXT: v_mov_b32_e32 v19, s31
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -5005,7 +4994,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 10
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -5016,20 +5005,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -5038,7 +5026,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 10
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index 97a89ec819bae..c0137b1a2ce95 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -1159,34 +1159,34 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
+; SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x40
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v21, s12
-; SDAG-NEXT: v_mov_b32_e32 v22, s13
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v21, s20
+; SDAG-NEXT: v_mov_b32_e32 v22, s21
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[22:23]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -1227,26 +1227,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -1290,26 +1290,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -1353,26 +1353,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -1416,26 +1416,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 0a36d3dd28f06..6629f528f16cb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -3195,22 +3195,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v20, s12
-; SDAG-NEXT: v_mov_b32_e32 v21, s13
-; SDAG-NEXT: v_mov_b32_e32 v22, s14
-; SDAG-NEXT: v_mov_b32_e32 v23, s15
-; SDAG-NEXT: v_mov_b32_e32 v24, s16
-; SDAG-NEXT: v_mov_b32_e32 v25, s17
-; SDAG-NEXT: v_mov_b32_e32 v26, s18
-; SDAG-NEXT: v_mov_b32_e32 v27, s19
-; SDAG-NEXT: v_mov_b32_e32 v28, s20
-; SDAG-NEXT: v_mov_b32_e32 v29, s21
-; SDAG-NEXT: v_mov_b32_e32 v30, s22
-; SDAG-NEXT: v_mov_b32_e32 v31, s23
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v28, s12
+; SDAG-NEXT: v_mov_b32_e32 v29, s13
+; SDAG-NEXT: v_mov_b32_e32 v30, s14
+; SDAG-NEXT: v_mov_b32_e32 v31, s15
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v20, s20
+; SDAG-NEXT: v_mov_b32_e32 v21, s21
+; SDAG-NEXT: v_mov_b32_e32 v22, s22
+; SDAG-NEXT: v_mov_b32_e32 v23, s23
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -3221,7 +3221,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: v_mov_b32_e32 v32, s0
; SDAG-NEXT: v_mov_b32_e32 v33, s1
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -3274,37 +3274,38 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v32, -2
; SDAG-NEXT: v_mov_b32_e32 v33, 0x41
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v20, s12
-; SDAG-NEXT: v_mov_b32_e32 v21, s13
-; SDAG-NEXT: v_mov_b32_e32 v22, s14
-; SDAG-NEXT: v_mov_b32_e32 v23, s15
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b32_e32 v24, s16
-; SDAG-NEXT: v_mov_b32_e32 v25, s17
-; SDAG-NEXT: v_mov_b32_e32 v26, s18
-; SDAG-NEXT: v_mov_b32_e32 v27, s19
-; SDAG-NEXT: v_mov_b32_e32 v28, s20
-; SDAG-NEXT: v_mov_b32_e32 v29, s21
-; SDAG-NEXT: v_mov_b32_e32 v30, s22
-; SDAG-NEXT: v_mov_b32_e32 v31, s23
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v20, s20
+; SDAG-NEXT: v_mov_b32_e32 v21, s21
+; SDAG-NEXT: v_mov_b32_e32 v22, s22
+; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: s_load_dwordx16 s[16:31], s[4:5], 0x40
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v28, s12
+; SDAG-NEXT: v_mov_b32_e32 v29, s13
+; SDAG-NEXT: v_mov_b32_e32 v30, s14
+; SDAG-NEXT: v_mov_b32_e32 v31, s15
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -3356,28 +3357,27 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %scale0, i32 %scale1) #1 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
+; SDAG-NEXT: v_mov_b32_e32 v26, s36
+; SDAG-NEXT: v_mov_b32_e32 v27, s37
+; SDAG-NEXT: v_mov_b32_e32 v28, s38
+; SDAG-NEXT: v_mov_b32_e32 v29, s39
+; SDAG-NEXT: v_mov_b32_e32 v30, s40
+; SDAG-NEXT: v_mov_b32_e32 v31, s41
+; SDAG-NEXT: v_mov_b32_e32 v32, s42
+; SDAG-NEXT: v_mov_b32_e32 v33, s43
+; SDAG-NEXT: v_mov_b32_e32 v18, s44
+; SDAG-NEXT: v_mov_b32_e32 v19, s45
+; SDAG-NEXT: v_mov_b32_e32 v20, s46
+; SDAG-NEXT: v_mov_b32_e32 v21, s47
+; SDAG-NEXT: v_mov_b32_e32 v22, s48
+; SDAG-NEXT: v_mov_b32_e32 v23, s49
+; SDAG-NEXT: v_mov_b32_e32 v24, s50
+; SDAG-NEXT: v_mov_b32_e32 v25, s51
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -3388,7 +3388,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
; SDAG-NEXT: v_mov_b32_e32 v16, s0
; SDAG-NEXT: v_mov_b32_e32 v17, s1
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v16, v17 op_sel_hi:[0,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[26:33], v[18:25], v[0:15], v16, v17 op_sel_hi:[0,0,0]
; SDAG-NEXT: v_mov_b32_e32 v18, s20
; SDAG-NEXT: v_mov_b32_e32 v19, s21
; SDAG-NEXT: v_mov_b32_e32 v20, s22
@@ -3492,29 +3492,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #1 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v16, 42
; SDAG-NEXT: v_mov_b32_e32 v17, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v26, s36
+; SDAG-NEXT: v_mov_b32_e32 v27, s37
+; SDAG-NEXT: v_mov_b32_e32 v28, s38
+; SDAG-NEXT: v_mov_b32_e32 v29, s39
+; SDAG-NEXT: v_mov_b32_e32 v30, s40
+; SDAG-NEXT: v_mov_b32_e32 v31, s41
+; SDAG-NEXT: v_mov_b32_e32 v32, s42
+; SDAG-NEXT: v_mov_b32_e32 v33, s43
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v18, s44
+; SDAG-NEXT: v_mov_b32_e32 v19, s45
+; SDAG-NEXT: v_mov_b32_e32 v20, s46
+; SDAG-NEXT: v_mov_b32_e32 v21, s47
+; SDAG-NEXT: v_mov_b32_e32 v22, s48
+; SDAG-NEXT: v_mov_b32_e32 v23, s49
+; SDAG-NEXT: v_mov_b32_e32 v24, s50
+; SDAG-NEXT: v_mov_b32_e32 v25, s51
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -3523,7 +3522,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[26:33], v[18:25], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v18, s20
; SDAG-NEXT: v_mov_b32_e32 v19, s21
; SDAG-NEXT: v_mov_b32_e32 v20, s22
@@ -3625,27 +3624,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s12
-; SDAG-NEXT: v_mov_b32_e32 v33, s13
-; SDAG-NEXT: v_mov_b32_e32 v34, s14
-; SDAG-NEXT: v_mov_b32_e32 v35, s15
-; SDAG-NEXT: v_mov_b32_e32 v36, s16
-; SDAG-NEXT: v_mov_b32_e32 v37, s17
-; SDAG-NEXT: v_mov_b32_e32 v38, s18
-; SDAG-NEXT: v_mov_b32_e32 v39, s19
-; SDAG-NEXT: v_mov_b32_e32 v40, s20
-; SDAG-NEXT: v_mov_b32_e32 v41, s21
-; SDAG-NEXT: v_mov_b32_e32 v42, s22
-; SDAG-NEXT: v_mov_b32_e32 v43, s23
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v44, s24
-; SDAG-NEXT: v_mov_b32_e32 v45, s25
-; SDAG-NEXT: v_mov_b32_e32 v46, s26
-; SDAG-NEXT: v_mov_b32_e32 v47, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v40, s36
+; SDAG-NEXT: v_mov_b32_e32 v41, s37
+; SDAG-NEXT: v_mov_b32_e32 v42, s38
+; SDAG-NEXT: v_mov_b32_e32 v43, s39
+; SDAG-NEXT: v_mov_b32_e32 v44, s40
+; SDAG-NEXT: v_mov_b32_e32 v45, s41
+; SDAG-NEXT: v_mov_b32_e32 v46, s42
+; SDAG-NEXT: v_mov_b32_e32 v47, s43
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b32_e32 v32, s44
+; SDAG-NEXT: v_mov_b32_e32 v33, s45
+; SDAG-NEXT: v_mov_b32_e32 v34, s46
+; SDAG-NEXT: v_mov_b32_e32 v35, s47
+; SDAG-NEXT: v_mov_b32_e32 v36, s48
+; SDAG-NEXT: v_mov_b32_e32 v37, s49
+; SDAG-NEXT: v_mov_b32_e32 v38, s50
+; SDAG-NEXT: v_mov_b32_e32 v39, s51
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3654,7 +3652,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
+; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[40:47], v[32:39], v[16:31] blgp:2
; SDAG-NEXT: s_nop 14
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -3749,29 +3747,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v32, 42
; SDAG-NEXT: v_mov_b32_e32 v33, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v24, s20
-; SDAG-NEXT: v_mov_b32_e32 v25, s21
-; SDAG-NEXT: v_mov_b32_e32 v26, s22
-; SDAG-NEXT: v_mov_b32_e32 v27, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v28, s24
-; SDAG-NEXT: v_mov_b32_e32 v29, s25
-; SDAG-NEXT: v_mov_b32_e32 v30, s26
-; SDAG-NEXT: v_mov_b32_e32 v31, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v24, s36
+; SDAG-NEXT: v_mov_b32_e32 v25, s37
+; SDAG-NEXT: v_mov_b32_e32 v26, s38
+; SDAG-NEXT: v_mov_b32_e32 v27, s39
+; SDAG-NEXT: v_mov_b32_e32 v28, s40
+; SDAG-NEXT: v_mov_b32_e32 v29, s41
+; SDAG-NEXT: v_mov_b32_e32 v30, s42
+; SDAG-NEXT: v_mov_b32_e32 v31, s43
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v16, s44
+; SDAG-NEXT: v_mov_b32_e32 v17, s45
+; SDAG-NEXT: v_mov_b32_e32 v18, s46
+; SDAG-NEXT: v_mov_b32_e32 v19, s47
+; SDAG-NEXT: v_mov_b32_e32 v20, s48
+; SDAG-NEXT: v_mov_b32_e32 v21, s49
+; SDAG-NEXT: v_mov_b32_e32 v22, s50
+; SDAG-NEXT: v_mov_b32_e32 v23, s51
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -3780,7 +3777,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
index 157b4fbe6803d..57b907f3123d8 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
@@ -30,26 +30,25 @@ define amdgpu_vs void @test(ptr addrspace(8) inreg %arg1, ptr addrspace(3) %arg2
define amdgpu_vs void @test_2(ptr addrspace(8) inreg %arg1, i32 %arg2, i32 inreg %arg3, ptr addrspace(3) %arg4) {
; CHECK-LABEL: test_2:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, 20, v1
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, 16, v1
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, 28, v1
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, 24, v1
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, 12, v1
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, 8, v1
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, 4, v1
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, 12, v1
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, 8, v1
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, 4, v1
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, 28, v1
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, 24, v1
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, 20, v1
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, 16, v1
; CHECK-NEXT: s_mov_b32 m0, -1
-; CHECK-NEXT: ds_read_b32 v2, v2
-; CHECK-NEXT: ds_read_b32 v5, v4
-; CHECK-NEXT: ds_read_b32 v4, v6
-; CHECK-NEXT: ds_read_b32 v9, v7
-; CHECK-NEXT: ds_read_b32 v8, v8
-; CHECK-NEXT: ds_read_b32 v7, v10
-; CHECK-NEXT: ds_read_b32 v6, v1
+; CHECK-NEXT: ds_read_b32 v4, v2
; CHECK-NEXT: ds_read_b32 v3, v3
-; CHECK-NEXT: s_waitcnt lgkmcnt(1)
-; CHECK-NEXT: tbuffer_store_format_xyzw v[6:9], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen glc slc
+; CHECK-NEXT: ds_read_b32 v2, v5
+; CHECK-NEXT: ds_read_b32 v8, v6
+; CHECK-NEXT: ds_read_b32 v7, v7
+; CHECK-NEXT: ds_read_b32 v6, v9
+; CHECK-NEXT: ds_read_b32 v5, v10
+; CHECK-NEXT: ds_read_b32 v1, v1
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: tbuffer_store_format_xyzw v[2:5], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:16 glc slc
+; CHECK-NEXT: tbuffer_store_format_xyzw v[1:4], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen glc slc
+; CHECK-NEXT: tbuffer_store_format_xyzw v[5:8], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:16 glc slc
; CHECK-NEXT: s_endpgm
%load = load <8 x float>, ptr addrspace(3) %arg4, align 4
%vec1 = shufflevector <8 x float> %load, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
index 5aa2cc713a4ec..d1db68f90dcce 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -769,37 +769,45 @@ define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class_subreg(
define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class_subreg_odd(ptr addrspace(1) %arg) #0 {
; CHECK-LABEL: test_rewrite_mfma_direct_copy_from_agpr_class_subreg_odd:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_mov_b32_e32 v1, 2.0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def a[0:31]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: v_mov_b32_e32 v18, 4.0
-; CHECK-NEXT: v_accvgpr_mov_b32 a0, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a1, a2
-; CHECK-NEXT: v_accvgpr_mov_b32 a2, a3
-; CHECK-NEXT: v_accvgpr_mov_b32 a3, a4
-; CHECK-NEXT: v_accvgpr_mov_b32 a4, a5
-; CHECK-NEXT: v_accvgpr_mov_b32 a5, a6
-; CHECK-NEXT: v_accvgpr_mov_b32 a6, a7
-; CHECK-NEXT: v_accvgpr_mov_b32 a7, a8
-; CHECK-NEXT: v_accvgpr_mov_b32 a8, a9
-; CHECK-NEXT: v_accvgpr_mov_b32 a9, a10
-; CHECK-NEXT: v_accvgpr_mov_b32 a10, a11
-; CHECK-NEXT: v_accvgpr_mov_b32 a11, a12
-; CHECK-NEXT: v_accvgpr_mov_b32 a12, a13
-; CHECK-NEXT: v_accvgpr_mov_b32 a13, a14
-; CHECK-NEXT: v_accvgpr_mov_b32 a14, a15
-; CHECK-NEXT: v_accvgpr_mov_b32 a15, a16
+; CHECK-NEXT: v_mov_b32_e32 v20, 2.0
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a15
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a14
+; CHECK-NEXT: v_pk_mov_b32 v[16:17], v[4:5], v[2:3] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a13
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a12
+; CHECK-NEXT: v_pk_mov_b32 v[14:15], v[2:3], v[4:5] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a11
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a10
+; CHECK-NEXT: v_pk_mov_b32 v[12:13], v[4:5], v[2:3] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a9
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a8
+; CHECK-NEXT: v_pk_mov_b32 v[10:11], v[2:3], v[4:5] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a7
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a6
+; CHECK-NEXT: v_pk_mov_b32 v[8:9], v[4:5], v[2:3] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a4
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a5
+; CHECK-NEXT: v_accvgpr_read_b32 v18, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v19, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
+; CHECK-NEXT: v_pk_mov_b32 v[6:7], v[2:3], v[4:5] op_sel:[1,0]
+; CHECK-NEXT: v_pk_mov_b32 v[4:5], v[18:19], v[2:3] op_sel:[1,0]
+; CHECK-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[18:19] op_sel:[1,0]
+; CHECK-NEXT: v_mov_b32_e32 v1, 4.0
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT: v_mfma_f32_16x16x1_4b_f32 a[0:15], v1, v18, a[0:15]
+; CHECK-NEXT: v_mfma_f32_16x16x1_4b_f32 v[2:17], v20, v1, v[2:17]
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 6, v0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_nop 7
-; CHECK-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
-; CHECK-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
-; CHECK-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
-; CHECK-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]
+; CHECK-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1] offset:48
+; CHECK-NEXT: global_store_dwordx4 v0, v[10:13], s[0:1] offset:32
+; CHECK-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
+; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; CHECK-NEXT: s_endpgm
%def = call <32 x float> asm sideeffect "; def $0", "=a"()
%src2 = shufflevector <32 x float> %def, <32 x float> poison, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
index f1d147947ccdf..f17edfa268423 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
@@ -879,9 +879,8 @@ define void @v_shuffle_v2f32_v4f32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +891,8 @@ define void @v_shuffle_v2f32_v4f32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1501,8 +1499,7 @@ define void @v_shuffle_v2f32_v4f32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1511,7 @@ define void @v_shuffle_v2f32_v4f32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
index 39c6a447788e4..f942c4c540fc9 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
@@ -879,9 +879,8 @@ define void @v_shuffle_v2i32_v4i32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +891,8 @@ define void @v_shuffle_v2i32_v4i32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1501,8 +1499,7 @@ define void @v_shuffle_v2i32_v4i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1511,7 @@ define void @v_shuffle_v2i32_v4i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
index c35361721e9b0..1f8745e242865 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
@@ -879,9 +879,8 @@ define void @v_shuffle_v2p3_v4p3__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +891,8 @@ define void @v_shuffle_v2p3_v4p3__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1501,8 +1499,7 @@ define void @v_shuffle_v2p3_v4p3__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1511,7 @@ define void @v_shuffle_v2p3_v4p3__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/ARM/bf16-shuffle.ll b/llvm/test/CodeGen/ARM/bf16-shuffle.ll
index a45ad8f698b23..e7cc3f052b39b 100644
--- a/llvm/test/CodeGen/ARM/bf16-shuffle.ll
+++ b/llvm/test/CodeGen/ARM/bf16-shuffle.ll
@@ -269,7 +269,8 @@ entry:
define dso_local <4 x bfloat> @test_vext_unaligned_bf16(<8 x bfloat> %a) {
; CHECK-LABEL: test_vext_unaligned_bf16:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vext.16 d0, d0, d1, #3
+; CHECK-NEXT: vext.16 q0, q0, q8, #3
+; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: bx lr
entry:
%vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
diff --git a/llvm/test/CodeGen/ARM/vext.ll b/llvm/test/CodeGen/ARM/vext.ll
index f1672643e4593..1a378283fe59b 100644
--- a/llvm/test/CodeGen/ARM/vext.ll
+++ b/llvm/test/CodeGen/ARM/vext.ll
@@ -239,9 +239,8 @@ define <4 x i16> @test_largespan(ptr %B) nounwind {
; CHECK-LABEL: test_largespan:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vorr d18, d16, d16
-; CHECK-NEXT: vuzp.16 d18, d17
-; CHECK-NEXT: vmov r0, r1, d18
+; CHECK-NEXT: vuzp.16 q8, q9
+; CHECK-NEXT: vmov r0, r1, d16
; CHECK-NEXT: mov pc, lr
%tmp1 = load <8 x i16>, ptr %B
%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
diff --git a/llvm/test/CodeGen/ARM/vpadd.ll b/llvm/test/CodeGen/ARM/vpadd.ll
index a98eabc63ef2a..24b6165033ee9 100644
--- a/llvm/test/CodeGen/ARM/vpadd.ll
+++ b/llvm/test/CodeGen/ARM/vpadd.ll
@@ -218,7 +218,8 @@ define void @addCombineToVPADD_i8(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADD_i8:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpadd.i8 d16, d16, d17
+; CHECK-NEXT: vuzp.8 q8, q9
+; CHECK-NEXT: vadd.i8 d16, d18, d16
; CHECK-NEXT: vstr d16, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <16 x i8>, ptr %cbcr
@@ -235,7 +236,8 @@ define void @addCombineToVPADD_i16(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADD_i16:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpadd.i16 d16, d16, d17
+; CHECK-NEXT: vuzp.16 q8, q9
+; CHECK-NEXT: vadd.i16 d16, d18, d16
; CHECK-NEXT: vstr d16, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <8 x i16>, ptr %cbcr
@@ -251,7 +253,8 @@ define void @addCombineToVPADD_i32(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADD_i32:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpadd.i32 d16, d16, d17
+; CHECK-NEXT: vuzp.32 q8, q9
+; CHECK-NEXT: vadd.i32 d16, d18, d16
; CHECK-NEXT: vstr d16, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <4 x i32>, ptr %cbcr
@@ -267,7 +270,8 @@ define void @addCombineToVPADDLq_s8(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_s8:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.s8 q8, q8
+; CHECK-NEXT: vuzp.8 q8, q9
+; CHECK-NEXT: vaddl.s8 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <16 x i8>, ptr %cbcr
@@ -308,7 +312,8 @@ define void @addCombineToVPADDLq_u8(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_u8:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.u8 q8, q8
+; CHECK-NEXT: vuzp.8 q8, q9
+; CHECK-NEXT: vaddl.u8 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <16 x i8>, ptr %cbcr
@@ -387,7 +392,8 @@ define void @addCombineToVPADDLq_s16(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_s16:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.s16 q8, q8
+; CHECK-NEXT: vuzp.16 q8, q9
+; CHECK-NEXT: vaddl.s16 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <8 x i16>, ptr %cbcr
@@ -405,7 +411,8 @@ define void @addCombineToVPADDLq_u16(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_u16:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.u16 q8, q8
+; CHECK-NEXT: vuzp.16 q8, q9
+; CHECK-NEXT: vaddl.u16 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <8 x i16>, ptr %cbcr
@@ -423,7 +430,8 @@ define void @addCombineToVPADDLq_s32(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_s32:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.s32 q8, q8
+; CHECK-NEXT: vuzp.32 q8, q9
+; CHECK-NEXT: vaddl.s32 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <4 x i32>, ptr %cbcr
@@ -441,7 +449,8 @@ define void @addCombineToVPADDLq_u32(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_u32:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.u32 q8, q8
+; CHECK-NEXT: vuzp.32 q8, q9
+; CHECK-NEXT: vaddl.u32 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <4 x i32>, ptr %cbcr
diff --git a/llvm/test/CodeGen/ARM/vuzp.ll b/llvm/test/CodeGen/ARM/vuzp.ll
index d24dadc7fc401..daaeef5795230 100644
--- a/llvm/test/CodeGen/ARM/vuzp.ll
+++ b/llvm/test/CodeGen/ARM/vuzp.ll
@@ -522,11 +522,11 @@ define <10 x i8> @vuzp_wide_type(<10 x i8> %tr0, <10 x i8> %tr1,
define %struct.uint8x8x2_t @vuzp_extract_subvector(<16 x i8> %t) #0 {
; CHECK-LABEL: vuzp_extract_subvector:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmov d16, r2, r3
-; CHECK-NEXT: vmov d17, r0, r1
-; CHECK-NEXT: vuzp.8 d17, d16
-; CHECK-NEXT: vmov r0, r1, d17
-; CHECK-NEXT: vmov r2, r3, d16
+; CHECK-NEXT: vmov d17, r2, r3
+; CHECK-NEXT: vmov d16, r0, r1
+; CHECK-NEXT: vuzp.8 q8, q9
+; CHECK-NEXT: vmov r0, r1, d16
+; CHECK-NEXT: vmov r2, r3, d18
; CHECK-NEXT: mov pc, lr
%vuzp.i = shufflevector <16 x i8> %t, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
diff --git a/llvm/test/CodeGen/ARM/vzip.ll b/llvm/test/CodeGen/ARM/vzip.ll
index ce40a2e48b6e8..bc9aaf22e5d6b 100644
--- a/llvm/test/CodeGen/ARM/vzip.ll
+++ b/llvm/test/CodeGen/ARM/vzip.ll
@@ -351,8 +351,9 @@ define void @vzip_vext_factor(ptr %A, ptr %B) {
; CHECK-LABEL: vzip_vext_factor:
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vext.16 d18, d16, d17, #1
-; CHECK-NEXT: vext.16 d16, d18, d17, #2
+; CHECK-NEXT: vext.16 d16, d16, d17, #3
+; CHECK-NEXT: vext.16 d17, d16, d16, #1
+; CHECK-NEXT: vzip.16 d16, d17
; CHECK-NEXT: vext.16 d16, d16, d16, #1
; CHECK-NEXT: vstr d16, [r1]
; CHECK-NEXT: mov pc, lr
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
index c05f306424519..6708bd2528d87 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
@@ -6,89 +6,57 @@
define <8 x i1> @v8i1_v16i1(<16 x i1>) {
; RV32-LABEL: v8i1_v16i1:
; RV32: # %bb.0:
-; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma
-; RV32-NEXT: vmv.x.s a0, v0
-; RV32-NEXT: slli a1, a0, 18
-; RV32-NEXT: srli a2, a0, 31
-; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; RV32-NEXT: vmv.v.x v8, a2
-; RV32-NEXT: slli a2, a0, 27
-; RV32-NEXT: srli a1, a1, 31
-; RV32-NEXT: vslide1down.vx v8, v8, a1
-; RV32-NEXT: slli a1, a0, 26
-; RV32-NEXT: srli a1, a1, 31
-; RV32-NEXT: vmv.v.x v9, a1
-; RV32-NEXT: slli a1, a0, 28
-; RV32-NEXT: srli a2, a2, 31
-; RV32-NEXT: vslide1down.vx v8, v8, a2
-; RV32-NEXT: slli a2, a0, 19
-; RV32-NEXT: srli a2, a2, 31
-; RV32-NEXT: vslide1down.vx v9, v9, a2
-; RV32-NEXT: slli a2, a0, 24
-; RV32-NEXT: slli a0, a0, 29
-; RV32-NEXT: srli a1, a1, 31
-; RV32-NEXT: srli a2, a2, 31
-; RV32-NEXT: srli a0, a0, 31
-; RV32-NEXT: vslide1down.vx v8, v8, a1
-; RV32-NEXT: vslide1down.vx v9, v9, a2
-; RV32-NEXT: vmv.v.i v0, 15
-; RV32-NEXT: vslide1down.vx v9, v9, a0
-; RV32-NEXT: vslidedown.vi v8, v9, 4, v0.t
-; RV32-NEXT: vand.vi v8, v8, 1
-; RV32-NEXT: vmsne.vi v0, v8, 0
+; RV32-NEXT: lui a0, %hi(.LCPI0_0)
+; RV32-NEXT: addi a0, a0, %lo(.LCPI0_0)
+; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; RV32-NEXT: vle8.v v8, (a0)
+; RV32-NEXT: vmv.v.i v9, 0
+; RV32-NEXT: vmerge.vim v9, v9, 1, v0
+; RV32-NEXT: vrgather.vv v10, v9, v8
+; RV32-NEXT: vmsne.vi v0, v10, 0
; RV32-NEXT: ret
;
; RV64-LABEL: v8i1_v16i1:
; RV64: # %bb.0:
-; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma
-; RV64-NEXT: vmv.x.s a0, v0
-; RV64-NEXT: slli a1, a0, 50
-; RV64-NEXT: srli a2, a0, 63
-; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; RV64-NEXT: vmv.v.x v8, a2
-; RV64-NEXT: slli a2, a0, 59
-; RV64-NEXT: srli a1, a1, 63
-; RV64-NEXT: vslide1down.vx v8, v8, a1
-; RV64-NEXT: slli a1, a0, 58
-; RV64-NEXT: srli a1, a1, 63
-; RV64-NEXT: vmv.v.x v9, a1
-; RV64-NEXT: slli a1, a0, 60
-; RV64-NEXT: srli a2, a2, 63
-; RV64-NEXT: vslide1down.vx v8, v8, a2
-; RV64-NEXT: slli a2, a0, 51
-; RV64-NEXT: srli a2, a2, 63
-; RV64-NEXT: vslide1down.vx v9, v9, a2
-; RV64-NEXT: slli a2, a0, 56
-; RV64-NEXT: slli a0, a0, 61
-; RV64-NEXT: srli a1, a1, 63
-; RV64-NEXT: srli a2, a2, 63
-; RV64-NEXT: srli a0, a0, 63
-; RV64-NEXT: vslide1down.vx v8, v8, a1
-; RV64-NEXT: vslide1down.vx v9, v9, a2
-; RV64-NEXT: vmv.v.i v0, 15
-; RV64-NEXT: vslide1down.vx v9, v9, a0
-; RV64-NEXT: vslidedown.vi v8, v9, 4, v0.t
-; RV64-NEXT: vand.vi v8, v8, 1
-; RV64-NEXT: vmsne.vi v0, v8, 0
+; RV64-NEXT: lui a0, %hi(.LCPI0_0)
+; RV64-NEXT: ld a0, %lo(.LCPI0_0)(a0)
+; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; RV64-NEXT: vmv.v.i v8, 0
+; RV64-NEXT: vmerge.vim v8, v8, 1, v0
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vmv.v.x v9, a0
+; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; RV64-NEXT: vrgather.vv v10, v8, v9
+; RV64-NEXT: vmsne.vi v0, v10, 0
; RV64-NEXT: ret
%2 = shufflevector <16 x i1> %0, <16 x i1> poison, <8 x i32> <i32 5, i32 12, i32 7, i32 2, i32 15, i32 13, i32 4, i32 3>
ret <8 x i1> %2
}
define <4 x i32> @v4i32_v8i32(<8 x i32>) {
-; CHECK-LABEL: v4i32_v8i32:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT: vmv.v.i v0, 8
-; CHECK-NEXT: vslidedown.vi v10, v8, 2
-; CHECK-NEXT: vslideup.vi v10, v8, 1, v0.t
-; CHECK-NEXT: vmv.v.i v0, 5
-; CHECK-NEXT: vsetivli zero, 4, e32, m2, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 4
-; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT: vslidedown.vi v10, v8, 1, v0.t
-; CHECK-NEXT: vmv.v.v v8, v10
-; CHECK-NEXT: ret
+; RV32-LABEL: v4i32_v8i32:
+; RV32: # %bb.0:
+; RV32-NEXT: lui a0, %hi(.LCPI1_0)
+; RV32-NEXT: addi a0, a0, %lo(.LCPI1_0)
+; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT: vle16.v v12, (a0)
+; RV32-NEXT: vrgatherei16.vv v10, v8, v12
+; RV32-NEXT: vmv1r.v v8, v10
+; RV32-NEXT: ret
+;
+; RV64-LABEL: v4i32_v8i32:
+; RV64: # %bb.0:
+; RV64-NEXT: lui a0, 131079
+; RV64-NEXT: slli a0, a0, 4
+; RV64-NEXT: addi a0, a0, 3
+; RV64-NEXT: slli a0, a0, 16
+; RV64-NEXT: addi a0, a0, 5
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vmv.v.x v12, a0
+; RV64-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT: vrgatherei16.vv v10, v8, v12
+; RV64-NEXT: vmv1r.v v8, v10
+; RV64-NEXT: ret
%2 = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> <i32 5, i32 3, i32 7, i32 2>
ret <4 x i32> %2
}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
index 63b96a96a284a..2b2a5a291c1f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
@@ -9,19 +9,14 @@
define void @deinterleave3_0_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave3_0_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: lui a0, %hi(.LCPI0_0)
-; CHECK-NEXT: addi a0, a0, %lo(.LCPI0_0)
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vle8.v v9, (a0)
-; CHECK-NEXT: li a0, 73
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v8, 8
+; CHECK-NEXT: lui a0, 9
+; CHECK-NEXT: addi a0, a0, 585
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT: vrgather.vv v10, v8, v9
; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
@@ -34,19 +29,14 @@ entry:
define void @deinterleave3_8_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave3_8_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: lui a0, %hi(.LCPI1_0)
-; CHECK-NEXT: addi a0, a0, %lo(.LCPI1_0)
+; CHECK-NEXT: lui a0, 2
+; CHECK-NEXT: addi a0, a0, 1170
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vle8.v v9, (a0)
-; CHECK-NEXT: li a0, 146
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT: vrgather.vv v10, v8, v9
; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
@@ -97,17 +87,15 @@ entry:
define void @deinterleave5_0_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave5_0_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: li a0, 33
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v9, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT: vmv.v.i v0, 10
-; CHECK-NEXT: vslidedown.vi v8, v8, 4, v0.t
-; CHECK-NEXT: vse8.v v8, (a1)
+; CHECK-NEXT: lui a0, 8
+; CHECK-NEXT: addi a0, a0, 1057
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -119,18 +107,15 @@ entry:
define void @deinterleave5_8_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave5_8_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v9, v8, 1
-; CHECK-NEXT: vslidedown.vi v9, v8, 5, v0.t
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vrgather.vi v9, v8, 3, v0.t
-; CHECK-NEXT: vse8.v v9, (a1)
+; CHECK-NEXT: lui a0, 1
+; CHECK-NEXT: addi a0, a0, -1982
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -142,16 +127,15 @@ entry:
define void @deinterleave6_0_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave6_0_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v9, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vrgather.vi v8, v9, 4, v0.t
-; CHECK-NEXT: vse8.v v8, (a1)
+; CHECK-NEXT: lui a0, 1
+; CHECK-NEXT: addi a0, a0, 65
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -163,18 +147,15 @@ entry:
define void @deinterleave6_8_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave6_8_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v9, v8, 1
-; CHECK-NEXT: vslidedown.vi v9, v8, 6, v0.t
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vrgather.vi v9, v8, 5, v0.t
-; CHECK-NEXT: vse8.v v9, (a1)
+; CHECK-NEXT: lui a0, 2
+; CHECK-NEXT: addi a0, a0, 130
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -186,16 +167,15 @@ entry:
define void @deinterleave7_0_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave7_0_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v9, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v8, v8, 6, v0.t
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vrgather.vi v8, v9, 6, v0.t
-; CHECK-NEXT: vse8.v v8, (a1)
+; CHECK-NEXT: lui a0, 4
+; CHECK-NEXT: addi a0, a0, 129
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -207,18 +187,15 @@ entry:
define void @deinterleave7_8_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave7_8_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v9, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v10, v9, 4
-; CHECK-NEXT: vslideup.vi v10, v9, 1, v0.t
-; CHECK-NEXT: vmv.v.i v0, 6
-; CHECK-NEXT: vrgather.vi v9, v8, 1
-; CHECK-NEXT: vmerge.vvm v8, v9, v10, v0
-; CHECK-NEXT: vse8.v v8, (a1)
+; CHECK-NEXT: lui a0, 4
+; CHECK-NEXT: addi a0, a0, 258
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -275,17 +252,11 @@ define void @deinterleave7_0_i64(ptr %in, ptr %out) {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma
; CHECK-NEXT: vle64.v v8, (a0)
-; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vmv4r.v v16, v8
-; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, mu
-; CHECK-NEXT: vslidedown.vi v16, v8, 6, v0.t
-; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vsetivli zero, 8, e64, m8, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, mu
-; CHECK-NEXT: vrgather.vi v16, v8, 6, v0.t
+; CHECK-NEXT: lui a0, 4
+; CHECK-NEXT: addi a0, a0, 129
+; CHECK-NEXT: vmv.s.x v24, a0
+; CHECK-NEXT: vcompress.vm v16, v8, v24
+; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
; CHECK-NEXT: vse64.v v16, (a1)
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
index 6ebb2f51b4f2a..bc99da33e1fbe 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
@@ -316,9 +316,9 @@ define void @vnsrl_0_i32(ptr %in, ptr %out) {
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, ma
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; ZVE32F-NEXT: vslideup.vi v8, v9, 1
+; ZVE32F-NEXT: vmv.v.i v0, 2
+; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, mu
+; ZVE32F-NEXT: vslidedown.vi v8, v8, 1, v0.t
; ZVE32F-NEXT: vse32.v v8, (a1)
; ZVE32F-NEXT: ret
;
@@ -361,10 +361,10 @@ define void @vnsrl_32_i32(ptr %in, ptr %out) {
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vmv.v.i v0, 1
+; ZVE32F-NEXT: vmv.v.i v0, 2
+; ZVE32F-NEXT: vslidedown.vi v9, v8, 1
; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, mu
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 1, v0.t
+; ZVE32F-NEXT: vslidedown.vi v9, v8, 2, v0.t
; ZVE32F-NEXT: vse32.v v9, (a1)
; ZVE32F-NEXT: ret
;
@@ -408,9 +408,9 @@ define void @vnsrl_0_float(ptr %in, ptr %out) {
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, ma
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; ZVE32F-NEXT: vslideup.vi v8, v9, 1
+; ZVE32F-NEXT: vmv.v.i v0, 2
+; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, mu
+; ZVE32F-NEXT: vslidedown.vi v8, v8, 1, v0.t
; ZVE32F-NEXT: vse32.v v8, (a1)
; ZVE32F-NEXT: ret
;
@@ -453,10 +453,10 @@ define void @vnsrl_32_float(ptr %in, ptr %out) {
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vmv.v.i v0, 1
+; ZVE32F-NEXT: vmv.v.i v0, 2
+; ZVE32F-NEXT: vslidedown.vi v9, v8, 1
; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, mu
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 1, v0.t
+; ZVE32F-NEXT: vslidedown.vi v9, v8, 2, v0.t
; ZVE32F-NEXT: vse32.v v9, (a1)
; ZVE32F-NEXT: ret
;
@@ -491,9 +491,9 @@ define void @vnsrl_0_i64(ptr %in, ptr %out) {
; V: # %bb.0: # %entry
; V-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; V-NEXT: vle64.v v8, (a0)
-; V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; V-NEXT: vslidedown.vi v9, v8, 2
-; V-NEXT: vslideup.vi v8, v9, 1
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT: vslidedown.vi v8, v8, 1, v0.t
; V-NEXT: vse64.v v8, (a1)
; V-NEXT: ret
;
@@ -518,9 +518,9 @@ define void @vnsrl_0_i64(ptr %in, ptr %out) {
; ZVZIP: # %bb.0: # %entry
; ZVZIP-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; ZVZIP-NEXT: vle64.v v8, (a0)
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vslidedown.vi v9, v8, 2
-; ZVZIP-NEXT: vslideup.vi v8, v9, 1
+; ZVZIP-NEXT: vmv.v.i v0, 2
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; ZVZIP-NEXT: vslidedown.vi v8, v8, 1, v0.t
; ZVZIP-NEXT: vse64.v v8, (a1)
; ZVZIP-NEXT: ret
entry:
@@ -535,10 +535,10 @@ define void @vnsrl_64_i64(ptr %in, ptr %out) {
; V: # %bb.0: # %entry
; V-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; V-NEXT: vle64.v v8, (a0)
-; V-NEXT: vmv.v.i v0, 1
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vslidedown.vi v9, v8, 1
; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
-; V-NEXT: vslidedown.vi v9, v8, 2
-; V-NEXT: vslidedown.vi v9, v8, 1, v0.t
+; V-NEXT: vslidedown.vi v9, v8, 2, v0.t
; V-NEXT: vse64.v v9, (a1)
; V-NEXT: ret
;
@@ -563,10 +563,11 @@ define void @vnsrl_64_i64(ptr %in, ptr %out) {
; ZVZIP: # %bb.0: # %entry
; ZVZIP-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; ZVZIP-NEXT: vle64.v v8, (a0)
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vslidedown.vi v9, v8, 2
-; ZVZIP-NEXT: vpairo.vv v10, v8, v9
-; ZVZIP-NEXT: vse64.v v10, (a1)
+; ZVZIP-NEXT: vmv.v.i v0, 2
+; ZVZIP-NEXT: vslidedown.vi v9, v8, 1
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; ZVZIP-NEXT: vslidedown.vi v9, v8, 2, v0.t
+; ZVZIP-NEXT: vse64.v v9, (a1)
; ZVZIP-NEXT: ret
entry:
%0 = load <4 x i64>, ptr %in, align 8
@@ -580,9 +581,9 @@ define void @vnsrl_0_double(ptr %in, ptr %out) {
; V: # %bb.0: # %entry
; V-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; V-NEXT: vle64.v v8, (a0)
-; V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; V-NEXT: vslidedown.vi v9, v8, 2
-; V-NEXT: vslideup.vi v8, v9, 1
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT: vslidedown.vi v8, v8, 1, v0.t
; V-NEXT: vse64.v v8, (a1)
; V-NEXT: ret
;
@@ -607,9 +608,9 @@ define void @vnsrl_0_double(ptr %in, ptr %out) {
; ZVZIP: # %bb.0: # %entry
; ZVZIP-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; ZVZIP-NEXT: vle64.v v8, (a0)
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vslidedown.vi v9, v8, 2
-; ZVZIP-NEXT: vslideup.vi v8, v9, 1
+; ZVZIP-NEXT: vmv.v.i v0, 2
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; ZVZIP-NEXT: vslidedown.vi v8, v8, 1, v0.t
; ZVZIP-NEXT: vse64.v v8, (a1)
; ZVZIP-NEXT: ret
entry:
@@ -624,10 +625,10 @@ define void @vnsrl_64_double(ptr %in, ptr %out) {
; V: # %bb.0: # %entry
; V-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; V-NEXT: vle64.v v8, (a0)
-; V-NEXT: vmv.v.i v0, 1
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vslidedown.vi v9, v8, 1
; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
-; V-NEXT: vslidedown.vi v9, v8, 2
-; V-NEXT: vslidedown.vi v9, v8, 1, v0.t
+; V-NEXT: vslidedown.vi v9, v8, 2, v0.t
; V-NEXT: vse64.v v9, (a1)
; V-NEXT: ret
;
@@ -652,10 +653,11 @@ define void @vnsrl_64_double(ptr %in, ptr %out) {
; ZVZIP: # %bb.0: # %entry
; ZVZIP-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; ZVZIP-NEXT: vle64.v v8, (a0)
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vslidedown.vi v9, v8, 2
-; ZVZIP-NEXT: vpairo.vv v10, v8, v9
-; ZVZIP-NEXT: vse64.v v10, (a1)
+; ZVZIP-NEXT: vmv.v.i v0, 2
+; ZVZIP-NEXT: vslidedown.vi v9, v8, 1
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; ZVZIP-NEXT: vslidedown.vi v9, v8, 2, v0.t
+; ZVZIP-NEXT: vse64.v v9, (a1)
; ZVZIP-NEXT: ret
entry:
%0 = load <4 x double>, ptr %in, align 8
@@ -714,27 +716,59 @@ entry:
; Not a vnsrl (checking for a prior pattern matching bug)
define void @vnsrl_0_i8_undef_negative(ptr %in, ptr %out) {
-; CHECK-LABEL: vnsrl_0_i8_undef_negative:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
-; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: li a0, 32
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: lui a0, %hi(.LCPI19_0)
-; CHECK-NEXT: addi a0, a0, %lo(.LCPI19_0)
-; CHECK-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
-; CHECK-NEXT: vle8.v v9, (a0)
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf4, ta, mu
-; CHECK-NEXT: vslideup.vi v11, v10, 4
-; CHECK-NEXT: vslideup.vi v11, v10, 3, v0.t
-; CHECK-NEXT: li a0, 48
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: vrgather.vv v10, v8, v9
-; CHECK-NEXT: vmerge.vvm v8, v10, v11, v0
-; CHECK-NEXT: vse8.v v8, (a1)
-; CHECK-NEXT: ret
+; V-LABEL: vnsrl_0_i8_undef_negative:
+; V: # %bb.0: # %entry
+; V-NEXT: lui a2, %hi(.LCPI19_0)
+; V-NEXT: ld a2, %lo(.LCPI19_0)(a2)
+; V-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; V-NEXT: vle8.v v8, (a0)
+; V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; V-NEXT: vmv.v.x v9, a2
+; V-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; V-NEXT: vrgather.vv v10, v8, v9
+; V-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
+; V-NEXT: vse8.v v10, (a1)
+; V-NEXT: ret
+;
+; ZVE32F-LABEL: vnsrl_0_i8_undef_negative:
+; ZVE32F: # %bb.0: # %entry
+; ZVE32F-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZVE32F-NEXT: vle8.v v8, (a0)
+; ZVE32F-NEXT: lui a0, %hi(.LCPI19_0)
+; ZVE32F-NEXT: addi a0, a0, %lo(.LCPI19_0)
+; ZVE32F-NEXT: vle8.v v9, (a0)
+; ZVE32F-NEXT: vrgather.vv v10, v8, v9
+; ZVE32F-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
+; ZVE32F-NEXT: vse8.v v10, (a1)
+; ZVE32F-NEXT: ret
+;
+; ZIP-LABEL: vnsrl_0_i8_undef_negative:
+; ZIP: # %bb.0: # %entry
+; ZIP-NEXT: lui a2, %hi(.LCPI19_0)
+; ZIP-NEXT: ld a2, %lo(.LCPI19_0)(a2)
+; ZIP-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZIP-NEXT: vle8.v v8, (a0)
+; ZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT: vmv.v.x v9, a2
+; ZIP-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZIP-NEXT: vrgather.vv v10, v8, v9
+; ZIP-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT: vse8.v v10, (a1)
+; ZIP-NEXT: ret
+;
+; ZVZIP-LABEL: vnsrl_0_i8_undef_negative:
+; ZVZIP: # %bb.0: # %entry
+; ZVZIP-NEXT: lui a2, %hi(.LCPI19_0)
+; ZVZIP-NEXT: ld a2, %lo(.LCPI19_0)(a2)
+; ZVZIP-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZVZIP-NEXT: vle8.v v8, (a0)
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; ZVZIP-NEXT: vmv.v.x v9, a2
+; ZVZIP-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZVZIP-NEXT: vrgather.vv v10, v8, v9
+; ZVZIP-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
+; ZVZIP-NEXT: vse8.v v10, (a1)
+; ZVZIP-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
%shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 poison, i32 1>
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index 7f624cffa014c..390b2650cbeb5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -337,11 +337,11 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle2step_i16(<16 x i16> %src) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u32 q2, q1, #16
; CHECK-NEXT: vstrh.32 q2, [r0, #8]
; CHECK-NEXT: vshr.u32 q2, q0, #16
-; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: vstrh.32 q2, [r0]
; CHECK-NEXT: vstrh.32 q1, [r1, #8]
; CHECK-NEXT: vstrh.32 q0, [r1]
@@ -627,11 +627,11 @@ define arm_aapcs_vfpcc <16 x i8> @shuffle2step_i8(<32 x i8> %src) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u16 q2, q1, #8
; CHECK-NEXT: vstrb.16 q2, [r0, #8]
; CHECK-NEXT: vshr.u16 q2, q0, #8
-; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: vstrb.16 q2, [r0]
; CHECK-NEXT: vstrb.16 q1, [r1, #8]
; CHECK-NEXT: vstrb.16 q0, [r1]
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index ac2f283166a93..c14d95839f481 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -5,10 +5,10 @@
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
@@ -1959,80 +1959,42 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT: vpbroadcastd %xmm1, %ymm1
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
-; AVX512F-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT: vpbroadcastd %xmm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
-; AVX512DQ-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
-; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512BW-SLOW: # %bb.0:
-; AVX512BW-SLOW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-SLOW-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX512BW-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
-; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rcx)
-; AVX512BW-SLOW-NEXT: vzeroupper
-; AVX512BW-SLOW-NEXT: retq
+; AVX512F-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
+; AVX512F-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
;
-; AVX512BW-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512BW-FAST: # %bb.0:
-; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,9,0,11,0,13,0,15]
-; AVX512BW-FAST-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vpermd %zmm0, %zmm1, %zmm0
-; AVX512BW-FAST-NEXT: vpaddb (%rdx), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, (%rcx)
-; AVX512BW-FAST-NEXT: vzeroupper
-; AVX512BW-FAST-NEXT: retq
+; AVX512DQ-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
+; AVX512DQ-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
+;
+; AVX512BW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,9,0,11,0,13,0,15]
+; AVX512BW-NEXT: vpermd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rcx)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64
%in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64
%in.vec = add <64 x i8> %in.vec.base, %in.vec.bias
@@ -2111,57 +2073,31 @@ define void @vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [8,1,2,3,8,5,6,7]
-; AVX512F-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [8,1,2,3,8,5,6,7]
-; AVX512DQ-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
+; AVX512F-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,10,11,0,13,14,15]
+; AVX512F-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512DQ-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,10,11,0,13,14,15]
+; AVX512DQ-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
;
; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
; AVX512BW-SLOW: # %bb.0:
@@ -2263,57 +2199,31 @@ define void @vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1
-; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
-; AVX512F-FAST-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
-; AVX512DQ-FAST-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
+; AVX512F-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
+; AVX512F-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512DQ-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
+; AVX512DQ-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
;
; AVX512BW-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
; AVX512BW-SLOW: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
index 4a542949c7859..89ae037bb64ae 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -5,10 +5,10 @@
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
@@ -1569,8 +1569,8 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
;
; AVX512F-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd (%rdi), %ymm0
-; AVX512F-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,0,11,0,13,0,15]
+; AVX512F-NEXT: vpermd (%rdi), %zmm0, %zmm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512F-NEXT: vzeroupper
@@ -1578,8 +1578,8 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
;
; AVX512DQ-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastd (%rdi), %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,0,11,0,13,0,15]
+; AVX512DQ-NEXT: vpermd (%rdi), %zmm0, %zmm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
@@ -1587,8 +1587,8 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
;
; AVX512BW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpbroadcastd (%rdi), %ymm0
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,0,11,0,13,0,15]
+; AVX512BW-NEXT: vpermd (%rdi), %zmm0, %zmm0
; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -1651,59 +1651,32 @@ define void @vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5,6,7]
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
-; AVX512F-FAST-NEXT: vpermd (%rdi), %zmm0, %zmm0
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5,6,7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
-; AVX512DQ-FAST-NEXT: vpermd (%rdi), %zmm0, %zmm0
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
-; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512BW-SLOW: # %bb.0:
-; AVX512BW-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX512BW-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5,6,7]
-; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-SLOW-NEXT: vzeroupper
-; AVX512BW-SLOW-NEXT: retq
+; AVX512F-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
+; AVX512F-NEXT: vpermd (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
;
-; AVX512BW-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512BW-FAST: # %bb.0:
-; AVX512BW-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
-; AVX512BW-FAST-NEXT: vpermd (%rdi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-FAST-NEXT: vzeroupper
-; AVX512BW-FAST-NEXT: retq
+; AVX512DQ-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
+; AVX512DQ-NEXT: vpermd (%rdi), %zmm0, %zmm0
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
+;
+; AVX512BW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
+; AVX512BW-NEXT: vpermd (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%in.vec = load <64 x i8>, ptr %in.elt.ptr, align 64
%in.vec.cast = bitcast <64 x i8> %in.vec to <16 x i32>
%broadcast.of.zextinreg = shufflevector <16 x i32> %in.vec.cast, <16 x i32> poison, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 0, i32 13, i32 14, i32 15>
@@ -1761,59 +1734,32 @@ define void @vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
-; AVX512F-FAST-NEXT: vpermq (%rdi), %zmm0, %zmm0
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
-; AVX512DQ-FAST-NEXT: vpermq (%rdi), %zmm0, %zmm0
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
-; AVX512BW-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512BW-SLOW: # %bb.0:
-; AVX512BW-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX512BW-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-SLOW-NEXT: vzeroupper
-; AVX512BW-SLOW-NEXT: retq
+; AVX512F-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
+; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
;
-; AVX512BW-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512BW-FAST: # %bb.0:
-; AVX512BW-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
-; AVX512BW-FAST-NEXT: vpermq (%rdi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-FAST-NEXT: vzeroupper
-; AVX512BW-FAST-NEXT: retq
+; AVX512DQ-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
+; AVX512DQ-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
+;
+; AVX512BW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
+; AVX512BW-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%in.vec = load <64 x i8>, ptr %in.elt.ptr, align 64
%in.vec.cast = bitcast <64 x i8> %in.vec to <8 x i64>
%broadcast.of.zextinreg = shufflevector <8 x i64> %in.vec.cast, <8 x i64> poison, <4 x i32> <i32 0, i32 5, i32 0, i32 7>
diff --git a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
index b3bf464b529d0..55cc050c80338 100644
--- a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
+++ b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
@@ -225,25 +225,15 @@ define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask1(ptr %vp, <8 x i16
}
define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; CHECK-FAST-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]
-; CHECK-FAST-NEXT: vptestnmw %xmm1, %xmm1, %k1
-; CHECK-FAST-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-FAST-NEXT: vzeroupper
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm2
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm3
-; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]
-; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3,4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm1, %xmm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm2, %xmm0 {%k1}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]
+; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1
+; CHECK-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
%vec = load <16 x i16>, ptr %vp
%shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
%cmp = icmp eq <8 x i16> %mask, zeroinitializer
@@ -252,24 +242,14 @@ define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16>
}
define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]
-; CHECK-FAST-NEXT: vptestnmw %xmm0, %xmm0, %k1
-; CHECK-FAST-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-FAST-NEXT: vzeroupper
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm1
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm2
-; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]
-; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm0, %xmm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} {z}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]
+; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1
+; CHECK-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
%vec = load <16 x i16>, ptr %vp
%shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
%cmp = icmp eq <8 x i16> %mask, zeroinitializer
@@ -1050,23 +1030,41 @@ define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec, <4 x i
ret <4 x i32> %res
}
define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask0(ptr %vp) {
-; CHECK-LABEL: test_8xi32_to_4xi32_perm_mem_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps 16(%rdi), %xmm0
-; CHECK-NEXT: vshufps $7, (%rdi), %xmm0, %xmm0 # xmm0 = xmm0[3,1],mem[0,0]
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [7,5,0,0,0,0,0,0]
+; CHECK-FAST-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vmovaps 16(%rdi), %xmm0
+; CHECK-FAST-PERLANE-NEXT: vshufps $7, (%rdi), %xmm0, %xmm0 # xmm0 = xmm0[3,1],mem[0,0]
+; CHECK-FAST-PERLANE-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>
ret <4 x i32> %res
}
define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {
-; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps 16(%rdi), %xmm2
-; CHECK-NEXT: vshufps $7, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[3,1],mem[0,0]
-; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k1
-; CHECK-NEXT: vmovdqa32 %xmm2, %xmm0 {%k1}
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [7,5,0,0,0,0,0,0]
+; CHECK-FAST-NEXT: vptestnmd %xmm1, %xmm1, %k1
+; CHECK-FAST-NEXT: vpermd (%rdi), %ymm2, %ymm0 {%k1}
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vmovaps 16(%rdi), %xmm2
+; CHECK-FAST-PERLANE-NEXT: vshufps $7, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[3,1],mem[0,0]
+; CHECK-FAST-PERLANE-NEXT: vptestnmd %xmm1, %xmm1, %k1
+; CHECK-FAST-PERLANE-NEXT: vmovdqa32 %xmm2, %xmm0 {%k1}
+; CHECK-FAST-PERLANE-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>
%cmp = icmp eq <4 x i32> %mask, zeroinitializer
@@ -1075,13 +1073,22 @@ define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %
}
define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %mask) {
-; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps 16(%rdi), %xmm1
-; CHECK-NEXT: vshufps $7, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[3,1],mem[0,0]
-; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k1
-; CHECK-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} {z}
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [7,5,0,0,0,0,0,0]
+; CHECK-FAST-NEXT: vptestnmd %xmm0, %xmm0, %k1
+; CHECK-FAST-NEXT: vpermd (%rdi), %ymm1, %ymm0 {%k1} {z}
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vmovaps 16(%rdi), %xmm1
+; CHECK-FAST-PERLANE-NEXT: vshufps $7, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[3,1],mem[0,0]
+; CHECK-FAST-PERLANE-NEXT: vptestnmd %xmm0, %xmm0, %k1
+; CHECK-FAST-PERLANE-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} {z}
+; CHECK-FAST-PERLANE-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>
%cmp = icmp eq <4 x i32> %mask, zeroinitializer
@@ -1158,9 +1165,10 @@ define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask2(ptr %vp, <4 x i32>
define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask3(ptr %vp) {
; CHECK-LABEL: test_8xi32_to_4xi32_perm_mem_mask3:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm1
-; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [5,1,2,7]
-; CHECK-NEXT: vpermi2d 16(%rdi), %xmm1, %xmm0
+; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [5,3,2,7]
+; CHECK-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>
@@ -1169,11 +1177,12 @@ define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask3(ptr %vp) {
define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {
; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask3:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm2
-; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [5,1,2,7]
-; CHECK-NEXT: vpermi2d 16(%rdi), %xmm2, %xmm3
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [5,3,2,7]
; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k1
-; CHECK-NEXT: vmovdqa32 %xmm3, %xmm0 {%k1}
+; CHECK-NEXT: vpermd (%rdi), %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>
@@ -1185,11 +1194,11 @@ define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %
define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %mask) {
; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask3:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm2
-; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [5,1,2,7]
+; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [5,3,2,7]
; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k1
-; CHECK-NEXT: vpermi2d 16(%rdi), %xmm2, %xmm1 {%k1} {z}
-; CHECK-NEXT: vmovdqa %xmm1, %xmm0
+; CHECK-NEXT: vpermd (%rdi), %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>
@@ -1864,8 +1873,9 @@ define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mask1(<4 x i64> %vec, <2 x i
define <2 x i64> @test_4xi64_to_2xi64_perm_mem_mask0(ptr %vp) {
; CHECK-LABEL: test_4xi64_to_2xi64_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps (%rdi), %xmm0
-; CHECK-NEXT: vunpckhpd 16(%rdi), %xmm0, %xmm0 # xmm0 = xmm0[1],mem[1]
+; CHECK-NEXT: vpermpd $237, (%rdi), %ymm0 # ymm0 = mem[1,3,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%res = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
@@ -1874,9 +1884,11 @@ define <2 x i64> @test_4xi64_to_2xi64_perm_mem_mask0(ptr %vp) {
define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %vec2, <2 x i64> %mask) {
; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqa (%rdi), %xmm2
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k1
-; CHECK-NEXT: vpunpckhqdq 16(%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[1],mem[1]
+; CHECK-NEXT: vpermq $237, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[1,3,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
@@ -1888,9 +1900,10 @@ define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %
define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %mask) {
; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqa (%rdi), %xmm1
; CHECK-NEXT: vptestnmq %xmm0, %xmm0, %k1
-; CHECK-NEXT: vpunpckhqdq 16(%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[1],mem[1]
+; CHECK-NEXT: vpermq $237, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[1,3,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
@@ -1902,10 +1915,11 @@ define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64>
define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %vec2, <2 x i64> %mask) {
; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mem_mask1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqa 16(%rdi), %xmm2
-; CHECK-NEXT: vpblendd $12, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k1
-; CHECK-NEXT: vmovdqa64 %xmm2, %xmm0 {%k1}
+; CHECK-NEXT: vpermq $230, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 1>
@@ -1917,10 +1931,10 @@ define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %
define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %mask) {
; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mem_mask1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqa 16(%rdi), %xmm1
-; CHECK-NEXT: vpblendd $12, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[0,1],mem[2,3]
; CHECK-NEXT: vptestnmq %xmm0, %xmm0, %k1
-; CHECK-NEXT: vmovdqa64 %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT: vpermq $230, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 1>
@@ -1965,23 +1979,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec, <4 x i
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask1(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,4,6,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,2,1]
-; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,4,6,1]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 4, i32 6, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -1989,44 +1994,27 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask1(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask1(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,4,6,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,2,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,4,6,1]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 4, i32 6, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask2(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,3,6,3]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,3,2,3]
-; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,3,6,3]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 3, i32 6, i32 3>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2034,61 +2022,37 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask2(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask2(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,3,6,3]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,2,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,3,6,3]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 3, i32 6, i32 3>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
ret <4 x i64> %res
}
define <4 x i64> @test_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec) {
-; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [6,0,0,7]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm0
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; CHECK-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,0,0,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_8xi64_to_4xi64_perm_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [6,0,0,7]
+; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,0,0,7]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,0,3]
-; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,0,0,7]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2096,44 +2060,27 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,0,0,7]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,0,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,0,0,7]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask4(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask4:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,7,7,5]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask4:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,3,3,1]
-; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,7,7,5]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 3, i32 7, i32 7, i32 5>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2141,21 +2088,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask4(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask4(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask4:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,7,7,5]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask4:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm2[1],ymm0[3],ymm2[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,3,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,7,7,5]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 3, i32 7, i32 7, i32 5>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
@@ -2190,40 +2129,24 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask5(<8 x i64> %vec, <4 x i
ret <4 x i64> %res
}
define <4 x i64> @test_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec) {
-; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,6,5,3]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm0
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[3,2,1,3]
-; CHECK-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_8xi64_to_4xi64_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,6,5,3]
+; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,6,5,3]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm3 = ymm3[3,2,1,3]
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,6,5,3]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2231,22 +2154,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,6,5,3]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm2 = ymm2[3,2,1,3]
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vmovdqa64 %ymm0, %ymm0 {%k1} {z}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,6,5,3]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
@@ -2383,22 +2297,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask0(ptr %vp, <4 x i64>
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,7,6,0]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd $15, (%rdi), %ymm2, %ymm2 # ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[0,3,2,0]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,7,6,0]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 7, i32 6, i32 0>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2407,21 +2313,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,6,0]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpblendd $15, (%rdi), %ymm1, %ymm1 # ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[0,3,2,0]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,6,0]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 7, i32 6, i32 0>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2430,22 +2328,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64>
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,1,1,5]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm2, %ymm2 # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,1,1,0]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,1,1,5]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 1, i32 1, i32 5>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2454,63 +2344,40 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,1,1,5]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm1, %ymm1 # ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,1,1,0]
-; CHECK-FAST-PERLANE-NEXT: retq
- %vec = load <8 x i64>, ptr %vp
- %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 1, i32 1, i32 5>
- %cmp = icmp eq <4 x i64> %mask, zeroinitializer
- %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,1,1,5]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
+ %vec = load <8 x i64>, ptr %vp
+ %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 1, i32 1, i32 5>
+ %cmp = icmp eq <4 x i64> %mask, zeroinitializer
+ %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
ret <4 x i64> %res
}
define <4 x i64> @test_8xi64_to_4xi64_perm_mem_mask3(ptr %vp) {
-; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [7,0,0,2]
-; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm0, %zmm0
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm0
-; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm0, %ymm0 # ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,1,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_8xi64_to_4xi64_perm_mem_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm0 = [7,0,0,2]
+; CHECK-NEXT: vpermpd (%rdi), %zmm0, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask3(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,0,0,2]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm2, %ymm2 # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,1,1,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,0,0,2]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2519,21 +2386,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask3(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask3(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,0,0,2]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm1, %ymm1 # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,1,1,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,0,0,2]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2573,22 +2432,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask4(ptr %vp, <4 x i64>
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask5(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask5:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,2,7,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask5:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd $192, 32(%rdi), %ymm2, %ymm2 # ymm2 = ymm2[0,1,2,3,4,5],mem[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[0,2,3,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,2,7,1]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 7, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2597,21 +2448,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask5(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask5(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask5:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,7,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask5:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpblendd $192, 32(%rdi), %ymm1, %ymm1 # ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[0,2,3,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,7,1]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 7, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2662,22 +2505,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask6(ptr %vp, <4 x i64>
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask7(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask7:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,7,5,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask7:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm2, %ymm2 # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,2,0,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,7,5,1]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 7, i32 5, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2686,21 +2521,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask7(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask7(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask7:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,7,5,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask7:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm1, %ymm1 # ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,2,0,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,7,5,1]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 7, i32 5, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2804,25 +2631,44 @@ define <2 x i64> @test_masked_z_8xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64>
}
define <4 x float> @test_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec) {
-; CHECK-LABEL: test_8xfloat_to_4xfloat_perm_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm1[0,1]
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,3,4,5,0,0,0,0]
+; CHECK-FAST-NEXT: vpermps %ymm0, %ymm1, %ymm0
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm1
+; CHECK-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm1[0,1]
+; CHECK-FAST-PERLANE-NEXT: vzeroupper
+; CHECK-FAST-PERLANE-NEXT: retq
%res = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>
ret <4 x float> %res
}
define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {
-; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm4
-; CHECK-NEXT: vcmpeqps %xmm4, %xmm2, %k1
-; CHECK-NEXT: vshufps {{.*#+}} xmm1 {%k1} = xmm0[0,3],xmm3[0,1]
-; CHECK-NEXT: vmovaps %xmm1, %xmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,3,4,5,0,0,0,0]
+; CHECK-FAST-NEXT: vxorps %xmm4, %xmm4, %xmm4
+; CHECK-FAST-NEXT: vcmpeqps %xmm4, %xmm2, %k1
+; CHECK-FAST-NEXT: vpermps %ymm0, %ymm3, %ymm1 {%k1}
+; CHECK-FAST-NEXT: vmovaps %xmm1, %xmm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm3
+; CHECK-FAST-PERLANE-NEXT: vxorps %xmm4, %xmm4, %xmm4
+; CHECK-FAST-PERLANE-NEXT: vcmpeqps %xmm4, %xmm2, %k1
+; CHECK-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm1 {%k1} = xmm0[0,3],xmm3[0,1]
+; CHECK-FAST-PERLANE-NEXT: vmovaps %xmm1, %xmm0
+; CHECK-FAST-PERLANE-NEXT: vzeroupper
+; CHECK-FAST-PERLANE-NEXT: retq
%shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>
%cmp = fcmp oeq <4 x float> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec2
@@ -2830,14 +2676,24 @@ define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec,
}
define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec, <4 x float> %mask) {
-; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm2
-; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k1
-; CHECK-NEXT: vshufps {{.*#+}} xmm0 {%k1} {z} = xmm0[0,3],xmm2[0,1]
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,3,4,5,0,0,0,0]
+; CHECK-FAST-NEXT: vxorps %xmm3, %xmm3, %xmm3
+; CHECK-FAST-NEXT: vcmpeqps %xmm3, %xmm1, %k1
+; CHECK-FAST-NEXT: vpermps %ymm0, %ymm2, %ymm0 {%k1} {z}
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm2
+; CHECK-FAST-PERLANE-NEXT: vxorps %xmm3, %xmm3, %xmm3
+; CHECK-FAST-PERLANE-NEXT: vcmpeqps %xmm3, %xmm1, %k1
+; CHECK-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 {%k1} {z} = xmm0[0,3],xmm2[0,1]
+; CHECK-FAST-PERLANE-NEXT: vzeroupper
+; CHECK-FAST-PERLANE-NEXT: retq
%shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>
%cmp = fcmp oeq <4 x float> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
@@ -3505,25 +3361,15 @@ define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mem_mask1(ptr %vp, <8
}
define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mem_mask2(ptr %vp, <8 x float> %vec2, <8 x float> %mask) {
-; CHECK-FAST-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,13,10,11,10,0,0,9]
-; CHECK-FAST-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqps %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm2
-; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,13,10,11,10,0,0,9]
-; CHECK-FAST-PERLANE-NEXT: vpermi2ps 32(%rdi), %ymm2, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; CHECK-FAST-PERLANE-NEXT: vcmpeqps %ymm2, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vmovaps %ymm3, %ymm0 {%k1}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,13,10,11,10,0,0,9]
+; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <16 x float>, ptr %vp
%shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 1, i32 13, i32 10, i32 11, i32 10, i32 0, i32 0, i32 9>
%cmp = fcmp oeq <8 x float> %mask, zeroinitializer
@@ -3532,24 +3378,14 @@ define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mem_mask2(ptr %vp, <8 x
}
define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2(ptr %vp, <8 x float> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,10,11,10,0,0,9]
-; CHECK-FAST-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; CHECK-FAST-NEXT: vcmpeqps %ymm2, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm2
-; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,10,11,10,0,0,9]
-; CHECK-FAST-PERLANE-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqps %ymm3, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermi2ps 32(%rdi), %ymm2, %ymm1 {%k1} {z}
-; CHECK-FAST-PERLANE-NEXT: vmovaps %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,10,11,10,0,0,9]
+; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqps %ymm2, %ymm0, %k1
+; CHECK-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <16 x float>, ptr %vp
%shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 1, i32 13, i32 10, i32 11, i32 10, i32 0, i32 0, i32 9>
%cmp = fcmp oeq <8 x float> %mask, zeroinitializer
@@ -3839,8 +3675,9 @@ define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mask1(<4 x double>
define <2 x double> @test_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp) {
; CHECK-LABEL: test_4xdouble_to_2xdouble_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps (%rdi), %xmm0
-; CHECK-NEXT: vblendps $3, 16(%rdi), %xmm0, %xmm0 # xmm0 = mem[0,1],xmm0[2,3]
+; CHECK-NEXT: vpermpd $230, (%rdi), %ymm0 # ymm0 = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%res = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>
@@ -3849,11 +3686,12 @@ define <2 x double> @test_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp) {
define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2 x double> %vec2, <2 x double> %mask) {
; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %xmm2
-; CHECK-NEXT: vblendpd $1, 16(%rdi), %xmm2, %xmm2 # xmm2 = mem[0],xmm2[1]
-; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k1
-; CHECK-NEXT: vmovapd %xmm2, %xmm0 {%k1}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqpd %xmm2, %xmm1, %k1
+; CHECK-NEXT: vpermpd $230, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>
@@ -3865,11 +3703,11 @@ define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2
define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2 x double> %mask) {
; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %xmm1
-; CHECK-NEXT: vblendpd $1, 16(%rdi), %xmm1, %xmm1 # xmm1 = mem[0],xmm1[1]
-; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT: vcmpeqpd %xmm2, %xmm0, %k1
-; CHECK-NEXT: vmovapd %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vcmpeqpd %xmm1, %xmm0, %k1
+; CHECK-NEXT: vpermpd $230, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>
@@ -3881,10 +3719,12 @@ define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp,
define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2 x double> %vec2, <2 x double> %mask) {
; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mem_mask1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd 16(%rdi), %xmm2
-; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k1
-; CHECK-NEXT: vunpcklpd (%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[0],mem[0]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqpd %xmm2, %xmm1, %k1
+; CHECK-NEXT: vpermpd $226, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[2,0,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>
@@ -3896,10 +3736,11 @@ define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2
define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2 x double> %mask) {
; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mem_mask1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd 16(%rdi), %xmm1
-; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT: vcmpeqpd %xmm2, %xmm0, %k1
-; CHECK-NEXT: vunpcklpd (%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[0],mem[0]
+; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vcmpeqpd %xmm1, %xmm0, %k1
+; CHECK-NEXT: vpermpd $226, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[2,0,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>
@@ -4045,25 +3886,15 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask3(<8 x double>
ret <4 x double> %res
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask4(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm3 = [1,5]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm0
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[0,0,1,1]
-; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $2, %zmm0, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[0,0,1,1]
-; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,1,5,5]
+; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm4
+; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovapd %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 1, i32 5, i32 5>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
@@ -4071,23 +3902,14 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask4(<8 x double> %v
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask4(<8 x double> %vec, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm2 = [1,5]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0
-; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,0,1,1]
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $2, %zmm0, %xmm2
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm2, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,0,1,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,1,5,5]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 1, i32 5, i32 5>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
@@ -4124,42 +3946,25 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask5(<8 x double>
ret <4 x double> %res
}
define <4 x double> @test_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec) {
-; CHECK-FAST-LABEL: test_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [5,0,7,0]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm0
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm0
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [5,0,7,0]
+; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
ret <4 x double> %res
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [5,0,7,0]
-; CHECK-FAST-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm0
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm1 {%k1} = ymm3[1],ymm0[1],ymm3[3],ymm0[3]
-; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [5,0,7,0]
+; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm4
+; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovapd %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
@@ -4167,48 +3972,29 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %v
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,0,7,0]
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm0
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm2[1],ymm0[1],ymm2[3],ymm0[3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,0,7,0]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
ret <4 x double> %res
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask7(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask7:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,5,0,6]
-; CHECK-FAST-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask7:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,3]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm1 {%k1} = ymm0[1],ymm3[1],ymm0[2],ymm3[2]
-; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,5,0,6]
+; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm4
+; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovapd %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 5, i32 0, i32 6>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
@@ -4216,23 +4002,14 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask7(<8 x double> %v
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask7(<8 x double> %vec, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask7:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,5,0,6]
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask7:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,3]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm2[1],ymm0[2],ymm2[2]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,5,0,6]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 5, i32 0, i32 6>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
@@ -4384,23 +4161,15 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask0(ptr %vp,
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vbroadcastsd 32(%rdi), %ymm2
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,0,6,2]
-; CHECK-FAST-NEXT: vpermi2pd (%rdi), %ymm2, %ymm3
-; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm1, %k1
-; CHECK-FAST-NEXT: vmovapd %ymm3, %ymm0 {%k1}
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vpermpd $236, (%rdi), %ymm2 # ymm2 = mem[0,3,2,3]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd $1, 32(%rdi){1to4}, %ymm2, %ymm0 {%k1}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,4,2,4]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 4, i32 2, i32 4>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -4409,23 +4178,14 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp, <4
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vbroadcastsd 32(%rdi), %ymm2
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,0,6,2]
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermi2pd (%rdi), %ymm2, %ymm1 {%k1} {z}
-; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vpermpd $236, (%rdi), %ymm1 # ymm1 = mem[0,3,2,3]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd $1, 32(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [3,4,2,4]
+; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
+; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 4, i32 2, i32 4>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -4434,24 +4194,15 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp,
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask2(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,2,3,4]
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovapd (%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vperm2f128 $33, 32(%rdi), %ymm2, %ymm3 # ymm3 = ymm2[2,3],mem[0,1]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} = ymm2[1],ymm3[0],ymm2[3],ymm3[2]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,2,3,4]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -4460,23 +4211,14 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask2(ptr %vp, <4
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2(ptr %vp, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,2,3,4]
-; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovapd (%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vperm2f128 $33, 32(%rdi), %ymm1, %ymm2 # ymm2 = ymm1[2,3],mem[0,1]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} {z} = ymm1[1],ymm2[0],ymm1[3],ymm2[2]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,2,3,4]
+; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
+; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -4564,11 +4306,12 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask4(ptr %vp,
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask5(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {
; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask5:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %ymm2
-; CHECK-NEXT: vperm2f128 $33, 32(%rdi), %ymm2, %ymm2 # ymm2 = ymm2[2,3],mem[0,1]
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [2,5,5,5]
; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-NEXT: vshufpd $14, 40(%rdi){1to4}, %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 5, i32 5, i32 5>
@@ -4580,11 +4323,11 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask5(ptr %vp, <4
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask5(ptr %vp, <4 x double> %mask) {
; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask5:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %ymm1
-; CHECK-NEXT: vperm2f128 $33, 32(%rdi), %ymm1, %ymm1 # ymm1 = ymm1[2,3],mem[0,1]
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [2,5,5,5]
; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
-; CHECK-NEXT: vshufpd $14, 40(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 5, i32 5, i32 5>
@@ -4640,10 +4383,12 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask6(ptr %vp,
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask7(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {
; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask7:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %ymm2
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,2,5]
; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-NEXT: vunpcklpd 40(%rdi){1to4}, %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 5>
@@ -4655,10 +4400,11 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask7(ptr %vp, <4
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask7(ptr %vp, <4 x double> %mask) {
; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask7:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %ymm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,5,2,5]
; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
-; CHECK-NEXT: vunpcklpd 40(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 5>
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 7c748439b1730..3eb5d73f78ec0 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -1827,10 +1827,27 @@ define <4 x i32> @pmaddwd_8(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_8:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%b = sext <8 x i16> %B to <8 x i32>
%m = mul nsw <8 x i32> %a, %b
@@ -1846,10 +1863,27 @@ define <4 x i32> @pmaddwd_8_swapped(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_8_swapped:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_8_swapped:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_8_swapped:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%b = sext <8 x i16> %B to <8 x i32>
%m = mul nsw <8 x i32> %a, %b
@@ -1921,10 +1955,21 @@ define <8 x i32> @pmaddwd_16(<16 x i16> %A, <16 x i16> %B) {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddwd_16:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddwd_16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: pmaddwd_16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,3,5,7,9,11,13,15]
+; AVX512-NEXT: vpermd %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT: retq
%a = sext <16 x i16> %A to <16 x i32>
%b = sext <16 x i16> %B to <16 x i32>
%m = mul nsw <16 x i32> %a, %b
@@ -1991,10 +2036,24 @@ define <4 x i32> @pmaddwd_const(<8 x i16> %A) {
; SSE2-NEXT: pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [32767,32768,0,0,1,7,42,32]
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_const:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32767,32768,0,0,1,7,42,32]
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_const:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32767,0,32768,0,0,0,0,0]
+; AVX1-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [1,0,7,0,42,0,32,0]
+; AVX1-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_const:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX256-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [32767,0,32768,0,0,0,0,0,1,0,7,0,42,0,32,0]
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%m = mul nsw <8 x i32> %a, <i32 32767, i32 -32768, i32 0, i32 0, i32 1, i32 7, i32 42, i32 32>
%odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -2098,10 +2157,29 @@ define <4 x i32> @jumbled_indices4(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: jumbled_indices4:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: jumbled_indices4:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,3]
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: jumbled_indices4:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,3]
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%exta = sext <8 x i16> %A to <8 x i32>
%extb = sext <8 x i16> %B to <8 x i32>
%m = mul <8 x i32> %exta, %extb
@@ -2127,10 +2205,22 @@ define <8 x i32> @jumbled_indices8(<16 x i16> %A, <16 x i16> %B) {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: jumbled_indices8:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: jumbled_indices8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: jumbled_indices8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,7,4,11,8,15,12]
+; AVX512-NEXT: vpermd %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,3,6,5,10,9,14,13]
+; AVX512-NEXT: vpermd %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT: retq
%exta = sext <16 x i16> %A to <16 x i32>
%extb = sext <16 x i16> %B to <16 x i32>
%m = mul <16 x i32> %exta, %extb
@@ -2306,10 +2396,44 @@ define <4 x i32> @pmaddwd_128(ptr %Aptr, ptr %Bptr) {
define <8 x i32> @pmaddwd_256(ptr %Aptr, ptr %Bptr) {
; SSE2-LABEL: pmaddwd_256:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: movdqa 16(%rdi), %xmm1
-; SSE2-NEXT: pmaddwd (%rsi), %xmm0
-; SSE2-NEXT: pmaddwd 16(%rsi), %xmm1
+; SSE2-NEXT: movdqa (%rdi), %xmm1
+; SSE2-NEXT: movdqa 16(%rdi), %xmm2
+; SSE2-NEXT: movdqa (%rsi), %xmm0
+; SSE2-NEXT: movdqa 16(%rsi), %xmm3
+; SSE2-NEXT: movdqa %xmm2, %xmm4
+; SSE2-NEXT: pslld $16, %xmm4
+; SSE2-NEXT: psrad $16, %xmm4
+; SSE2-NEXT: movdqa %xmm1, %xmm5
+; SSE2-NEXT: pslld $16, %xmm5
+; SSE2-NEXT: psrad $16, %xmm5
+; SSE2-NEXT: packssdw %xmm4, %xmm5
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: psrad $16, %xmm1
+; SSE2-NEXT: packssdw %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: pslld $16, %xmm4
+; SSE2-NEXT: psrad $16, %xmm4
+; SSE2-NEXT: packssdw %xmm2, %xmm4
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: packssdw %xmm3, %xmm0
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pmulhw %xmm4, %xmm2
+; SSE2-NEXT: pmullw %xmm5, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: pmulhw %xmm0, %xmm2
+; SSE2-NEXT: pmullw %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm4, %xmm0
; SSE2-NEXT: retq
;
; AVX1-LABEL: pmaddwd_256:
@@ -2321,11 +2445,52 @@ define <8 x i32> @pmaddwd_256(ptr %Aptr, ptr %Bptr) {
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddwd_256:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vmovdqa (%rdi), %ymm0
-; AVX256-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddwd_256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rsi), %xmm1
+; AVX2-NEXT: vmovdqa 16(%rsi), %xmm4
+; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm2[1],xmm4[2],xmm2[3],xmm4[4],xmm2[5],xmm4[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm5, %xmm2, %xmm2
+; AVX2-NEXT: vpsrld $16, %xmm4, %xmm4
+; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
+; AVX2-NEXT: vpmovsxwd %xmm3, %ymm3
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX2-NEXT: vpmulld %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: pmaddwd_256:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512-NEXT: vpmovdw %zmm0, %ymm2
+; AVX512-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX512-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512-NEXT: vpmovdw %zmm1, %ymm3
+; AVX512-NEXT: vpsrld $16, %ymm1, %ymm1
+; AVX512-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX512-NEXT: vpmovsxwd %xmm3, %ymm3
+; AVX512-NEXT: vpmulld %ymm3, %ymm2, %ymm2
+; AVX512-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
%A = load <16 x i16>, ptr %Aptr
%B = load <16 x i16>, ptr %Bptr
%A_even = shufflevector <16 x i16> %A, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -2373,23 +2538,84 @@ define <16 x i32> @pmaddwd_512(ptr %Aptr, ptr %Bptr) {
; AVX2: # %bb.0:
; AVX2-NEXT: vmovdqa (%rdi), %ymm0
; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX2-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX2-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
+; AVX2-NEXT: vmovdqa (%rsi), %ymm2
+; AVX2-NEXT: vmovdqa 32(%rsi), %ymm3
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm4 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vshufps {{.*#+}} ymm4 = ymm5[0,2],ymm4[0,2],ymm5[4,6],ymm4[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm5, %ymm0, %ymm0
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm1 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm6 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm6[0,2],ymm1[0,2],ymm6[4,6],ymm1[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpshufb %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vpshufb %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT: vpmovsxwd %xmm4, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm4
+; AVX2-NEXT: vpmovsxwd %xmm4, %ymm4
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm5
+; AVX2-NEXT: vpmulld %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT: vpmulld %ymm1, %ymm4, %ymm1
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm4
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm5
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm0
+; AVX2-NEXT: vpmulld %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX2-NEXT: vpmulld %ymm2, %ymm5, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
; AVX2-NEXT: retq
;
; AVX512F-LABEL: pmaddwd_512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT: vmovdqa64 (%rsi), %zmm1
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm2
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm3
+; AVX512F-NEXT: vpsrld $16, %zmm1, %zmm1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512F-NEXT: vpmulld %zmm3, %zmm2, %zmm2
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddwd_512:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm1
+; AVX512BW-NEXT: vpmovdw %zmm0, %ymm2
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-NEXT: vpermw %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT: vpmovdw %zmm1, %ymm4
+; AVX512BW-NEXT: vpermw %zmm1, %zmm3, %zmm1
+; AVX512BW-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512BW-NEXT: vpmovsxwd %ymm4, %zmm3
+; AVX512BW-NEXT: vpmulld %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512BW-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512BW-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: retq
%A = load <32 x i16>, ptr %Aptr
%B = load <32 x i16>, ptr %Bptr
@@ -2475,24 +2701,69 @@ define <32 x i32> @pmaddwd_1024(ptr %Aptr, ptr %Bptr) {
;
; AVX512F-LABEL: pmaddwd_1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm2
-; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm3
-; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpmaddwd 96(%rsi), %ymm3, %ymm1
-; AVX512F-NEXT: vpmaddwd 64(%rsi), %ymm2, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT: vmovdqa64 64(%rdi), %zmm1
+; AVX512F-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512F-NEXT: vmovdqa64 64(%rsi), %zmm3
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm4
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm5
+; AVX512F-NEXT: vpsrld $16, %zmm1, %zmm1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: vpmovdw %zmm3, %ymm6
+; AVX512F-NEXT: vpmovdw %zmm2, %ymm7
+; AVX512F-NEXT: vpsrld $16, %zmm3, %zmm3
+; AVX512F-NEXT: vpmovdw %zmm3, %ymm3
+; AVX512F-NEXT: vpsrld $16, %zmm2, %zmm2
+; AVX512F-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512F-NEXT: vpmovsxwd %ymm5, %zmm5
+; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm4
+; AVX512F-NEXT: vpmovsxwd %ymm7, %zmm7
+; AVX512F-NEXT: vpmulld %zmm7, %zmm5, %zmm5
+; AVX512F-NEXT: vpmovsxwd %ymm6, %zmm6
+; AVX512F-NEXT: vpmulld %zmm6, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512F-NEXT: vpmulld %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm5, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm2
+; AVX512F-NEXT: vpmulld %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vpaddd %zmm1, %zmm4, %zmm1
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddwd_1024:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
-; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmaddwd 64(%rsi), %zmm1, %zmm1
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm3
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
+; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm5
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
+; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
+; AVX512BW-NEXT: vpmovsxwd %ymm5, %zmm1
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm5, %ymm3
+; AVX512BW-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512BW-NEXT: vpmovsxwd %ymm4, %zmm5
+; AVX512BW-NEXT: vpmulld %zmm5, %zmm1, %zmm1
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm4, %ymm4
+; AVX512BW-NEXT: vpmovsxwd %ymm4, %zmm4
+; AVX512BW-NEXT: vpmulld %zmm4, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovsxwd %ymm0, %zmm4
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512BW-NEXT: vpmovsxwd %ymm0, %zmm5
+; AVX512BW-NEXT: vpmovsxwd %ymm2, %zmm0
+; AVX512BW-NEXT: vpmulld %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm2, %ymm1
+; AVX512BW-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512BW-NEXT: vpmulld %zmm1, %zmm5, %zmm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm3, %zmm1
; AVX512BW-NEXT: retq
%A = load <64 x i16>, ptr %Aptr
%B = load <64 x i16>, ptr %Bptr
diff --git a/llvm/test/CodeGen/X86/pclmulqdq.ll b/llvm/test/CodeGen/X86/pclmulqdq.ll
index 92df2d9987dbe..a95e65ef0b4f3 100644
--- a/llvm/test/CodeGen/X86/pclmulqdq.ll
+++ b/llvm/test/CodeGen/X86/pclmulqdq.ll
@@ -163,10 +163,19 @@ define <4 x i64> @pclmul256_hi_hi_vector(<4 x i64> %a0, <4 x i64> %a1) {
; AVX-PCLMUL-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX-PCLMUL-NEXT: retq
;
-; AVX-VPCLMULQDQ-LABEL: pclmul256_hi_hi_vector:
-; AVX-VPCLMULQDQ: # %bb.0:
-; AVX-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm0
-; AVX-VPCLMULQDQ-NEXT: retq
+; AVX2-VPCLMULQDQ-LABEL: pclmul256_hi_hi_vector:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: retq
+;
+; AVX512-VPCLMULQDQ-LABEL: pclmul256_hi_hi_vector:
+; AVX512-VPCLMULQDQ: # %bb.0:
+; AVX512-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,2,3]
+; AVX512-VPCLMULQDQ-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX512-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
+; AVX512-VPCLMULQDQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-VPCLMULQDQ-NEXT: retq
%s0 = shufflevector <4 x i64> %a0, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
%s1 = shufflevector <4 x i64> %a1, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
%x0 = zext <2 x i64> %s0 to <2 x i128>
@@ -365,12 +374,9 @@ define <8 x i64> @pclmul512_lo_lo(<8 x i64> %a0, <8 x i64> %a1) {
;
; AVX512-VPCLMULQDQ-LABEL: pclmul512_lo_lo:
; AVX512-VPCLMULQDQ: # %bb.0:
-; AVX512-VPCLMULQDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; AVX512-VPCLMULQDQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
-; AVX512-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512-VPCLMULQDQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512-VPCLMULQDQ-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
-; AVX512-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512-VPCLMULQDQ-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,2,4,6]
+; AVX512-VPCLMULQDQ-NEXT: vpermq %zmm0, %zmm2, %zmm0
+; AVX512-VPCLMULQDQ-NEXT: vpermq %zmm1, %zmm2, %zmm1
; AVX512-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX512-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm1, %xmm3
; AVX512-VPCLMULQDQ-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
diff --git a/llvm/test/CodeGen/X86/pmaddubsw.ll b/llvm/test/CodeGen/X86/pmaddubsw.ll
index d6c9877cd99b6..deb8a60e529fb 100644
--- a/llvm/test/CodeGen/X86/pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/pmaddubsw.ll
@@ -43,26 +43,183 @@ define <8 x i16> @pmaddubsw_128(ptr %Aptr, ptr %Bptr) {
define <16 x i16> @pmaddubsw_256(ptr %Aptr, ptr %Bptr) {
; SSE-LABEL: pmaddubsw_256:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa (%rsi), %xmm0
-; SSE-NEXT: movdqa 16(%rsi), %xmm1
-; SSE-NEXT: pmaddubsw (%rdi), %xmm0
-; SSE-NEXT: pmaddubsw 16(%rdi), %xmm1
+; SSE-NEXT: movdqa (%rdi), %xmm2
+; SSE-NEXT: movdqa 16(%rdi), %xmm0
+; SSE-NEXT: movdqa (%rsi), %xmm3
+; SSE-NEXT: movdqa 16(%rsi), %xmm7
+; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT: movdqa %xmm7, %xmm1
+; SSE-NEXT: pand %xmm5, %xmm1
+; SSE-NEXT: pand %xmm3, %xmm5
+; SSE-NEXT: packuswb %xmm1, %xmm5
+; SSE-NEXT: pxor %xmm6, %xmm6
+; SSE-NEXT: movdqa %xmm5, %xmm4
+; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; SSE-NEXT: movdqa %xmm3, %xmm1
+; SSE-NEXT: psrlw $8, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE-NEXT: movdqa %xmm7, %xmm1
+; SSE-NEXT: psrlw $8, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; SSE-NEXT: movdqa %xmm2, %xmm8
+; SSE-NEXT: psllw $8, %xmm8
+; SSE-NEXT: psraw $8, %xmm8
+; SSE-NEXT: movdqa %xmm0, %xmm9
+; SSE-NEXT: psllw $8, %xmm9
+; SSE-NEXT: psraw $8, %xmm9
+; SSE-NEXT: psraw $8, %xmm2
+; SSE-NEXT: psraw $8, %xmm0
+; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; SSE-NEXT: movdqa %xmm9, %xmm1
+; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT: pmaddwd %xmm7, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
+; SSE-NEXT: pmaddwd %xmm5, %xmm9
+; SSE-NEXT: packssdw %xmm9, %xmm1
+; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; SSE-NEXT: movdqa %xmm8, %xmm0
+; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE-NEXT: pmaddwd %xmm3, %xmm0
+; SSE-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
+; SSE-NEXT: pmaddwd %xmm4, %xmm8
+; SSE-NEXT: packssdw %xmm8, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: pmaddubsw_256:
; AVX1: # %bb.0:
+; AVX1-NEXT: vmovdqa (%rdi), %xmm0
+; AVX1-NEXT: vmovdqa 16(%rdi), %xmm2
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm1
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm4
+; AVX1-NEXT: vpackuswb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vmovq {{.*#+}} xmm4 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpshufb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm0[0],xmm2[0]
; AVX1-NEXT: vmovdqa (%rsi), %xmm0
-; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1
-; AVX1-NEXT: vpmaddubsw 16(%rdi), %xmm1, %xmm1
-; AVX1-NEXT: vpmaddubsw (%rdi), %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vmovdqa 16(%rsi), %xmm5
+; AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm3
+; AVX1-NEXT: vpackuswb %xmm6, %xmm3, %xmm3
+; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm2[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[2,3,2,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm2[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; AVX1-NEXT: vpmovsxbw %xmm11, %xmm11
+; AVX1-NEXT: vpmovsxbw %xmm8, %xmm8
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm11[0],xmm8[1],xmm11[1],xmm8[2],xmm11[2],xmm8[3],xmm11[3]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm8, %xmm3
+; AVX1-NEXT: vpmovsxbw %xmm10, %xmm4
+; AVX1-NEXT: vpmovsxbw %xmm7, %xmm7
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX1-NEXT: vpmaddwd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpackssdw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpmovsxbw %xmm2, %xmm2
+; AVX1-NEXT: vpmovsxbw %xmm1, %xmm1
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT: vpmaddwd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpmovsxbw %xmm9, %xmm2
+; AVX1-NEXT: vpmovsxbw %xmm6, %xmm4
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; AVX1-NEXT: vpmaddwd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vpackssdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddubsw_256:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vmovdqa (%rsi), %ymm0
-; AVX256-NEXT: vpmaddubsw (%rdi), %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddubsw_256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vmovdqa (%rsi), %ymm1
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,2,3]
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm1[0,2,2,3]
+; AVX2-NEXT: vpmovsxbd %xmm3, %ymm6
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
+; AVX2-NEXT: vpmovsxbd %xmm3, %ymm3
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm2, %ymm6, %ymm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vpmovsxbd %xmm0, %ymm5
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovsxbd %xmm0, %ymm0
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm5, %ymm1
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: vpackssdw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: pmaddubsw_256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vpshufb %ymm2, %ymm0, %ymm3
+; AVX512F-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm4 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512F-NEXT: vpshufb %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX512F-NEXT: vpshufb %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm2, %zmm3, %zmm2
+; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT: retq
+;
+; AVX512BW-LABEL: pmaddubsw_256:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm2
+; AVX512BW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512BW-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512BW-NEXT: vpmovwb %zmm1, %ymm4
+; AVX512BW-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512BW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512BW-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512BW-NEXT: retq
%A = load <32 x i8>, ptr %Aptr
%B = load <32 x i8>, ptr %Bptr
%A_even = shufflevector <32 x i8> %A, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -152,24 +309,167 @@ define <64 x i16> @pmaddubsw_512(ptr %Aptr, ptr %Bptr) {
;
; AVX512F-LABEL: pmaddubsw_512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rsi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1
-; AVX512F-NEXT: vmovdqa 64(%rsi), %ymm2
-; AVX512F-NEXT: vmovdqa 96(%rsi), %ymm3
-; AVX512F-NEXT: vpmaddubsw 32(%rdi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddubsw (%rdi), %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm3
+; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm4
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm5 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vpshufb %ymm5, %ymm4, %ymm2
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3],ymm7[4,5],ymm2[6,7]
+; AVX512F-NEXT: vpshufb %ymm5, %ymm1, %ymm7
+; AVX512F-NEXT: vpshufb %ymm6, %ymm0, %ymm8
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm7, %zmm2
+; AVX512F-NEXT: vpermq {{.*#+}} zmm2 = zmm2[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm7 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm8 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm8, %ymm3, %ymm3
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm1, %ymm1
+; AVX512F-NEXT: vpshufb %ymm8, %ymm0, %ymm0
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; AVX512F-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm3
+; AVX512F-NEXT: vmovdqa 64(%rsi), %ymm4
+; AVX512F-NEXT: vmovdqa 96(%rsi), %ymm9
+; AVX512F-NEXT: vpshufb %ymm5, %ymm9, %ymm10
+; AVX512F-NEXT: vpshufb %ymm6, %ymm4, %ymm11
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,3],ymm11[4,5],ymm10[6,7]
+; AVX512F-NEXT: vpshufb %ymm5, %ymm3, %ymm5
+; AVX512F-NEXT: vpshufb %ymm6, %ymm1, %ymm6
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm10, %zmm5, %zmm5
+; AVX512F-NEXT: vpermq {{.*#+}} zmm5 = zmm5[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm9, %ymm6
+; AVX512F-NEXT: vpshufb %ymm8, %ymm4, %ymm4
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3],ymm4[4,5],ymm6[6,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT: vpshufb %ymm8, %ymm1, %ymm1
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; AVX512F-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm3
+; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX512F-NEXT: vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm6
+; AVX512F-NEXT: vpmovsxbd %xmm6, %zmm6
+; AVX512F-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm5, %ymm7
+; AVX512F-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero,xmm8[4],zero,zero,zero,xmm8[5],zero,zero,zero,xmm8[6],zero,zero,zero,xmm8[7],zero,zero,zero,xmm8[8],zero,zero,zero,xmm8[9],zero,zero,zero,xmm8[10],zero,zero,zero,xmm8[11],zero,zero,zero,xmm8[12],zero,zero,zero,xmm8[13],zero,zero,zero,xmm8[14],zero,zero,zero,xmm8[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm8, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm5, %xmm7
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm6, %zmm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm5, %zmm2, %zmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm5
+; AVX512F-NEXT: vextracti128 $1, %ymm5, %xmm7
+; AVX512F-NEXT: vpmovsxbd %xmm7, %zmm7
+; AVX512F-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm8
+; AVX512F-NEXT: vpmovsxbd %xmm8, %zmm8
+; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm9
+; AVX512F-NEXT: vextracti128 $1, %ymm9, %xmm10
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero,xmm10[8],zero,zero,zero,xmm10[9],zero,zero,zero,xmm10[10],zero,zero,zero,xmm10[11],zero,zero,zero,xmm10[12],zero,zero,zero,xmm10[13],zero,zero,zero,xmm10[14],zero,zero,zero,xmm10[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm10, %zmm7, %zmm7
+; AVX512F-NEXT: vpaddd %zmm7, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero,xmm9[8],zero,zero,zero,xmm9[9],zero,zero,zero,xmm9[10],zero,zero,zero,xmm9[11],zero,zero,zero,xmm9[12],zero,zero,zero,xmm9[13],zero,zero,zero,xmm9[14],zero,zero,zero,xmm9[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm5, %zmm5
+; AVX512F-NEXT: vpaddd %zmm5, %zmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm5, %zmm8, %zmm5
+; AVX512F-NEXT: vpaddd %zmm5, %zmm6, %zmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT: vpmovsdw %zmm5, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpmaddubsw 96(%rdi), %ymm3, %ymm1
-; AVX512F-NEXT: vpmaddubsw 64(%rdi), %ymm2, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT: vpmovsdw %zmm3, %ymm1
+; AVX512F-NEXT: vpmovsdw %zmm4, %ymm2
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddubsw_512:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0
-; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm1
-; AVX512BW-NEXT: vpmaddubsw (%rdi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmaddubsw 64(%rdi), %zmm1, %zmm1
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm3
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,0,2,4,6,8,10,12,14,128,128,128,128,128,128,128,128,16,18,20,22,24,26,28,30,128,128,128,128,128,128,128,128,32,34,36,38,40,42,44,46,128,128,128,128,128,128,128,128,48,50,52,54,56,58,60,62]
+; AVX512BW-NEXT: vpshufb %zmm4, %zmm1, %zmm5
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm6 = [0,2,4,6,8,10,12,14,128,128,128,128,128,128,128,128,16,18,20,22,24,26,28,30,128,128,128,128,128,128,128,128,32,34,36,38,40,42,44,46,128,128,128,128,128,128,128,128,48,50,52,54,56,58,60,62,128,128,128,128,128,128,128,128]
+; AVX512BW-NEXT: vpshufb %zmm6, %zmm0, %zmm7
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm8 = [0,2,4,6,9,11,13,15]
+; AVX512BW-NEXT: vpermt2q %zmm5, %zmm8, %zmm7
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm5 = [128,128,128,128,128,128,128,128,1,3,5,7,9,11,13,15,128,128,128,128,128,128,128,128,17,19,21,23,25,27,29,31,128,128,128,128,128,128,128,128,33,35,37,39,41,43,45,47,128,128,128,128,128,128,128,128,49,51,53,55,57,59,61,63]
+; AVX512BW-NEXT: vpshufb %zmm5, %zmm1, %zmm9
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm10 = [1,3,5,7,9,11,13,15,128,128,128,128,128,128,128,128,17,19,21,23,25,27,29,31,128,128,128,128,128,128,128,128,33,35,37,39,41,43,45,47,128,128,128,128,128,128,128,128,49,51,53,55,57,59,61,63,128,128,128,128,128,128,128,128]
+; AVX512BW-NEXT: vpshufb %zmm10, %zmm0, %zmm1
+; AVX512BW-NEXT: vpermt2q %zmm9, %zmm8, %zmm1
+; AVX512BW-NEXT: vpshufb %zmm4, %zmm3, %zmm0
+; AVX512BW-NEXT: vpshufb %zmm6, %zmm2, %zmm4
+; AVX512BW-NEXT: vpermt2q %zmm0, %zmm8, %zmm4
+; AVX512BW-NEXT: vpshufb %zmm5, %zmm3, %zmm3
+; AVX512BW-NEXT: vpshufb %zmm10, %zmm2, %zmm0
+; AVX512BW-NEXT: vpermt2q %zmm3, %zmm8, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm7, %ymm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX512BW-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512BW-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm7, %xmm5
+; AVX512BW-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512BW-NEXT: vpmovsxbd %xmm7, %zmm6
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm4, %ymm7
+; AVX512BW-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero,xmm8[4],zero,zero,zero,xmm8[5],zero,zero,zero,xmm8[6],zero,zero,zero,xmm8[7],zero,zero,zero,xmm8[8],zero,zero,zero,xmm8[9],zero,zero,zero,xmm8[10],zero,zero,zero,xmm8[11],zero,zero,zero,xmm8[12],zero,zero,zero,xmm8[13],zero,zero,zero,xmm8[14],zero,zero,zero,xmm8[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm8, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm4, %xmm7
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm5, %zmm5
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm4, %zmm6, %zmm4
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm1, %ymm6
+; AVX512BW-NEXT: vextracti128 $1, %ymm6, %xmm7
+; AVX512BW-NEXT: vpmovsxbd %xmm7, %zmm7
+; AVX512BW-NEXT: vpmovsxbd %xmm6, %zmm6
+; AVX512BW-NEXT: vextracti128 $1, %ymm1, %xmm8
+; AVX512BW-NEXT: vpmovsxbd %xmm8, %zmm8
+; AVX512BW-NEXT: vpmovsxbd %xmm1, %zmm1
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm9
+; AVX512BW-NEXT: vextracti128 $1, %ymm9, %xmm10
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero,xmm10[8],zero,zero,zero,xmm10[9],zero,zero,zero,xmm10[10],zero,zero,zero,xmm10[11],zero,zero,zero,xmm10[12],zero,zero,zero,xmm10[13],zero,zero,zero,xmm10[14],zero,zero,zero,xmm10[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm10, %zmm7, %zmm7
+; AVX512BW-NEXT: vpaddd %zmm7, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero,xmm9[8],zero,zero,zero,xmm9[9],zero,zero,zero,xmm9[10],zero,zero,zero,xmm9[11],zero,zero,zero,xmm9[12],zero,zero,zero,xmm9[13],zero,zero,zero,xmm9[14],zero,zero,zero,xmm9[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm6, %zmm6
+; AVX512BW-NEXT: vpaddd %zmm6, %zmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm6
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm6, %zmm8, %zmm6
+; AVX512BW-NEXT: vpaddd %zmm6, %zmm5, %zmm5
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,6,7],zmm5[2,3,6,7]
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm5[0,1,4,5]
+; AVX512BW-NEXT: vpackssdw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[2,3,6,7],zmm3[2,3,6,7]
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,4,5],zmm3[0,1,4,5]
+; AVX512BW-NEXT: vpackssdw %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: retq
%A = load <128 x i8>, ptr %Aptr
%B = load <128 x i8>, ptr %Bptr
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index 625d18bcf4edf..3b4f657b90a31 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -1183,9 +1183,9 @@ define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) {
; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512-NEXT: vpermd %zmm0, %zmm1, %zmm0
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512-NEXT: retq
entry:
%val1a = zext <8 x i32> %val1 to <8 x i64>
diff --git a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
index 2ac2be5545dfd..d3493733124d9 100644
--- a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
+++ b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
@@ -61,22 +61,22 @@ define void @failing(ptr %0, ptr %1) nounwind {
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
; CHECK-NEXT: movdqu 1024(%rdx,%rsi), %xmm5
; CHECK-NEXT: movdqu 1040(%rdx,%rsi), %xmm6
-; CHECK-NEXT: movq %xmm5, %rdi
-; CHECK-NEXT: movq %xmm6, %r8
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm5[2,3,2,3]
+; CHECK-NEXT: movq %xmm7, %rdi
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; CHECK-NEXT: movq %xmm7, %r8
; CHECK-NEXT: movq %xmm5, %r9
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm6[2,3,2,3]
-; CHECK-NEXT: movq %xmm5, %r10
-; CHECK-NEXT: negq %r8
-; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: sbbq %r10, %r8
+; CHECK-NEXT: movq %xmm6, %r10
+; CHECK-NEXT: negq %r10
+; CHECK-NEXT: movq %rcx, %r10
+; CHECK-NEXT: sbbq %r8, %r10
; CHECK-NEXT: setge %r8b
; CHECK-NEXT: movzbl %r8b, %r8d
; CHECK-NEXT: negq %r8
; CHECK-NEXT: movq %r8, %xmm5
-; CHECK-NEXT: negq %rdi
-; CHECK-NEXT: movq %rcx, %rdi
-; CHECK-NEXT: sbbq %r9, %rdi
+; CHECK-NEXT: negq %r9
+; CHECK-NEXT: movq %rcx, %r8
+; CHECK-NEXT: sbbq %rdi, %r8
; CHECK-NEXT: setge %dil
; CHECK-NEXT: movzbl %dil, %edi
; CHECK-NEXT: negq %rdi
@@ -119,12 +119,13 @@ define void @failing(ptr %0, ptr %1) nounwind {
; CHECK-AVX2-NEXT: .LBB0_2: # %vector.body
; CHECK-AVX2-NEXT: # Parent Loop BB0_1 Depth=1
; CHECK-AVX2-NEXT: # => This Inner Loop Header: Depth=2
-; CHECK-AVX2-NEXT: vmovdqu 1024(%rdx,%rsi), %xmm5
-; CHECK-AVX2-NEXT: vmovdqu 1040(%rdx,%rsi), %xmm6
-; CHECK-AVX2-NEXT: vpextrq $1, %xmm5, %rdi
-; CHECK-AVX2-NEXT: vpextrq $1, %xmm6, %r8
-; CHECK-AVX2-NEXT: vmovq %xmm5, %r9
-; CHECK-AVX2-NEXT: vmovq %xmm6, %r10
+; CHECK-AVX2-NEXT: vmovdqu 1024(%rdx,%rsi), %ymm5
+; CHECK-AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm5[0,2,2,3]
+; CHECK-AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[1,3,2,3]
+; CHECK-AVX2-NEXT: vmovq %xmm5, %rdi
+; CHECK-AVX2-NEXT: vpextrq $1, %xmm5, %r8
+; CHECK-AVX2-NEXT: vmovq %xmm6, %r9
+; CHECK-AVX2-NEXT: vpextrq $1, %xmm6, %r10
; CHECK-AVX2-NEXT: negq %r10
; CHECK-AVX2-NEXT: movq %rcx, %r10
; CHECK-AVX2-NEXT: sbbq %r8, %r10
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
index f632654f89e04..024ce23d9ba4c 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
@@ -1,65 +1,45 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST,AVX2-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST,AVX2-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512VL-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512VL-FAST-PERLANE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BW-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BW-FAST-PERLANE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512BWVL-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512BWVL-FAST-PERLANE
define void @shuffle_v32i8_to_v16i8_1(ptr %L, ptr %S) nounwind {
-; AVX-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovdqa (%rdi), %xmm0
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
-; AVX-NEXT: vmovq {{.*#+}} xmm2 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX-NEXT: vpshufb %xmm2, %xmm1, %xmm1
-; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: retq
-;
-; AVX512F-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT: vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT: vmovq {{.*#+}} xmm2 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512F-NEXT: vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT: vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512F-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT: vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm2 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512VL-NEXT: vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT: vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512VL-NEXT: retq
+; AVX1-LABEL: shuffle_v32i8_to_v16i8_1:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovdqa (%rdi), %xmm0
+; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT: vmovq {{.*#+}} xmm2 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: retq
;
-; AVX512BW-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
-; AVX512BW-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
+; AVX2-LABEL: shuffle_v32i8_to_v16i8_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $8, (%rdi), %ymm0
-; AVX512BWVL-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: shuffle_v32i8_to_v16i8_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%vec = load <32 x i8>, ptr %L
%strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
store <16 x i8> %strided.vec, ptr %S
@@ -115,19 +95,87 @@ define void @shuffle_v16i16_to_v8i16_1(ptr %L, ptr %S) nounwind {
}
define void @shuffle_v8i32_to_v4i32_1(ptr %L, ptr %S) nounwind {
-; AVX-LABEL: shuffle_v8i32_to_v4i32_1:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovaps (%rdi), %xmm0
-; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
-; AVX-NEXT: vmovaps %xmm0, (%rsi)
-; AVX-NEXT: retq
+; AVX1-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovaps (%rdi), %xmm0
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX1-NEXT: vmovaps %xmm0, (%rsi)
+; AVX1-NEXT: retq
;
-; AVX512-LABEL: shuffle_v8i32_to_v4i32_1:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovaps (%rdi), %xmm0
-; AVX512-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
-; AVX512-NEXT: vmovaps %xmm0, (%rsi)
-; AVX512-NEXT: retq
+; AVX2-SLOW-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX2-SLOW-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX2-FAST-ALL: # %bb.0:
+; AVX2-FAST-ALL-NEXT: vbroadcastf128 {{.*#+}} ymm0 = [1,3,5,7,1,3,5,7]
+; AVX2-FAST-ALL-NEXT: # ymm0 = mem[0,1,0,1]
+; AVX2-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX2-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-FAST-ALL-NEXT: vzeroupper
+; AVX2-FAST-ALL-NEXT: retq
+;
+; AVX2-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX2-FAST-PERLANE: # %bb.0:
+; AVX2-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX2-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-FAST-PERLANE-NEXT: retq
+;
+; AVX512F-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovaps (%rdi), %xmm0
+; AVX512F-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX512F-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512VL-FAST-ALL: # %bb.0:
+; AVX512VL-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,0,0,0,0]
+; AVX512VL-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX512VL-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512VL-FAST-ALL-NEXT: vzeroupper
+; AVX512VL-FAST-ALL-NEXT: retq
+;
+; AVX512VL-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512VL-FAST-PERLANE: # %bb.0:
+; AVX512VL-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX512VL-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX512VL-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512VL-FAST-PERLANE-NEXT: retq
+;
+; AVX512BW-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512BW-FAST-ALL: # %bb.0:
+; AVX512BW-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,0,0,0,0]
+; AVX512BW-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX512BW-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BW-FAST-ALL-NEXT: vzeroupper
+; AVX512BW-FAST-ALL-NEXT: retq
+;
+; AVX512BW-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512BW-FAST-PERLANE: # %bb.0:
+; AVX512BW-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX512BW-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX512BW-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BW-FAST-PERLANE-NEXT: retq
+;
+; AVX512BWVL-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512BWVL-FAST-ALL: # %bb.0:
+; AVX512BWVL-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,0,0,0,0]
+; AVX512BWVL-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX512BWVL-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BWVL-FAST-ALL-NEXT: vzeroupper
+; AVX512BWVL-FAST-ALL-NEXT: retq
+;
+; AVX512BWVL-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512BWVL-FAST-PERLANE: # %bb.0:
+; AVX512BWVL-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX512BWVL-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX512BWVL-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BWVL-FAST-PERLANE-NEXT: retq
%vec = load <8 x i32>, ptr %L
%strided.vec = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
store <4 x i32> %strided.vec, ptr %S
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
index 571915b47d297..bc94ba6782c3c 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
@@ -48,15 +48,21 @@ define void @shuffle_v64i8_to_v32i8_1(ptr %L, ptr %S) nounwind {
;
; AVX512BW-LABEL: shuffle_v64i8_to_v32i8_1:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsrlw $8, (%rdi), %zmm0
-; AVX512BW-NEXT: vpmovwb %zmm0, (%rsi)
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vmovdqa %ymm0, (%rsi)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: shuffle_v64i8_to_v32i8_1:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $8, (%rdi), %zmm0
-; AVX512BWVL-NEXT: vpmovwb %zmm0, (%rsi)
+; AVX512BWVL-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BWVL-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BWVL-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BWVL-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512BWVL-NEXT: vmovdqa %ymm0, (%rsi)
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
%vec = load <64 x i8>, ptr %L
@@ -66,12 +72,35 @@ define void @shuffle_v64i8_to_v32i8_1(ptr %L, ptr %S) nounwind {
}
define void @shuffle_v32i16_to_v16i16_1(ptr %L, ptr %S) nounwind {
-; AVX512-LABEL: shuffle_v32i16_to_v16i16_1:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrld $16, (%rdi), %zmm0
-; AVX512-NEXT: vpmovdw %zmm0, (%rsi)
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX512F-LABEL: shuffle_v32i16_to_v16i16_1:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpsrld $16, (%rdi), %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: shuffle_v32i16_to_v16i16_1:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpsrld $16, (%rdi), %zmm0
+; AVX512VL-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512BW-LABEL: shuffle_v32i16_to_v16i16_1:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-NEXT: vpermw (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BWVL-LABEL: shuffle_v32i16_to_v16i16_1:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpmovsxbw {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BWVL-NEXT: vpermw (%rdi), %zmm0, %zmm0
+; AVX512BWVL-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
%vec = load <32 x i16>, ptr %L
%strided.vec = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
store <16 x i16> %strided.vec, ptr %S
@@ -79,65 +108,13 @@ define void @shuffle_v32i16_to_v16i16_1(ptr %L, ptr %S) nounwind {
}
define void @shuffle_v16i32_to_v8i32_1(ptr %L, ptr %S) nounwind {
-; AVX512F-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovaps (%rdi), %ymm0
-; AVX512F-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],mem[1,3],ymm0[5,7],mem[5,7]
-; AVX512F-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512F-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512VL-FAST-ALL-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512VL-FAST-ALL: # %bb.0:
-; AVX512VL-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15]
-; AVX512VL-FAST-ALL-NEXT: vpermps (%rdi), %zmm0, %zmm0
-; AVX512VL-FAST-ALL-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512VL-FAST-ALL-NEXT: vzeroupper
-; AVX512VL-FAST-ALL-NEXT: retq
-;
-; AVX512VL-FAST-PERLANE-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512VL-FAST-PERLANE: # %bb.0:
-; AVX512VL-FAST-PERLANE-NEXT: vmovaps (%rdi), %ymm0
-; AVX512VL-FAST-PERLANE-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],mem[1,3],ymm0[5,7],mem[5,7]
-; AVX512VL-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VL-FAST-PERLANE-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512VL-FAST-PERLANE-NEXT: vzeroupper
-; AVX512VL-FAST-PERLANE-NEXT: retq
-;
-; AVX512BW-FAST-ALL-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512BW-FAST-ALL: # %bb.0:
-; AVX512BW-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15]
-; AVX512BW-FAST-ALL-NEXT: vpermps (%rdi), %zmm0, %zmm0
-; AVX512BW-FAST-ALL-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BW-FAST-ALL-NEXT: vzeroupper
-; AVX512BW-FAST-ALL-NEXT: retq
-;
-; AVX512BW-FAST-PERLANE-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512BW-FAST-PERLANE: # %bb.0:
-; AVX512BW-FAST-PERLANE-NEXT: vmovaps (%rdi), %ymm0
-; AVX512BW-FAST-PERLANE-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],mem[1,3],ymm0[5,7],mem[5,7]
-; AVX512BW-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BW-FAST-PERLANE-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BW-FAST-PERLANE-NEXT: vzeroupper
-; AVX512BW-FAST-PERLANE-NEXT: retq
-;
-; AVX512BWVL-FAST-ALL-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512BWVL-FAST-ALL: # %bb.0:
-; AVX512BWVL-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15]
-; AVX512BWVL-FAST-ALL-NEXT: vpermps (%rdi), %zmm0, %zmm0
-; AVX512BWVL-FAST-ALL-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BWVL-FAST-ALL-NEXT: vzeroupper
-; AVX512BWVL-FAST-ALL-NEXT: retq
-;
-; AVX512BWVL-FAST-PERLANE-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512BWVL-FAST-PERLANE: # %bb.0:
-; AVX512BWVL-FAST-PERLANE-NEXT: vmovaps (%rdi), %ymm0
-; AVX512BWVL-FAST-PERLANE-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],mem[1,3],ymm0[5,7],mem[5,7]
-; AVX512BWVL-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BWVL-FAST-PERLANE-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BWVL-FAST-PERLANE-NEXT: vzeroupper
-; AVX512BWVL-FAST-PERLANE-NEXT: retq
+; AVX512-LABEL: shuffle_v16i32_to_v8i32_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15]
+; AVX512-NEXT: vpermps (%rdi), %zmm0, %zmm0
+; AVX512-NEXT: vmovaps %ymm0, (%rsi)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%vec = load <16 x i32>, ptr %L
%strided.vec = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
store <8 x i32> %strided.vec, ptr %S
@@ -314,4 +291,7 @@ define void @shuffle_v64i8_to_v8i8_7(ptr %L, ptr %S) nounwind {
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512VL: {{.*}}
+; AVX512BW-FAST-ALL: {{.*}}
+; AVX512BW-FAST-PERLANE: {{.*}}
+; AVX512BWVL-FAST-ALL: {{.*}}
+; AVX512BWVL-FAST-PERLANE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
index 26af46263c0e2..63d0de1522c2e 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
@@ -889,253 +889,54 @@ define <16 x i8> @evenelts_v32i16_trunc_v16i16_to_v16i8(<32 x i16> %n2) nounwind
;
; AVX1-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %rbx
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpextrw $6, %xmm2, %eax
-; AVX1-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX1-NEXT: vpextrw $2, %xmm2, %edx
-; AVX1-NEXT: vmovd %xmm2, %esi
-; AVX1-NEXT: vpextrw $6, %xmm1, %edi
-; AVX1-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX1-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX1-NEXT: vmovd %xmm1, %r10d
+; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3],xmm1[4],xmm3[5],xmm1[6],xmm3[7]
+; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX1-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX1-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX1-NEXT: vmovd %xmm1, %r14d
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrw $6, %xmm2, %eax
-; AVX2-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm2, %edx
-; AVX2-NEXT: vmovd %xmm2, %esi
-; AVX2-NEXT: vpextrw $6, %xmm1, %edi
-; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX2-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX2-NEXT: vmovd %xmm1, %r10d
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX2-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX2-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX2-NEXT: vmovd %xmm1, %r14d
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
-;
-; AVX512F-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vpmovdb %zmm0, %xmm1
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrw $6, %xmm2, %eax
-; AVX512F-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX512F-NEXT: vpextrw $2, %xmm2, %edx
-; AVX512F-NEXT: vmovd %xmm2, %esi
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrw $6, %xmm2, %edi
-; AVX512F-NEXT: vpextrw $4, %xmm2, %r8d
-; AVX512F-NEXT: vpextrw $2, %xmm2, %r9d
-; AVX512F-NEXT: vmovd %xmm2, %r10d
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512F-NEXT: vpextrw $4, %xmm0, %ebx
-; AVX512F-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512F-NEXT: vpinsrb $5, %ebp, %xmm1, %xmm0
-; AVX512F-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %rbp
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbp
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512VL-NEXT: vmovd %xmm1, %esi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %edi
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX512VL-NEXT: vmovd %xmm1, %r10d
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX512VL-NEXT: vmovd %xmm1, %r14d
-; AVX512VL-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %rbp
-; AVX512VL-NEXT: vzeroupper
-; AVX512VL-NEXT: retq
+; AVX2-SLOW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-SLOW-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vzeroupper
+; AVX2-SLOW-NEXT: retq
;
-; AVX512BW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: pushq %rbp
-; AVX512BW-NEXT: pushq %rbx
-; AVX512BW-NEXT: vpmovdb %zmm0, %xmm1
-; AVX512BW-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512BW-NEXT: vpextrw $6, %xmm2, %eax
-; AVX512BW-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX512BW-NEXT: vpextrw $2, %xmm2, %edx
-; AVX512BW-NEXT: vmovd %xmm2, %esi
-; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX512BW-NEXT: vpextrw $6, %xmm2, %edi
-; AVX512BW-NEXT: vpextrw $4, %xmm2, %r8d
-; AVX512BW-NEXT: vpextrw $2, %xmm2, %r9d
-; AVX512BW-NEXT: vmovd %xmm2, %r10d
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512BW-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512BW-NEXT: vpextrw $4, %xmm0, %ebx
-; AVX512BW-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512BW-NEXT: vpinsrb $5, %ebp, %xmm1, %xmm0
-; AVX512BW-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BW-NEXT: popq %rbx
-; AVX512BW-NEXT: popq %rbp
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
+; AVX2-FAST-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpmovsxdq {{.*#+}} ymm2 = [84148480,218892552,353636624,488380696]
+; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX2-FAST-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-FAST-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
;
-; AVX512BWVL-ONLY-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BWVL-ONLY: # %bb.0:
-; AVX512BWVL-ONLY-NEXT: pushq %rbp
-; AVX512BWVL-ONLY-NEXT: pushq %r14
-; AVX512BWVL-ONLY-NEXT: pushq %rbx
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %esi
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %edi
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %r10d
-; AVX512BWVL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %r14d
-; AVX512BWVL-ONLY-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: popq %rbx
-; AVX512BWVL-ONLY-NEXT: popq %r14
-; AVX512BWVL-ONLY-NEXT: popq %rbp
-; AVX512BWVL-ONLY-NEXT: vzeroupper
-; AVX512BWVL-ONLY-NEXT: retq
-;
-; AVX512VBMI-FAST-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-FAST: # %bb.0:
-; AVX512VBMI-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,79]
-; AVX512VBMI-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI-FAST-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
-; AVX512VBMI-FAST-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-FAST-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512VBMI-FAST-NEXT: vpinsrb $15, %eax, %xmm1, %xmm0
-; AVX512VBMI-FAST-NEXT: vzeroupper
-; AVX512VBMI-FAST-NEXT: retq
-;
-; AVX512VBMI-SLOW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-SLOW: # %bb.0:
-; AVX512VBMI-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [0,4,8,12,16,20,24,28,32,36,40,44,48,77,78,79]
-; AVX512VBMI-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI-SLOW-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
-; AVX512VBMI-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512VBMI-SLOW-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512VBMI-SLOW-NEXT: vpextrw $2, %xmm0, %edx
-; AVX512VBMI-SLOW-NEXT: vpinsrb $13, %edx, %xmm1, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vzeroupper
-; AVX512VBMI-SLOW-NEXT: retq
+; AVX512-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%n0 = shufflevector <32 x i16> %n2, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%n1 = trunc <16 x i16> %n0 to <16 x i8>
ret <16 x i8> %n1
@@ -1230,260 +1031,70 @@ define <16 x i8> @oddelts_v32i16_trunc_v16i16_to_v16i8(<32 x i16> %n2) nounwind
;
; AVX1-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %rbx
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpextrw $7, %xmm2, %eax
-; AVX1-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX1-NEXT: vpextrw $3, %xmm2, %edx
-; AVX1-NEXT: vpextrw $1, %xmm2, %esi
-; AVX1-NEXT: vpextrw $7, %xmm1, %edi
-; AVX1-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX1-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX1-NEXT: vpextrw $1, %xmm1, %r10d
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX1-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX1-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX1-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrw $7, %xmm2, %eax
-; AVX2-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX2-NEXT: vpextrw $3, %xmm2, %edx
-; AVX2-NEXT: vpextrw $1, %xmm2, %esi
-; AVX2-NEXT: vpextrw $7, %xmm1, %edi
-; AVX2-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX2-NEXT: vpextrw $1, %xmm1, %r10d
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX2-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX2-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX2-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: pushq %r14
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512F-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512F-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512F-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512F-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512F-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512F-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512F-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512F-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512F-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %r14
-; AVX512F-NEXT: popq %rbp
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbp
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512VL-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %rbp
+; AVX512VL-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: pushq %rbp
-; AVX512BW-NEXT: pushq %r14
-; AVX512BW-NEXT: pushq %rbx
-; AVX512BW-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BW-NEXT: popq %rbx
-; AVX512BW-NEXT: popq %r14
-; AVX512BW-NEXT: popq %rbp
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-NEXT: vpermw %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
-; AVX512BWVL-ONLY-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BWVL-ONLY: # %bb.0:
-; AVX512BWVL-ONLY-NEXT: pushq %rbp
-; AVX512BWVL-ONLY-NEXT: pushq %r14
-; AVX512BWVL-ONLY-NEXT: pushq %rbx
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512BWVL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512BWVL-ONLY-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BWVL-ONLY-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: popq %rbx
-; AVX512BWVL-ONLY-NEXT: popq %r14
-; AVX512BWVL-ONLY-NEXT: popq %rbp
-; AVX512BWVL-ONLY-NEXT: vzeroupper
-; AVX512BWVL-ONLY-NEXT: retq
-;
-; AVX512VBMI-FAST-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-FAST: # %bb.0:
-; AVX512VBMI-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]
-; AVX512VBMI-FAST-NEXT: vpermb %zmm0, %zmm1, %zmm0
-; AVX512VBMI-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512VBMI-FAST-NEXT: vzeroupper
-; AVX512VBMI-FAST-NEXT: retq
-;
-; AVX512VBMI-SLOW-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-SLOW: # %bb.0:
-; AVX512VBMI-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [2,6,10,14,18,22,26,30,34,38,42,46,50,u,u,u]
-; AVX512VBMI-SLOW-NEXT: vpermb %zmm0, %zmm1, %zmm1
-; AVX512VBMI-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512VBMI-SLOW-NEXT: vpextrw $5, %xmm0, %ecx
-; AVX512VBMI-SLOW-NEXT: vpextrw $3, %xmm0, %edx
-; AVX512VBMI-SLOW-NEXT: vpinsrb $13, %edx, %xmm1, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vzeroupper
-; AVX512VBMI-SLOW-NEXT: retq
+; AVX512BWVL-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BWVL-NEXT: vpermw %zmm0, %zmm1, %zmm0
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
%n0 = shufflevector <32 x i16> %n2, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
%n1 = trunc <16 x i16> %n0 to <16 x i8>
ret <16 x i8> %n1
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512BWVL-ONLY: {{.*}}
; AVX512VBMI: {{.*}}
+; AVX512VBMI-FAST: {{.*}}
+; AVX512VBMI-SLOW: {{.*}}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 05c855ed90b3f..cd86b7b91505b 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -6,8 +6,8 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BWVL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BWVL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VBMIVL
@@ -29,29 +29,29 @@ define void @shuffle_v32i8_to_v16i8(ptr %L, ptr %S) nounwind {
;
; AVX2-LABEL: shuffle_v32i8_to_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT: vpand 16(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vpand (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: shuffle_v32i8_to_v16i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512F-NEXT: vpand 16(%rdi), %xmm0, %xmm1
-; AVX512F-NEXT: vpand (%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
; AVX512F-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: shuffle_v32i8_to_v16i8:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT: vpand 16(%rdi), %xmm0, %xmm1
-; AVX512VL-NEXT: vpand (%rdi), %xmm0, %xmm0
-; AVX512VL-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
; AVX512VL-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: shuffle_v32i8_to_v16i8:
@@ -260,12 +260,35 @@ define void @trunc_v8i32_to_v8i16(ptr %L, ptr %S) nounwind {
}
define void @shuffle_v8i32_to_v4i32(ptr %L, ptr %S) nounwind {
-; AVX-LABEL: shuffle_v8i32_to_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovaps (%rdi), %xmm0
-; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX-NEXT: vmovaps %xmm0, (%rsi)
-; AVX-NEXT: retq
+; AVX1-LABEL: shuffle_v8i32_to_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovaps (%rdi), %xmm0
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX1-NEXT: vmovaps %xmm0, (%rsi)
+; AVX1-NEXT: retq
+;
+; AVX2-SLOW-LABEL: shuffle_v8i32_to_v4i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX2-SLOW-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32:
+; AVX2-FAST-ALL: # %bb.0:
+; AVX2-FAST-ALL-NEXT: vbroadcastf128 {{.*#+}} ymm0 = [0,2,4,6,0,2,4,6]
+; AVX2-FAST-ALL-NEXT: # ymm0 = mem[0,1,0,1]
+; AVX2-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX2-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-FAST-ALL-NEXT: vzeroupper
+; AVX2-FAST-ALL-NEXT: retq
+;
+; AVX2-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32:
+; AVX2-FAST-PERLANE: # %bb.0:
+; AVX2-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX2-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-FAST-PERLANE-NEXT: retq
;
; AVX512F-LABEL: shuffle_v8i32_to_v4i32:
; AVX512F: # %bb.0:
@@ -281,12 +304,20 @@ define void @shuffle_v8i32_to_v4i32(ptr %L, ptr %S) nounwind {
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
-; AVX512BW-LABEL: shuffle_v8i32_to_v4i32:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovaps (%rdi), %xmm0
-; AVX512BW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512BW-NEXT: vmovaps %xmm0, (%rsi)
-; AVX512BW-NEXT: retq
+; AVX512BW-ALL-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512BW-ALL: # %bb.0:
+; AVX512BW-ALL-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512BW-ALL-NEXT: vpmovqd %zmm0, %ymm0
+; AVX512BW-ALL-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512BW-ALL-NEXT: vzeroupper
+; AVX512BW-ALL-NEXT: retq
+;
+; AVX512BW-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512BW-FAST-PERLANE: # %bb.0:
+; AVX512BW-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX512BW-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX512BW-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BW-FAST-PERLANE-NEXT: retq
;
; AVX512BWVL-LABEL: shuffle_v8i32_to_v4i32:
; AVX512BWVL: # %bb.0:
@@ -1301,73 +1332,65 @@ define void @trunc_v4i64_to_v4i8(ptr %L, ptr %S) nounwind {
define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
; AVX1-LABEL: negative:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[u,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u],zero,zero,zero,zero,zero,zero,zero,xmm0[0,2,4,6,8,10,12,14]
-; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615]
-; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vmovd %xmm1, %eax
+; AVX1-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: negative:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX2-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vmovd %xmm1, %eax
+; AVX2-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: negative:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512F-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX512F-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: negative:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm0[2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm1 & ~ymm2) | ymm0
-; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm2[0,3,2,3]
-; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: negative:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BW-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX512BW-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vmovd %xmm1, %eax
+; AVX512BW-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: negative:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BWVL-NEXT: movl $65537, %eax # imm = 0x10001
-; AVX512BWVL-NEXT: kmovd %eax, %k1
-; AVX512BWVL-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
-; AVX512BWVL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512BWVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vmovd %xmm1, %eax
+; AVX512BWVL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
;
; AVX512VBMIVL-LABEL: negative:
; AVX512VBMIVL: # %bb.0:
-; AVX512VBMIVL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [32,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
-; AVX512VBMIVL-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512VBMIVL-NEXT: vpermt2b %ymm1, %ymm2, %ymm0
-; AVX512VBMIVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMIVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512VBMIVL-NEXT: vmovd %xmm1, %eax
+; AVX512VBMIVL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512VBMIVL-NEXT: vzeroupper
; AVX512VBMIVL-NEXT: retq
%strided.vec = shufflevector <32 x i8> %v, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
index e27a77ed2293d..0275c14bacc19 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
@@ -359,20 +359,25 @@ define <32 x i8> @trunc_shuffle_v32i16_v32i8_ofs1(<32 x i16> %a0) {
;
; AVX512BW-LABEL: trunc_shuffle_v32i16_v32i8_ofs1:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: trunc_shuffle_v32i16_v32i8_ofs1:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BWVL-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BWVL-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BWVL-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512BWVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512BWVL-NEXT: retq
;
; AVX512VBMI-LABEL: trunc_shuffle_v32i16_v32i8_ofs1:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512VBMI-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512VBMI-NEXT: vmovdqa {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
+; AVX512VBMI-NEXT: vpermb %zmm0, %zmm1, %zmm0
+; AVX512VBMI-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512VBMI-NEXT: retq
%bc = bitcast <32 x i16> %a0 to <64 x i8>
%res = shufflevector <64 x i8> %bc, <64 x i8> poison, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
diff --git a/llvm/test/CodeGen/X86/sse2.ll b/llvm/test/CodeGen/X86/sse2.ll
index 6e77d3e4fd134..1eb27b88a6c8e 100644
--- a/llvm/test/CodeGen/X86/sse2.ll
+++ b/llvm/test/CodeGen/X86/sse2.ll
@@ -571,12 +571,20 @@ define <2 x double> @test16(ptr nocapture %srcA, ptr nocapture %dst) {
; X86-SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
; X86-SSE-NEXT: retl
;
-; X86-AVX-LABEL: test16:
-; X86-AVX: # %bb.0:
-; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-AVX-NEXT: vmovaps 96(%eax), %xmm0
-; X86-AVX-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
-; X86-AVX-NEXT: retl
+; X86-AVX1-LABEL: test16:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-AVX1-NEXT: vmovaps 96(%eax), %xmm0
+; X86-AVX1-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
+; X86-AVX1-NEXT: retl
+;
+; X86-AVX512-LABEL: test16:
+; X86-AVX512: # %bb.0:
+; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512-NEXT: vpermpd {{.*#+}} ymm0 = mem[0,2,2,3]
+; X86-AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X86-AVX512-NEXT: vzeroupper
+; X86-AVX512-NEXT: retl
;
; X64-SSE-LABEL: test16:
; X64-SSE: # %bb.0:
@@ -584,11 +592,18 @@ define <2 x double> @test16(ptr nocapture %srcA, ptr nocapture %dst) {
; X64-SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
; X64-SSE-NEXT: retq
;
-; X64-AVX-LABEL: test16:
-; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: vmovaps 96(%rdi), %xmm0
-; X64-AVX-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
-; X64-AVX-NEXT: retq
+; X64-AVX1-LABEL: test16:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vmovaps 96(%rdi), %xmm0
+; X64-AVX1-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
+; X64-AVX1-NEXT: retq
+;
+; X64-AVX512-LABEL: test16:
+; X64-AVX512: # %bb.0:
+; X64-AVX512-NEXT: vpermpd {{.*#+}} ymm0 = mem[0,2,2,3]
+; X64-AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X64-AVX512-NEXT: vzeroupper
+; X64-AVX512-NEXT: retq
%i5 = getelementptr inbounds <4 x double>, ptr %srcA, i32 3
%i6 = load <4 x double>, ptr %i5, align 32
%i7 = shufflevector <4 x double> %i6, <4 x double> undef, <2 x i32> <i32 0, i32 2>
@@ -702,8 +717,3 @@ define <4 x i32> @test_mul(<4 x i32> %x, <4 x i32> %y) {
%m = mul <4 x i32> %x, %y
ret <4 x i32> %m
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; X64-AVX1: {{.*}}
-; X64-AVX512: {{.*}}
-; X86-AVX1: {{.*}}
-; X86-AVX512: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
index e0410ae0cc5cb..eeef87606376e 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
@@ -164,62 +164,62 @@ define void @load_i16_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i16_stride2_vf8:
; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i16_stride2_vf8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i16_stride2_vf8:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-FP-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FP-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-FP-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FP-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-FP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i16_stride2_vf8:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FCP-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-FCP-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FCP-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i16_stride2_vf8:
@@ -273,27 +273,27 @@ define void @load_i16_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) no
;
; AVX-LABEL: load_i16_stride2_vf16:
; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
-; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
-; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
-; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vmovdqa 32(%rdi), %xmm2
+; AVX-NEXT: vmovdqa 48(%rdi), %xmm3
+; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm3[0],xmm4[1],xmm3[2],xmm4[3],xmm3[4],xmm4[5],xmm3[6],xmm4[7]
+; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm4[1],xmm2[2],xmm4[3],xmm2[4],xmm4[5],xmm2[6],xmm4[7]
; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
-; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
-; AVX-NEXT: vpsrld $16, %xmm4, %xmm4
+; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm1[0],xmm4[1],xmm1[2],xmm4[3],xmm1[4],xmm4[5],xmm1[6],xmm4[7]
+; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm4[1],xmm0[2],xmm4[3],xmm0[4],xmm4[5],xmm0[6],xmm4[7]
+; AVX-NEXT: vpackusdw %xmm6, %xmm4, %xmm4
; AVX-NEXT: vpsrld $16, %xmm3, %xmm3
-; AVX-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa %xmm4, (%rsi)
; AVX-NEXT: vmovdqa %xmm5, 16(%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
-; AVX-NEXT: vmovdqa %xmm3, 16(%rdx)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX-NEXT: vmovdqa %xmm2, 16(%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i16_stride2_vf16:
@@ -354,14 +354,81 @@ define void @load_i16_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) no
; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
-; AVX512-LABEL: load_i16_stride2_vf16:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512-NEXT: vpsrld $16, %zmm0, %zmm1
-; AVX512-NEXT: vpmovdw %zmm0, (%rsi)
-; AVX512-NEXT: vpmovdw %zmm1, (%rdx)
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX512-VL-LABEL: load_i16_stride2_vf16:
+; AVX512-VL: # %bb.0:
+; AVX512-VL-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512-VL-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512-VL-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512-VL-NEXT: vpmovdw %zmm1, (%rdx)
+; AVX512-VL-NEXT: vzeroupper
+; AVX512-VL-NEXT: retq
+;
+; AVX512-FCP-LABEL: load_i16_stride2_vf16:
+; AVX512-FCP: # %bb.0:
+; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512-FCP-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512-FCP-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512-FCP-NEXT: vpmovdw %zmm1, (%rdx)
+; AVX512-FCP-NEXT: vzeroupper
+; AVX512-FCP-NEXT: retq
+;
+; AVX512DQ-LABEL: load_i16_stride2_vf16:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512DQ-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512DQ-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512DQ-NEXT: vpmovdw %zmm1, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
+;
+; AVX512DQ-FCP-LABEL: load_i16_stride2_vf16:
+; AVX512DQ-FCP: # %bb.0:
+; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512DQ-FCP-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vzeroupper
+; AVX512DQ-FCP-NEXT: retq
+;
+; AVX512BW-LABEL: load_i16_stride2_vf16:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-NEXT: vpermw %zmm0, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512BW-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BW-FCP-LABEL: load_i16_stride2_vf16:
+; AVX512BW-FCP: # %bb.0:
+; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512BW-FCP-NEXT: vzeroupper
+; AVX512BW-FCP-NEXT: retq
+;
+; AVX512DQ-BW-LABEL: load_i16_stride2_vf16:
+; AVX512DQ-BW: # %bb.0:
+; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512DQ-BW-NEXT: vpermw %zmm0, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-BW-NEXT: vzeroupper
+; AVX512DQ-BW-NEXT: retq
+;
+; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf16:
+; AVX512DQ-BW-FCP: # %bb.0:
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512DQ-BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vzeroupper
+; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <32 x i16>, ptr %in.vec, align 64
%strided.vec0 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%strided.vec1 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-2.ll
index 995d641644dfa..9f5beac10018b 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-2.ll
@@ -184,12 +184,16 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX2-FCP-LABEL: load_i32_stride2_vf4:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX2-FCP-NEXT: vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,2]
-; AVX2-FCP-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
-; AVX2-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX2-FCP-NEXT: vbroadcastf128 {{.*#+}} ymm1 = [0,2,4,6,0,2,4,6]
+; AVX2-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX2-FCP-NEXT: vpermps %ymm0, %ymm1, %ymm1
+; AVX2-FCP-NEXT: vbroadcastf128 {{.*#+}} ymm2 = [1,3,5,7,1,3,5,7]
+; AVX2-FCP-NEXT: # ymm2 = mem[0,1,0,1]
+; AVX2-FCP-NEXT: vpermps %ymm0, %ymm2, %ymm0
+; AVX2-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX2-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i32_stride2_vf4:
@@ -205,10 +209,10 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512-FCP-LABEL: load_i32_stride2_vf4:
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovaps (%rdi), %xmm1
-; AVX512-FCP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,3],mem[1,3]
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,0,0,0,0]
+; AVX512-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512-FCP-NEXT: vpmovqd %ymm0, (%rsi)
-; AVX512-FCP-NEXT: vmovaps %xmm1, (%rdx)
+; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -225,10 +229,10 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512DQ-FCP-LABEL: load_i32_stride2_vf4:
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,3],mem[1,3]
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,0,0,0,0]
+; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpmovqd %ymm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovaps %xmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
@@ -245,10 +249,10 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512BW-FCP-LABEL: load_i32_stride2_vf4:
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vmovaps (%rdi), %xmm1
-; AVX512BW-FCP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,3],mem[1,3]
+; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,0,0,0,0]
+; AVX512BW-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512BW-FCP-NEXT: vpmovqd %ymm0, (%rsi)
-; AVX512BW-FCP-NEXT: vmovaps %xmm1, (%rdx)
+; AVX512BW-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
@@ -265,10 +269,10 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512DQ-BW-FCP-LABEL: load_i32_stride2_vf4:
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,3],mem[1,3]
+; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,0,0,0,0]
+; AVX512DQ-BW-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512DQ-BW-FCP-NEXT: vpmovqd %ymm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <8 x i32>, ptr %in.vec, align 64
@@ -352,11 +356,10 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512-LABEL: load_i32_stride2_vf8:
; AVX512: # %bb.0:
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512-NEXT: vmovaps (%rdi), %ymm1
-; AVX512-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,3],mem[1,3],ymm1[5,7],mem[5,7]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -364,20 +367,19 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT: vpermps (%rdi), %zmm1, %zmm1
+; AVX512-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512-FCP-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i32_stride2_vf8:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-NEXT: vmovaps (%rdi), %ymm1
-; AVX512DQ-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,3],mem[1,3],ymm1[5,7],mem[5,7]
-; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512DQ-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512DQ-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512DQ-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -385,20 +387,19 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT: vpermps (%rdi), %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512DQ-FCP-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i32_stride2_vf8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT: vmovaps (%rdi), %ymm1
-; AVX512BW-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,3],mem[1,3],ymm1[5,7],mem[5,7]
-; AVX512BW-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512BW-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512BW-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512BW-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512BW-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -406,20 +407,19 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; AVX512BW-FCP-NEXT: vpermps (%rdi), %zmm1, %zmm1
+; AVX512BW-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512BW-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i32_stride2_vf8:
; AVX512DQ-BW: # %bb.0:
; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-BW-NEXT: vmovaps (%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,3],mem[1,3],ymm1[5,7],mem[5,7]
-; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512DQ-BW-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512DQ-BW-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512DQ-BW-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -427,9 +427,9 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; AVX512DQ-BW-FCP-NEXT: vpermps (%rdi), %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <16 x i32>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-2.ll
index aa7d8ceb14950..6833bc4063d5f 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-2.ll
@@ -40,112 +40,112 @@ define void @load_i64_stride2_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX2-LABEL: load_i64_stride2_vf2:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX2-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX2-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX2-NEXT: vmovaps %xmm2, (%rsi)
+; AVX2-NEXT: vmovaps (%rdi), %ymm0
+; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX2-NEXT: vmovaps %xmm1, (%rsi)
; AVX2-NEXT: vmovaps %xmm0, (%rdx)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i64_stride2_vf2:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-FP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX2-FP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX2-FP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX2-FP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX2-FP-NEXT: vmovaps (%rdi), %ymm0
+; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX2-FP-NEXT: vmovaps %xmm1, (%rsi)
; AVX2-FP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX2-FP-NEXT: vzeroupper
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i64_stride2_vf2:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX2-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX2-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX2-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX2-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i64_stride2_vf2:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovaps (%rdi), %xmm0
-; AVX512-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512-NEXT: vmovaps (%rdi), %ymm0
+; AVX512-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512-NEXT: vmovaps %xmm1, (%rsi)
; AVX512-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i64_stride2_vf2:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX512-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX512-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i64_stride2_vf2:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vmovaps (%rdi), %xmm0
-; AVX512DQ-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512DQ-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512DQ-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512DQ-NEXT: vmovaps (%rdi), %ymm0
+; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512DQ-NEXT: vmovaps %xmm1, (%rsi)
; AVX512DQ-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i64_stride2_vf2:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512DQ-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512DQ-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512DQ-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX512DQ-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i64_stride2_vf2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovaps (%rdi), %xmm0
-; AVX512BW-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512BW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512BW-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512BW-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512BW-NEXT: vmovaps (%rdi), %ymm0
+; AVX512BW-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512BW-NEXT: vmovaps %xmm1, (%rsi)
; AVX512BW-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: load_i64_stride2_vf2:
; AVX512BW-FCP: # %bb.0:
-; AVX512BW-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX512BW-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512BW-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512BW-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512BW-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512BW-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512BW-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512BW-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX512BW-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i64_stride2_vf2:
; AVX512DQ-BW: # %bb.0:
-; AVX512DQ-BW-NEXT: vmovaps (%rdi), %xmm0
-; AVX512DQ-BW-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512DQ-BW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512DQ-BW-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512DQ-BW-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512DQ-BW-NEXT: vmovaps (%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512DQ-BW-NEXT: vmovaps %xmm1, (%rsi)
; AVX512DQ-BW-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: load_i64_stride2_vf2:
; AVX512DQ-BW-FCP: # %bb.0:
-; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX512DQ-BW-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512DQ-BW-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512DQ-BW-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <4 x i64>, ptr %in.vec, align 64
%strided.vec0 = shufflevector <4 x i64> %wide.vec, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
@@ -231,101 +231,97 @@ define void @load_i64_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512-LABEL: load_i64_stride2_vf4:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovaps (%rdi), %ymm0
-; AVX512-NEXT: vmovaps 32(%rdi), %ymm1
-; AVX512-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512-NEXT: vmovaps %ymm2, (%rsi)
+; AVX512-NEXT: vmovaps (%rdi), %zmm0
+; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512-NEXT: vpermpd %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
+; AVX512-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: vmovaps %ymm1, (%rsi)
; AVX512-NEXT: vmovaps %ymm0, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i64_stride2_vf4:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,2,4,6]
-; AVX512-FCP-NEXT: vmovaps (%rdi), %zmm1
-; AVX512-FCP-NEXT: vpermpd %zmm1, %zmm0, %zmm0
+; AVX512-FCP-NEXT: vmovaps (%rdi), %zmm0
+; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm1
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
-; AVX512-FCP-NEXT: vpermpd %zmm1, %zmm2, %zmm1
-; AVX512-FCP-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512-FCP-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512-FCP-NEXT: vmovaps %ymm1, (%rsi)
+; AVX512-FCP-NEXT: vmovaps %ymm0, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i64_stride2_vf4:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vmovaps (%rdi), %ymm0
-; AVX512DQ-NEXT: vmovaps 32(%rdi), %ymm1
-; AVX512DQ-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512DQ-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512DQ-NEXT: vmovaps %ymm2, (%rsi)
+; AVX512DQ-NEXT: vmovaps (%rdi), %zmm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-NEXT: vpermpd %zmm0, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
+; AVX512DQ-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512DQ-NEXT: vmovaps %ymm1, (%rsi)
; AVX512DQ-NEXT: vmovaps %ymm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i64_stride2_vf4:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,2,4,6]
-; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %zmm1
-; AVX512DQ-FCP-NEXT: vpermpd %zmm1, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %zmm0
+; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm1
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
-; AVX512DQ-FCP-NEXT: vpermpd %zmm1, %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512DQ-FCP-NEXT: vmovaps %ymm1, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovaps %ymm0, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i64_stride2_vf4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovaps (%rdi), %ymm0
-; AVX512BW-NEXT: vmovaps 32(%rdi), %ymm1
-; AVX512BW-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512BW-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512BW-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BW-NEXT: vmovaps %ymm2, (%rsi)
+; AVX512BW-NEXT: vmovaps (%rdi), %zmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-NEXT: vpermpd %zmm0, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
+; AVX512BW-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vmovaps %ymm1, (%rsi)
; AVX512BW-NEXT: vmovaps %ymm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: load_i64_stride2_vf4:
; AVX512BW-FCP: # %bb.0:
-; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,2,4,6]
-; AVX512BW-FCP-NEXT: vmovaps (%rdi), %zmm1
-; AVX512BW-FCP-NEXT: vpermpd %zmm1, %zmm0, %zmm0
+; AVX512BW-FCP-NEXT: vmovaps (%rdi), %zmm0
+; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
-; AVX512BW-FCP-NEXT: vpermpd %zmm1, %zmm2, %zmm1
-; AVX512BW-FCP-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BW-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512BW-FCP-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512BW-FCP-NEXT: vmovaps %ymm1, (%rsi)
+; AVX512BW-FCP-NEXT: vmovaps %ymm0, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i64_stride2_vf4:
; AVX512DQ-BW: # %bb.0:
-; AVX512DQ-BW-NEXT: vmovaps (%rdi), %ymm0
-; AVX512DQ-BW-NEXT: vmovaps 32(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512DQ-BW-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512DQ-BW-NEXT: vmovaps %ymm2, (%rsi)
+; AVX512DQ-BW-NEXT: vmovaps (%rdi), %zmm0
+; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-BW-NEXT: vpermpd %zmm0, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
+; AVX512DQ-BW-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512DQ-BW-NEXT: vmovaps %ymm1, (%rsi)
; AVX512DQ-BW-NEXT: vmovaps %ymm0, (%rdx)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: load_i64_stride2_vf4:
; AVX512DQ-BW-FCP: # %bb.0:
-; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,2,4,6]
-; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %zmm1
-; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm1, %zmm0, %zmm0
+; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %zmm0
+; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
-; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm1, %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm1, (%rsi)
+; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm0, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <8 x i64>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
index b609299e5f757..b765cda89da01 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
@@ -416,165 +416,141 @@ define void @load_i8_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i8_stride2_vf16:
; AVX: # %bb.0:
-; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i8_stride2_vf16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i8_stride2_vf16:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-FP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-FP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX2-FP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-FP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-FP-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX2-FP-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-FP-NEXT: vzeroupper
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i8_stride2_vf16:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i8_stride2_vf16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i8_stride2_vf16:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i8_stride2_vf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512DQ-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512DQ-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512DQ-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512DQ-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX512DQ-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i8_stride2_vf16:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512DQ-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512DQ-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i8_stride2_vf16:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT: vpsrlw $8, %ymm0, %ymm1
+; AVX512BW-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX512BW-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512BW-NEXT: vpmovwb %ymm1, (%rdx)
+; AVX512BW-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: load_i8_stride2_vf16:
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vpsrlw $8, %ymm0, %ymm1
+; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX512BW-FCP-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512BW-FCP-NEXT: vpmovwb %ymm1, (%rdx)
+; AVX512BW-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i8_stride2_vf16:
; AVX512DQ-BW: # %bb.0:
; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-BW-NEXT: vpsrlw $8, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX512DQ-BW-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512DQ-BW-NEXT: vpmovwb %ymm1, (%rdx)
+; AVX512DQ-BW-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: load_i8_stride2_vf16:
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vpsrlw $8, %ymm0, %ymm1
+; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX512DQ-BW-FCP-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vpmovwb %ymm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <32 x i8>, ptr %in.vec, align 64
@@ -616,28 +592,28 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i8_stride2_vf32:
; AVX: # %bb.0:
-; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
-; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
-; AVX-NEXT: vpand %xmm0, %xmm4, %xmm5
-; AVX-NEXT: vpand %xmm0, %xmm3, %xmm6
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vmovdqa 32(%rdi), %xmm2
+; AVX-NEXT: vmovdqa 48(%rdi), %xmm3
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; AVX-NEXT: vpand %xmm4, %xmm3, %xmm5
+; AVX-NEXT: vpand %xmm4, %xmm2, %xmm6
; AVX-NEXT: vpackuswb %xmm5, %xmm6, %xmm5
-; AVX-NEXT: vpand %xmm0, %xmm2, %xmm6
-; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX-NEXT: vpackuswb %xmm6, %xmm0, %xmm0
+; AVX-NEXT: vpand %xmm4, %xmm1, %xmm6
+; AVX-NEXT: vpand %xmm4, %xmm0, %xmm4
+; AVX-NEXT: vpackuswb %xmm6, %xmm4, %xmm4
; AVX-NEXT: vmovq {{.*#+}} xmm6 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX-NEXT: vpshufb %xmm6, %xmm4, %xmm4
; AVX-NEXT: vpshufb %xmm6, %xmm3, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
; AVX-NEXT: vpshufb %xmm6, %xmm2, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; AVX-NEXT: vpshufb %xmm6, %xmm1, %xmm1
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX-NEXT: vpshufb %xmm6, %xmm0, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT: vmovdqa %xmm4, (%rsi)
; AVX-NEXT: vmovdqa %xmm5, 16(%rsi)
-; AVX-NEXT: vmovaps %ymm1, (%rdx)
+; AVX-NEXT: vmovaps %ymm0, (%rdx)
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
;
@@ -700,20 +676,20 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512-LABEL: load_i8_stride2_vf32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512-NEXT: vpshufb %ymm0, %ymm1, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
-; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -735,20 +711,20 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512DQ-LABEL: load_i8_stride2_vf32:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT: vpshufb %ymm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512DQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512DQ-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -771,36 +747,44 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512BW-LABEL: load_i8_stride2_vf32:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm1
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-NEXT: vpshufb {{.*#+}} zmm2 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT: vpermq %zmm2, %zmm1, %zmm1
; AVX512BW-NEXT: vpmovwb %zmm0, (%rsi)
-; AVX512BW-NEXT: vpmovwb %zmm1, (%rdx)
+; AVX512BW-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: load_i8_stride2_vf32:
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FCP-NEXT: vpsrlw $8, %zmm0, %zmm1
+; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} zmm2 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT: vpermq %zmm2, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vpmovwb %zmm0, (%rsi)
-; AVX512BW-FCP-NEXT: vpmovwb %zmm1, (%rdx)
+; AVX512BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i8_stride2_vf32:
; AVX512DQ-BW: # %bb.0:
; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-BW-NEXT: vpsrlw $8, %zmm0, %zmm1
+; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} zmm2 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-NEXT: vpermq %zmm2, %zmm1, %zmm1
; AVX512DQ-BW-NEXT: vpmovwb %zmm0, (%rsi)
-; AVX512DQ-BW-NEXT: vpmovwb %zmm1, (%rdx)
+; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: load_i8_stride2_vf32:
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-BW-FCP-NEXT: vpsrlw $8, %zmm0, %zmm1
+; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} zmm2 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT: vpermq %zmm2, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpmovwb %zmm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vpmovwb %zmm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <64 x i8>, ptr %in.vec, align 64
@@ -1012,123 +996,123 @@ define void @load_i8_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512-LABEL: load_i8_stride2_vf64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX512-NEXT: vpshufb %ymm0, %ymm4, %ymm5
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa 64(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa 96(%rdi), %ymm3
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-NEXT: vpshufb %ymm4, %ymm3, %ymm5
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512-NEXT: vpshufb %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm0
-; AVX512-NEXT: vpshufb %ymm6, %ymm1, %ymm6
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5],ymm0[6,7]
-; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0
-; AVX512-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512-NEXT: vpshufb %ymm4, %ymm1, %ymm4
+; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm6
+; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]
+; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512-NEXT: vpermq {{.*#+}} zmm4 = zmm4[0,2,1,3,4,6,5,7]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX512-NEXT: vpshufb %ymm5, %ymm3, %ymm3
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-NEXT: vpshufb %ymm6, %ymm3, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
-; AVX512-NEXT: vpshufb %ymm5, %ymm2, %ymm2
-; AVX512-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,2,1,3,4,6,5,7]
-; AVX512-NEXT: vmovdqa64 %zmm0, (%rsi)
-; AVX512-NEXT: vmovdqa64 %zmm1, (%rdx)
+; AVX512-NEXT: vpshufb %ymm6, %ymm2, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm0
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i8_stride2_vf64:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm5
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm3
+; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm7
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,2,5,7]
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm8, %ymm7
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm5
-; AVX512-FCP-NEXT: vpermt2q %ymm0, %ymm8, %ymm5
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm0
+; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm4
+; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm5
+; AVX512-FCP-NEXT: vpermt2q %ymm4, %ymm8, %ymm5
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm4
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm3
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
-; AVX512-FCP-NEXT: vpermt2q %ymm4, %ymm8, %ymm3
-; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm2
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-FCP-NEXT: vpermt2q %ymm2, %ymm8, %ymm1
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rsi)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm1, (%rdx)
+; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
+; AVX512-FCP-NEXT: vpermt2q %ymm3, %ymm8, %ymm2
+; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
+; AVX512-FCP-NEXT: vpermt2q %ymm1, %ymm8, %ymm0
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-FCP-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i8_stride2_vf64:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm4, %ymm5
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm3
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-NEXT: vpshufb %ymm4, %ymm3, %ymm5
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512DQ-NEXT: vpshufb %ymm6, %ymm2, %ymm7
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-NEXT: vpshufb %ymm6, %ymm1, %ymm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5],ymm0[6,7]
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512DQ-NEXT: vpshufb %ymm4, %ymm1, %ymm4
+; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm6
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512DQ-NEXT: vpermq {{.*#+}} zmm4 = zmm4[0,2,1,3,4,6,5,7]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX512DQ-NEXT: vpshufb %ymm5, %ymm3, %ymm3
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-NEXT: vpshufb %ymm6, %ymm3, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpshufb %ymm5, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,2,1,3,4,6,5,7]
-; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa64 %zmm1, (%rdx)
+; AVX512DQ-NEXT: vpshufb %ymm6, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512DQ-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i8_stride2_vf64:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm7
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,2,5,7]
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm8, %ymm7
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm5
-; AVX512DQ-FCP-NEXT: vpermt2q %ymm0, %ymm8, %ymm5
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm4
+; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm5
+; AVX512DQ-FCP-NEXT: vpermt2q %ymm4, %ymm8, %ymm5
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm4
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm3
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT: vpermt2q %ymm4, %ymm8, %ymm3
-; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vpermt2q %ymm2, %ymm8, %ymm1
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT: vpermt2q %ymm3, %ymm8, %ymm2
+; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT: vpermt2q %ymm1, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-narrow-binop.ll b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
index ad345213c1472..801f5929e35d8 100644
--- a/llvm/test/CodeGen/X86/vector-narrow-binop.ll
+++ b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
@@ -60,15 +60,15 @@ define <8 x i32> @PR32790(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d
define <4 x i32> @do_not_use_256bit_op(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) {
; SSE-LABEL: do_not_use_256bit_op:
; SSE: # %bb.0:
-; SSE-NEXT: pand %xmm2, %xmm0
; SSE-NEXT: pand %xmm3, %xmm1
+; SSE-NEXT: pand %xmm2, %xmm0
; SSE-NEXT: psubd %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX-LABEL: do_not_use_256bit_op:
; AVX: # %bb.0:
-; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX-NEXT: vpand %xmm3, %xmm1, %xmm1
+; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%concat1 = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
index 995607a6857bd..fb051be560f0d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -412,19 +412,12 @@ define <16 x i32> @shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u(<16 x i32> %a
;FIXME: can do better with vpcompress
define <8 x i32> @test_v16i32_1_3_5_7_9_11_13_15(<16 x i32> %v) {
-; SLOW-LABEL: test_v16i32_1_3_5_7_9_11_13_15:
-; SLOW: # %bb.0:
-; SLOW-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; SLOW-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; SLOW-NEXT: retq
-;
-; FAST-LABEL: test_v16i32_1_3_5_7_9_11_13_15:
-; FAST: # %bb.0:
-; FAST-NEXT: vmovaps {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; FAST-NEXT: vpermps %zmm0, %zmm1, %zmm0
-; FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; FAST-NEXT: retq
+; ALL-LABEL: test_v16i32_1_3_5_7_9_11_13_15:
+; ALL: # %bb.0:
+; ALL-NEXT: vmovaps {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; ALL-NEXT: vpermps %zmm0, %zmm1, %zmm0
+; ALL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; ALL-NEXT: retq
%res = shufflevector <16 x i32> %v, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
ret <8 x i32> %res
}
diff --git a/llvm/test/CodeGen/X86/vpdpwssd.ll b/llvm/test/CodeGen/X86/vpdpwssd.ll
index ea97800505bc2..339c04611e318 100644
--- a/llvm/test/CodeGen/X86/vpdpwssd.ll
+++ b/llvm/test/CodeGen/X86/vpdpwssd.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI,AVX512VL-VNNI-SLOW
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512-VNNI
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI,AVX512VL-VNNI-FAST
define <16 x i32> @vpdpwssd_test(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) {
; CHECK-LABEL: vpdpwssd_test:
@@ -31,23 +31,53 @@ define <16 x i32> @vpdpwssd_v16i32_accumulate(<32 x i16> %a0, <32 x i16> %a1, <1
}
define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x i32> %a2) {
-; AVX512VL-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
-; AVX512VL-VNNI: # %bb.0:
-; AVX512VL-VNNI-NEXT: vpdpwssd %ymm1, %ymm0, %ymm2
-; AVX512VL-VNNI-NEXT: vmovdqa %ymm2, %ymm0
-; AVX512VL-VNNI-NEXT: retq
+; AVX512VL-VNNI-SLOW-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI-SLOW: # %bb.0:
+; AVX512VL-VNNI-SLOW-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512VL-VNNI-SLOW-NEXT: vmovdqa {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
+; AVX512VL-VNNI-SLOW-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512VL-VNNI-SLOW-NEXT: vpermd %zmm0, %zmm3, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512VL-VNNI-SLOW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512VL-VNNI-SLOW-NEXT: retq
;
; AVX-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
; AVX-VNNI: # %bb.0:
-; AVX-VNNI-NEXT: {vex} vpdpwssd %ymm1, %ymm0, %ymm2
-; AVX-VNNI-NEXT: vmovdqa %ymm2, %ymm0
+; AVX-VNNI-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX-VNNI-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX-VNNI-NEXT: vmovdqa {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
+; AVX-VNNI-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX-VNNI-NEXT: vpmovqd %zmm0, %ymm1
+; AVX-VNNI-NEXT: vpermd %zmm0, %zmm3, %zmm0
+; AVX-VNNI-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX-VNNI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX-VNNI-NEXT: retq
;
; AVX512-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
; AVX512-VNNI: # %bb.0:
-; AVX512-VNNI-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-VNNI-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-VNNI-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-VNNI-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-VNNI-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512-VNNI-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
+; AVX512-VNNI-NEXT: vpermd %zmm0, %zmm3, %zmm0
+; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512-VNNI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX512-VNNI-NEXT: retq
+;
+; AVX512VL-VNNI-FAST-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI-FAST: # %bb.0:
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512VL-VNNI-FAST-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
+; AVX512VL-VNNI-FAST-NEXT: vpermd %zmm0, %zmm3, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512VL-VNNI-FAST-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512VL-VNNI-FAST-NEXT: retq
%x0 = sext <16 x i16> %a0 to <16 x i32>
%x1 = sext <16 x i16> %a1 to <16 x i32>
%m = mul nsw <16 x i32> %x0, %x1
@@ -61,20 +91,41 @@ define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x
define <4 x i32> @vpdpwssd_v4i32_accumulate(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
; AVX512VL-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX512VL-VNNI: # %bb.0:
-; AVX512VL-VNNI-NEXT: vpdpwssd %xmm1, %xmm0, %xmm2
-; AVX512VL-VNNI-NEXT: vmovdqa %xmm2, %xmm0
+; AVX512VL-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512VL-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512VL-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX512VL-VNNI-NEXT: vpmovqd %ymm0, %xmm1
+; AVX512VL-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512VL-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
+; AVX512VL-VNNI-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512VL-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512VL-VNNI-NEXT: vzeroupper
; AVX512VL-VNNI-NEXT: retq
;
; AVX-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX-VNNI: # %bb.0:
-; AVX-VNNI-NEXT: {vex} vpdpwssd %xmm1, %xmm0, %xmm2
-; AVX-VNNI-NEXT: vmovdqa %xmm2, %xmm0
+; AVX-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX-VNNI-NEXT: vpmovqd %ymm0, %xmm1
+; AVX-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
+; AVX-VNNI-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX-VNNI-NEXT: vzeroupper
; AVX-VNNI-NEXT: retq
;
; AVX512-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX512-VNNI: # %bb.0:
-; AVX512-VNNI-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} xmm3 = xmm0[0,2],xmm1[0,2]
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
+; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm3, %xmm1
+; AVX512-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-VNNI-NEXT: vzeroupper
; AVX512-VNNI-NEXT: retq
%x0 = sext <8 x i16> %a0 to <8 x i32>
%x1 = sext <8 x i16> %a1 to <8 x i32>
diff --git a/llvm/test/CodeGen/X86/vselect-avx.ll b/llvm/test/CodeGen/X86/vselect-avx.ll
index ac330a7e60396..8c5db490cce73 100644
--- a/llvm/test/CodeGen/X86/vselect-avx.ll
+++ b/llvm/test/CodeGen/X86/vselect-avx.ll
@@ -263,11 +263,11 @@ define <4 x i64> @vselect_concat_split_v16i8(<4 x i64> %a, <4 x i64> %b, <4 x i6
; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5
; AVX1-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
; AVX1-NEXT: vpcmpgtb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT: vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: vselect_concat_split_v16i8:
More information about the llvm-commits
mailing list