[llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 14:16:35 PDT 2026
- Previous message: [llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
- Next message: [llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/180573
>From eb180585c69e29f08267b8b1edb25d021a928f03 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 9 Feb 2026 17:46:25 +0000
Subject: [PATCH] [DAG] SelectionDAGBuilder::visitShuffleVector - split
shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns
If an unary shuffle sources are twice the width of the result, then split the lower/upper subvectors and perform a binary shuffle instead of scalarizing
I'm still cautious here as shuffle legalisation of illegal types can be tricky.
Fixes #88030
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 24 +
.../aarch64-bf16-dotprod-intrinsics.ll | 3 +-
llvm/test/CodeGen/AArch64/addp-shuffle.ll | 12 +-
llvm/test/CodeGen/AArch64/arm64-fp.ll | 6 +-
.../test/CodeGen/AArch64/arm64-neon-2velem.ll | 21 +-
llvm/test/CodeGen/AArch64/arm64-vmul.ll | 4 +-
.../CodeGen/AArch64/bf16-vector-shuffle.ll | 3 +-
.../complex-deinterleaving-mixed-cases.ll | 146 +-
.../complex-deinterleaving-multiuses.ll | 142 +-
.../complex-deinterleaving-uniform-cases.ll | 34 +-
llvm/test/CodeGen/AArch64/concat-vector.ll | 76 +-
.../CodeGen/AArch64/highextractbitcast.ll | 16 +-
.../sve-fixed-length-extract-subvector.ll | 7 +-
.../v8.2a-neon-intrinsics-constrained.ll | 6 +-
...ffer-fat-pointers-contents-legalization.ll | 44 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr.ll | 54 +-
llvm/test/CodeGen/AMDGPU/sdiv.ll | 40 +-
.../AMDGPU/shufflevector.v2f32.v4f32.ll | 16 +-
.../AMDGPU/shufflevector.v2i32.v4i32.ll | 16 +-
.../CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll | 16 +-
llvm/test/CodeGen/AMDGPU/sra.ll | 20 +-
llvm/test/CodeGen/AMDGPU/srem.ll | 26 +-
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 544 ++++----
.../ARM/arm-bf16-dotprod-intrinsics.ll | 3 +-
.../CodeGen/ARM/bf16-create-get-set-dup.ll | 3 +-
llvm/test/CodeGen/ARM/bf16-shuffle.ll | 3 +-
llvm/test/CodeGen/ARM/nop_concat_vectors.ll | 2 +
llvm/test/CodeGen/ARM/vext.ll | 5 +-
llvm/test/CodeGen/ARM/vpadd.ll | 27 +-
llvm/test/CodeGen/ARM/vuzp.ll | 10 +-
llvm/test/CodeGen/ARM/vzip.ll | 5 +-
.../CodeGen/LoongArch/lasx/and-not-combine.ll | 20 +-
.../lasx/insert-extract-subvector.ll | 24 +-
.../fixed-vectors-shuffle-changes-length.ll | 114 +-
.../rvv/fixed-vectors-shuffle-deinterleave.ll | 159 +--
.../fixed-vectors-shuffle-deinterleave2.ll | 152 ++-
.../RISCV/rvv/fixed-vectors-shuffle-int.ll | 7 +-
llvm/test/CodeGen/Thumb2/mve-shuffle.ll | 8 +-
.../any_extend_vector_inreg_of_broadcast.ll | 268 ++--
...d_vector_inreg_of_broadcast_from_memory.ll | 174 +--
llvm/test/CodeGen/X86/avx512-mask-op.ll | 5 +-
.../X86/avx512-shuffles/partial_permute.ll | 1207 +++++++----------
.../test/CodeGen/X86/horizontal-reduce-add.ll | 12 +-
.../CodeGen/X86/horizontal-reduce-fadd.ll | 8 +-
llvm/test/CodeGen/X86/madd.ll | 377 ++++-
llvm/test/CodeGen/X86/matrix-multiply.ll | 43 +-
llvm/test/CodeGen/X86/pclmulqdq.ll | 26 +-
llvm/test/CodeGen/X86/pmaddubsw.ll | 352 ++++-
llvm/test/CodeGen/X86/pmul.ll | 6 +-
llvm/test/CodeGen/X86/pr167793.ll | 9 +-
.../test/CodeGen/X86/setcc-non-simple-type.ll | 35 +-
.../X86/shuffle-strided-with-offset-256.ll | 180 ++-
.../X86/shuffle-strided-with-offset-512.ll | 120 +-
llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll | 541 ++------
llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll | 149 +-
llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll | 17 +-
llvm/test/CodeGen/X86/sse2.ll | 42 +-
.../vector-interleaved-load-i16-stride-2.ll | 195 ++-
.../vector-interleaved-load-i32-stride-2.ll | 82 +-
.../vector-interleaved-load-i64-stride-2.ll | 210 ++-
.../vector-interleaved-load-i8-stride-2.ll | 452 +++---
.../CodeGen/X86/vector-shuffle-512-v16.ll | 19 +-
llvm/test/CodeGen/X86/vpdpwssd.ll | 85 +-
63 files changed, 3225 insertions(+), 3207 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1f3b099c9c577..2d9261a727abf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4280,6 +4280,30 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
assert(SrcNumElts > MaskNumElts);
+ // See if we can recreate a binary shuffle by splitting or widening an unary
+ // shuffle with sources twice the destination size.
+ if (SrcNumElts == (MaskNumElts * 2) && Src2.isUndef() &&
+ TLI.isTypeLegal(VT)) {
+ if (Src1.getOpcode() == ISD::CONCAT_VECTORS) {
+ SmallVector<int, 16> SplitMask(Mask.begin(), Mask.end());
+ for (int &M : SplitMask)
+ M = M >= (int)SrcNumElts ? -1 : M;
+ SDValue LHS = DAG.getExtractSubvector(DL, VT, Src1, 0);
+ SDValue RHS = DAG.getExtractSubvector(DL, VT, Src1, MaskNumElts);
+ SDValue Result = DAG.getVectorShuffle(VT, DL, LHS, RHS, SplitMask);
+ setValue(&I, Result);
+ return;
+ }
+ if (TLI.isTypeLegal(SrcVT) || isa<LoadSDNode>(Src1)) {
+ SmallVector<int, 16> WideMask(Mask.begin(), Mask.end());
+ WideMask.append(SrcNumElts - MaskNumElts, -1);
+ SDValue Wide = DAG.getVectorShuffle(SrcVT, DL, Src1, Src2, WideMask);
+ SDValue Result = DAG.getExtractSubvector(DL, VT, Wide, 0);
+ setValue(&I, Result);
+ return;
+ }
+ }
+
// Analyze the access pattern of the vector to see if we can extract
// two subvectors and do the shuffle.
int StartIdx[2] = {-1, -1}; // StartIdx to extract from
diff --git a/llvm/test/CodeGen/AArch64/aarch64-bf16-dotprod-intrinsics.ll b/llvm/test/CodeGen/AArch64/aarch64-bf16-dotprod-intrinsics.ll
index ca3cd6bbae549..319b42f73a434 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-bf16-dotprod-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-bf16-dotprod-intrinsics.ll
@@ -51,7 +51,8 @@ entry:
define <2 x float> @test_vbfdot_laneq_f32(<2 x float> %r, <4 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-LABEL: test_vbfdot_laneq_f32:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: bfdot v0.2s, v1.4h, v2.2h[3]
+; CHECK-NEXT: dup v2.4s, v2.s[3]
+; CHECK-NEXT: bfdot v0.2s, v1.4h, v2.4h
; CHECK-NEXT: ret
entry:
%.cast = bitcast <8 x bfloat> %b to <4 x float>
diff --git a/llvm/test/CodeGen/AArch64/addp-shuffle.ll b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
index cf02a090b8e3c..70949bb64180d 100644
--- a/llvm/test/CodeGen/AArch64/addp-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
@@ -27,8 +27,9 @@ define <4 x i32> @deinterleave_shuffle_v8i32_c(<8 x i32> %a) {
define <2 x i32> @deinterleave_shuffle_v4i32(<4 x i32> %a) {
; CHECK-LABEL: deinterleave_shuffle_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: addp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: xtn v1.2s, v0.2d
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: add v0.2s, v1.2s, v0.2s
; CHECK-NEXT: ret
%r0 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
%r1 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
@@ -387,10 +388,9 @@ define <4 x i32> @or_deinterleave_shuffle_v8i32_c(<8 x i32> %a) {
define <2 x i32> @or_deinterleave_shuffle_v4i32(<4 x i32> %a) {
; CHECK-LABEL: or_deinterleave_shuffle_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-NEXT: xtn v1.2s, v0.2d
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: orr v0.8b, v1.8b, v0.8b
; CHECK-NEXT: ret
%r0 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
%r1 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
diff --git a/llvm/test/CodeGen/AArch64/arm64-fp.ll b/llvm/test/CodeGen/AArch64/arm64-fp.ll
index c7ef439b764dd..f347953216907 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fp.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fp.ll
@@ -52,7 +52,8 @@ define <2 x float> @fake_fneg_splat_extract(<4 x float> %rhs) {
; CHECK-LABEL: fake_fneg_splat_extract:
; CHECK: // %bb.0:
; CHECK-NEXT: fneg v0.4s, v0.4s
-; CHECK-NEXT: dup v0.2s, v0.s[3]
+; CHECK-NEXT: dup v0.4s, v0.s[3]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
%rhs_neg = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %rhs
%splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <2 x i32> <i32 3, i32 3>
@@ -63,7 +64,8 @@ define <2 x float> @fake_fneg_splat_extract_undef(<4 x float> %rhs) {
; CHECK-LABEL: fake_fneg_splat_extract_undef:
; CHECK: // %bb.0:
; CHECK-NEXT: fneg v0.4s, v0.4s
-; CHECK-NEXT: dup v0.2s, v0.s[3]
+; CHECK-NEXT: dup v0.4s, v0.s[3]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
%rhs_neg = fsub <4 x float> <float undef, float -0.0, float -0.0, float -0.0>, %rhs
%splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <2 x i32> <i32 3, i32 3>
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll b/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
index 85d8b7c3e2866..ffdbc8005d4ab 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-2velem.ll
@@ -519,10 +519,16 @@ entry:
declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
define <2 x float> @test_vfma_laneq_f32(<2 x float> %a, <2 x float> %b, <4 x float> %v) {
-; CHECK-LABEL: test_vfma_laneq_f32:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fmla v0.2s, v1.2s, v2.s[3]
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test_vfma_laneq_f32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: dup v2.4s, v2.s[3]
+; CHECK-SD-NEXT: fmla v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_vfma_laneq_f32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: fmla v0.2s, v1.2s, v2.s[3]
+; CHECK-GI-NEXT: ret
entry:
%lane = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>
%0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)
@@ -569,7 +575,9 @@ entry:
define <2 x float> @test_vfms_laneq_f32(<2 x float> %a, <2 x float> %b, <4 x float> %v) {
; CHECK-LABEL: test_vfms_laneq_f32:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fmls v0.2s, v1.2s, v2.s[3]
+; CHECK-NEXT: fneg v2.4s, v2.4s
+; CHECK-NEXT: dup v2.4s, v2.s[3]
+; CHECK-NEXT: fmla v0.2s, v1.2s, v2.2s
; CHECK-NEXT: ret
entry:
%sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v
@@ -2262,7 +2270,8 @@ define <4 x i16> @test_vadd_lane2_i16_bitcast_bigger_aligned(<4 x i16> %a, <16 x
define <4 x i16> @test_vadd_lane5_i16_bitcast_bigger_aligned(<4 x i16> %a, <16 x i8> %v) {
; CHECK-SD-LABEL: test_vadd_lane5_i16_bitcast_bigger_aligned:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: dup v1.4h, v1.h[5]
+; CHECK-SD-NEXT: dup v1.2d, v1.d[1]
+; CHECK-SD-NEXT: dup v1.4h, v1.h[1]
; CHECK-SD-NEXT: add v0.4h, v1.4h, v0.4h
; CHECK-SD-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/arm64-vmul.ll b/llvm/test/CodeGen/AArch64/arm64-vmul.ll
index 70c02a2a20353..5266aeea0e12e 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vmul.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vmul.ll
@@ -3166,7 +3166,9 @@ declare double @llvm.fma.f64(double, double, double)
define <2 x float> @fmls_with_fneg_before_extract_v2f32(<2 x float> %accum, <2 x float> %lhs, <4 x float> %rhs) {
; CHECK-LABEL: fmls_with_fneg_before_extract_v2f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmls v0.2s, v1.2s, v2.s[3]
+; CHECK-NEXT: fneg v2.4s, v2.4s
+; CHECK-NEXT: dup v2.4s, v2.s[3]
+; CHECK-NEXT: fmla v0.2s, v2.2s, v1.2s
; CHECK-NEXT: ret
%rhs_neg = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %rhs
%splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <2 x i32> <i32 3, i32 3>
diff --git a/llvm/test/CodeGen/AArch64/bf16-vector-shuffle.ll b/llvm/test/CodeGen/AArch64/bf16-vector-shuffle.ll
index 222d7435ff742..b2c8ca8d05f2c 100644
--- a/llvm/test/CodeGen/AArch64/bf16-vector-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-vector-shuffle.ll
@@ -66,7 +66,8 @@ entry:
define <4 x bfloat> @test_vdup_laneq_bf16(<8 x bfloat> %v) nounwind {
; CHECK-LABEL: test_vdup_laneq_bf16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: dup v0.4h, v0.h[7]
+; CHECK-NEXT: dup v0.8h, v0.h[7]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
entry:
%lane = shufflevector <8 x bfloat> %v, <8 x bfloat> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
index 1ed9cf2db24f7..5d89a51597bf1 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
@@ -44,16 +44,13 @@ define <4 x float> @add_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: fsub v0.4s, v1.4s, v0.4s
; CHECK-NEXT: fsub v1.4s, v1.4s, v2.4s
-; CHECK-NEXT: ext v3.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v5.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip2 v4.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: fmul v5.2s, v4.2s, v0.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fneg v4.2s, v5.2s
+; CHECK-NEXT: uzp2 v3.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v4.2s, v3.2s, v0.2s
+; CHECK-NEXT: uzp1 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmul v3.2s, v1.2s, v3.2s
+; CHECK-NEXT: fneg v4.2s, v4.2s
; CHECK-NEXT: fmla v3.2s, v0.2s, v2.2s
; CHECK-NEXT: fmla v4.2s, v1.2s, v2.2s
; CHECK-NEXT: zip1 v0.4s, v4.4s, v3.4s
@@ -79,24 +76,21 @@ entry:
define <4 x float> @mul_mul270_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: mul_mul270_mul:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v3.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v5.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: fmul v7.2s, v6.2s, v5.2s
-; CHECK-NEXT: fneg v4.2s, v7.2s
-; CHECK-NEXT: zip2 v7.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
-; CHECK-NEXT: fmla v4.2s, v2.2s, v1.2s
-; CHECK-NEXT: fmul v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v4.2s, v7.2s
-; CHECK-NEXT: fmla v1.2s, v2.2s, v6.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v0.2s
+; CHECK-NEXT: uzp1 v3.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v6.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v5.2s, v4.2s, v3.2s
+; CHECK-NEXT: fneg v5.2s, v5.2s
+; CHECK-NEXT: fmla v5.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmul v1.2s, v1.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v5.2s, v6.2s
+; CHECK-NEXT: fmla v1.2s, v2.2s, v4.2s
+; CHECK-NEXT: fmul v2.2s, v5.2s, v0.2s
; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v2.2s, v7.2s, v1.2s
+; CHECK-NEXT: fmla v2.2s, v6.2s, v1.2s
; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
; CHECK-NEXT: zip1 v0.4s, v3.4s, v2.4s
; CHECK-NEXT: ret
@@ -258,25 +252,22 @@ entry:
define <4 x float> @mul_triangle_addmul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: mul_triangle_addmul:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v5.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
-; CHECK-NEXT: ext v4.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v3.2s
-; CHECK-NEXT: fmul v7.2s, v6.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v4.2s
-; CHECK-NEXT: fmov d4, d7
-; CHECK-NEXT: fmov d16, d5
-; CHECK-NEXT: fmls v7.2s, v0.2s, v2.2s
-; CHECK-NEXT: fmla v5.2s, v0.2s, v3.2s
-; CHECK-NEXT: fmls v4.2s, v0.2s, v1.2s
-; CHECK-NEXT: fmla v16.2s, v0.2s, v6.2s
-; CHECK-NEXT: fsub v0.2s, v7.2s, v16.2s
-; CHECK-NEXT: fadd v1.2s, v5.2s, v4.2s
+; CHECK-NEXT: uzp1 v3.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v5.2s, v4.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v1.2s, v3.2s
+; CHECK-NEXT: uzp1 v6.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmov d7, d5
+; CHECK-NEXT: fmov d16, d3
+; CHECK-NEXT: fmls v5.2s, v0.2s, v2.2s
+; CHECK-NEXT: fmla v3.2s, v0.2s, v6.2s
+; CHECK-NEXT: fmls v7.2s, v0.2s, v1.2s
+; CHECK-NEXT: fmla v16.2s, v0.2s, v4.2s
+; CHECK-NEXT: fsub v0.2s, v5.2s, v16.2s
+; CHECK-NEXT: fadd v1.2s, v3.2s, v7.2s
; CHECK-NEXT: zip1 v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
entry:
@@ -311,24 +302,22 @@ entry:
define <4 x float> @mul_triangle_multiuses(<4 x float> %a, <4 x float> %b, ptr %p) {
; CHECK-LABEL: mul_triangle_multiuses:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v2.2s, v0.2s
-; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
-; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: mov v1.d[1], v2.d[0]
-; CHECK-NEXT: zip1 v0.4s, v3.4s, v5.4s
+; CHECK-NEXT: uzp2 v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v4.2s, v2.2s, v3.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v2.2s
+; CHECK-NEXT: fmla v4.2s, v0.2s, v1.2s
+; CHECK-NEXT: fneg v1.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v4.2s, v2.2s
+; CHECK-NEXT: fmla v1.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v4.2s, v0.2s
+; CHECK-NEXT: fneg v5.2s, v5.2s
+; CHECK-NEXT: fmla v3.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
+; CHECK-NEXT: mov v1.d[1], v4.d[0]
+; CHECK-NEXT: zip1 v0.4s, v5.4s, v3.4s
; CHECK-NEXT: str q1, [x0]
; CHECK-NEXT: ret
entry:
@@ -442,23 +431,20 @@ entry:
define <4 x float> @mul_divequal(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: mul_divequal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip2 v5.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip2 v6.2s, v1.2s, v4.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v4.2s
-; CHECK-NEXT: ext v3.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: fmul v7.2s, v5.2s, v6.2s
-; CHECK-NEXT: fneg v4.2s, v7.2s
-; CHECK-NEXT: zip1 v7.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: fmla v4.2s, v0.2s, v1.2s
-; CHECK-NEXT: fmul v0.2s, v6.2s, v0.2s
-; CHECK-NEXT: fmla v0.2s, v5.2s, v1.2s
-; CHECK-NEXT: fdiv v4.2s, v4.2s, v7.2s
+; CHECK-NEXT: uzp2 v3.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v4.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v5.2s, v3.2s, v4.2s
+; CHECK-NEXT: uzp1 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v6.4s, v2.4s, v0.4s
+; CHECK-NEXT: fneg v5.2s, v5.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
+; CHECK-NEXT: fmul v0.2s, v4.2s, v0.2s
+; CHECK-NEXT: uzp2 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmla v0.2s, v3.2s, v1.2s
+; CHECK-NEXT: fdiv v5.2s, v5.2s, v6.2s
; CHECK-NEXT: fdiv v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v0.4s, v4.4s, v0.4s
+; CHECK-NEXT: zip1 v0.4s, v5.4s, v0.4s
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
index 039025dafa0d6..5654feb5a4356 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
@@ -46,24 +46,22 @@ entry:
define <4 x float> @mul_triangle_external_use(<4 x float> %a, <4 x float> %b, ptr %p) {
; CHECK-LABEL: mul_triangle_external_use:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: str d2, [x0]
-; CHECK-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v2.2s, v0.2s
-; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
-; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: zip1 v0.4s, v3.4s, v5.4s
+; CHECK-NEXT: uzp2 v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v4.2s, v2.2s, v3.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v2.2s
+; CHECK-NEXT: fmla v4.2s, v0.2s, v1.2s
+; CHECK-NEXT: fneg v1.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v4.2s, v2.2s
+; CHECK-NEXT: str d4, [x0]
+; CHECK-NEXT: fmla v1.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmul v3.2s, v4.2s, v0.2s
+; CHECK-NEXT: fneg v5.2s, v5.2s
+; CHECK-NEXT: fmla v3.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
+; CHECK-NEXT: zip1 v0.4s, v5.4s, v3.4s
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -94,29 +92,26 @@ entry:
define <4 x float> @multiple_muls_shuffle_external(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
; CHECK-LABEL: multiple_muls_shuffle_external:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v5.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v6.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: ext v4.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: zip2 v7.2s, v0.2s, v5.2s
-; CHECK-NEXT: zip1 v16.2s, v1.2s, v6.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v6.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v16.2s, v7.2s
-; CHECK-NEXT: fmul v6.2s, v1.2s, v7.2s
-; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v6.2s
-; CHECK-NEXT: zip1 v6.2s, v2.2s, v4.2s
-; CHECK-NEXT: zip2 v4.2s, v2.2s, v4.2s
-; CHECK-NEXT: fmla v1.2s, v0.2s, v16.2s
-; CHECK-NEXT: fmul v17.2s, v6.2s, v5.2s
+; CHECK-NEXT: uzp2 v5.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v6.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v4.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v7.2s, v6.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v5.2s
+; CHECK-NEXT: fmla v7.2s, v0.2s, v1.2s
+; CHECK-NEXT: fneg v1.2s, v5.2s
+; CHECK-NEXT: uzp2 v5.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmla v1.2s, v0.2s, v6.2s
+; CHECK-NEXT: fmul v17.2s, v4.2s, v7.2s
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: fmul v5.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmla v17.2s, v1.2s, v4.2s
+; CHECK-NEXT: fmul v6.2s, v5.2s, v7.2s
+; CHECK-NEXT: fmla v17.2s, v1.2s, v5.2s
; CHECK-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
; CHECK-NEXT: str d1, [x0]
-; CHECK-NEXT: fneg v16.2s, v5.2s
+; CHECK-NEXT: fneg v16.2s, v6.2s
; CHECK-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-NEXT: fmla v16.2s, v1.2s, v6.2s
+; CHECK-NEXT: fmla v16.2s, v1.2s, v4.2s
; CHECK-NEXT: st2 { v16.2s, v17.2s }, [x1]
; CHECK-NEXT: ret
entry:
@@ -162,25 +157,24 @@ define <4 x float> @multiple_muls_shuffle_external_with_loads(ptr %ptr_a, ptr %p
; CHECK-NEXT: ld2 { v0.2s, v1.2s }, [x0]
; CHECK-NEXT: ld2 { v2.2s, v3.2s }, [x1]
; CHECK-NEXT: fmul v4.2s, v3.2s, v1.2s
-; CHECK-NEXT: fmul v6.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmul v5.2s, v2.2s, v1.2s
; CHECK-NEXT: fneg v4.2s, v4.2s
-; CHECK-NEXT: fmla v6.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v3.2s
; CHECK-NEXT: fmla v4.2s, v0.2s, v2.2s
; CHECK-NEXT: str d4, [x4]
-; CHECK-NEXT: ldr q5, [x2]
-; CHECK-NEXT: ext v7.16b, v5.16b, v5.16b, #8
-; CHECK-NEXT: zip1 v0.2s, v5.2s, v7.2s
-; CHECK-NEXT: zip2 v1.2s, v5.2s, v7.2s
-; CHECK-NEXT: fmul v3.2s, v0.2s, v6.2s
-; CHECK-NEXT: fmul v6.2s, v1.2s, v6.2s
+; CHECK-NEXT: ldr q6, [x2]
+; CHECK-NEXT: uzp1 v0.4s, v6.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v6.4s, v0.4s
+; CHECK-NEXT: fmul v3.2s, v0.2s, v5.2s
+; CHECK-NEXT: fmul v5.2s, v1.2s, v5.2s
; CHECK-NEXT: fmla v3.2s, v4.2s, v1.2s
-; CHECK-NEXT: fneg v2.2s, v6.2s
+; CHECK-NEXT: fneg v2.2s, v5.2s
; CHECK-NEXT: fmla v2.2s, v4.2s, v0.2s
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: st2 { v2.2s, v3.2s }, [x5]
; CHECK-NEXT: ldr q1, [x3]
-; CHECK-NEXT: fcmla v0.4s, v5.4s, v1.4s, #0
-; CHECK-NEXT: fcmla v0.4s, v5.4s, v1.4s, #90
+; CHECK-NEXT: fcmla v0.4s, v6.4s, v1.4s, #0
+; CHECK-NEXT: fcmla v0.4s, v6.4s, v1.4s, #90
; CHECK-NEXT: ret
entry:
%a = load <4 x float>, ptr %ptr_a
@@ -227,36 +221,32 @@ entry:
define <4 x float> @multiple_muls_mul_external(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
; CHECK-LABEL: multiple_muls_mul_external:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v5.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: ext v16.16b, v2.16b, v2.16b, #8
-; CHECK-NEXT: ext v17.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT: zip2 v6.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip2 v7.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v19.2s, v2.2s, v16.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v16.2s
-; CHECK-NEXT: zip2 v16.2s, v3.2s, v17.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v3.2s, v3.2s, v17.2s
-; CHECK-NEXT: fmul v18.2s, v6.2s, v7.2s
-; CHECK-NEXT: fmul v5.2s, v19.2s, v16.2s
+; CHECK-NEXT: uzp2 v4.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v5.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v7.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: uzp2 v16.4s, v3.4s, v0.4s
+; CHECK-NEXT: uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: fmul v6.2s, v4.2s, v5.2s
+; CHECK-NEXT: uzp1 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v3.4s, v0.4s
+; CHECK-NEXT: fmul v17.2s, v7.2s, v16.2s
; CHECK-NEXT: fmul v16.2s, v2.2s, v16.2s
-; CHECK-NEXT: fmul v7.2s, v0.2s, v7.2s
-; CHECK-NEXT: fneg v4.2s, v18.2s
-; CHECK-NEXT: fmla v5.2s, v3.2s, v2.2s
+; CHECK-NEXT: fmul v5.2s, v0.2s, v5.2s
+; CHECK-NEXT: fneg v6.2s, v6.2s
+; CHECK-NEXT: fmla v17.2s, v3.2s, v2.2s
; CHECK-NEXT: fneg v2.2s, v16.2s
-; CHECK-NEXT: fmla v7.2s, v1.2s, v6.2s
-; CHECK-NEXT: fmla v4.2s, v1.2s, v0.2s
-; CHECK-NEXT: fmla v2.2s, v3.2s, v19.2s
-; CHECK-NEXT: fmul v0.2s, v7.2s, v5.2s
-; CHECK-NEXT: fmul v17.2s, v4.2s, v5.2s
-; CHECK-NEXT: str d4, [x0]
-; CHECK-NEXT: fmla v17.2s, v2.2s, v7.2s
-; CHECK-NEXT: fneg v16.2s, v0.2s
-; CHECK-NEXT: zip1 v0.4s, v2.4s, v5.4s
-; CHECK-NEXT: fmla v16.2s, v2.2s, v4.2s
-; CHECK-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-NEXT: fmla v5.2s, v1.2s, v4.2s
+; CHECK-NEXT: fmla v6.2s, v1.2s, v0.2s
+; CHECK-NEXT: fmla v2.2s, v3.2s, v7.2s
+; CHECK-NEXT: fmul v0.2s, v5.2s, v17.2s
+; CHECK-NEXT: fmul v4.2s, v6.2s, v17.2s
+; CHECK-NEXT: str d6, [x0]
+; CHECK-NEXT: fmla v4.2s, v2.2s, v5.2s
+; CHECK-NEXT: fneg v3.2s, v0.2s
+; CHECK-NEXT: zip1 v0.4s, v2.4s, v17.4s
+; CHECK-NEXT: fmla v3.2s, v2.2s, v6.2s
+; CHECK-NEXT: st2 { v3.2s, v4.2s }, [x1]
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
index 13434fabefa78..6ec1517cc1177 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
@@ -31,18 +31,16 @@ entry:
define <4 x float> @simple_mul_no_contract(<4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: simple_mul_no_contract:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v2.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmul v4.2s, v2.2s, v4.2s
+; CHECK-NEXT: uzp1 v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fmul v4.2s, v1.2s, v2.2s
+; CHECK-NEXT: fmul v2.2s, v3.2s, v2.2s
; CHECK-NEXT: fmul v1.2s, v0.2s, v1.2s
-; CHECK-NEXT: fmla v3.2s, v0.2s, v2.2s
-; CHECK-NEXT: fsub v0.2s, v4.2s, v1.2s
-; CHECK-NEXT: zip1 v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: fmla v4.2s, v0.2s, v3.2s
+; CHECK-NEXT: fsub v0.2s, v2.2s, v1.2s
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v4.4s
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -149,14 +147,12 @@ entry:
define <4 x float> @add_external_use(<4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: add_external_use:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT: zip1 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v2.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fadd v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: fsub v1.2s, v4.2s, v1.2s
+; CHECK-NEXT: uzp1 v2.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: uzp1 v3.4s, v1.4s, v0.4s
+; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: fsub v1.2s, v2.2s, v1.2s
+; CHECK-NEXT: fadd v0.2s, v0.2s, v3.2s
; CHECK-NEXT: zip1 v0.4s, v1.4s, v0.4s
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/concat-vector.ll b/llvm/test/CodeGen/AArch64/concat-vector.ll
index e6e863b074592..4d105215a82c3 100644
--- a/llvm/test/CodeGen/AArch64/concat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/concat-vector.ll
@@ -351,8 +351,8 @@ entry:
define <8 x i16> @concat_low_high_v8i16(<8 x i16> %a_vec, <8 x i16> %b_vec) {
; CHECK-SD-LABEL: concat_low_high_v8i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT: dup v1.2d, v1.d[1]
+; CHECK-SD-NEXT: zip1 v0.2d, v0.2d, v1.2d
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: concat_low_high_v8i16:
@@ -368,11 +368,17 @@ entry:
}
define <8 x i16> @concat_high_high_v8i16(<8 x i16> %a_vec, <8 x i16> %b_vec) {
-; CHECK-LABEL: concat_high_high_v8i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov v1.d[0], v0.d[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: concat_high_high_v8i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: dup v1.2d, v1.d[1]
+; CHECK-SD-NEXT: ext v0.16b, v0.16b, v1.16b, #8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: concat_high_high_v8i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov v1.d[0], v0.d[1]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
entry:
%shuffle.i3 = shufflevector <8 x i16> %a_vec, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%shuffle.i = shufflevector <8 x i16> %b_vec, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -381,11 +387,17 @@ entry:
}
define <8 x half> @concat_high_high_v8f16(<8 x half> %a_vec, <8 x half> %b_vec) {
-; CHECK-LABEL: concat_high_high_v8f16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov v1.d[0], v0.d[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: concat_high_high_v8f16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: dup v1.2d, v1.d[1]
+; CHECK-SD-NEXT: ext v0.16b, v0.16b, v1.16b, #8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: concat_high_high_v8f16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov v1.d[0], v0.d[1]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
entry:
%shuffle.i3 = shufflevector <8 x half> %a_vec, <8 x half> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%shuffle.i = shufflevector <8 x half> %b_vec, <8 x half> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -394,11 +406,17 @@ entry:
}
define <8 x bfloat> @concat_high_high_v8bf16(<8 x bfloat> %a_vec, <8 x bfloat> %b_vec) {
-; CHECK-LABEL: concat_high_high_v8bf16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov v1.d[0], v0.d[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: concat_high_high_v8bf16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: dup v1.2d, v1.d[1]
+; CHECK-SD-NEXT: ext v0.16b, v0.16b, v1.16b, #8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: concat_high_high_v8bf16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov v1.d[0], v0.d[1]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
entry:
%shuffle.i3 = shufflevector <8 x bfloat> %a_vec, <8 x bfloat> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%shuffle.i = shufflevector <8 x bfloat> %b_vec, <8 x bfloat> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -443,11 +461,17 @@ entry:
}
define <16 x i8> @concat_high_high_v16i8(<16 x i8> %a_vec, <16 x i8> %b_vec) {
-; CHECK-LABEL: concat_high_high_v16i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov v1.d[0], v0.d[1]
-; CHECK-NEXT: mov v0.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: concat_high_high_v16i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: dup v1.2d, v1.d[1]
+; CHECK-SD-NEXT: ext v0.16b, v0.16b, v1.16b, #8
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: concat_high_high_v16i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov v1.d[0], v0.d[1]
+; CHECK-GI-NEXT: mov v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
entry:
%shuffle.i3 = shufflevector <16 x i8> %a_vec, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%shuffle.i = shufflevector <16 x i8> %b_vec, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -458,7 +482,9 @@ entry:
define <2 x i64> @concat_high_high_v2i64(<2 x i64> %a_vec, <2 x i64> %b_vec) {
; CHECK-SD-LABEL: concat_high_high_v2i64:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: dup v0.2d, v0.d[1]
+; CHECK-SD-NEXT: dup v1.2d, v1.d[1]
+; CHECK-SD-NEXT: zip1 v0.2d, v0.2d, v1.2d
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: concat_high_high_v2i64:
@@ -476,7 +502,9 @@ entry:
define <2 x double> @concat_high_high_v2f64(<2 x double> %a_vec, <2 x double> %b_vec) {
; CHECK-SD-LABEL: concat_high_high_v2f64:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: zip2 v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: dup v0.2d, v0.d[1]
+; CHECK-SD-NEXT: dup v1.2d, v1.d[1]
+; CHECK-SD-NEXT: zip1 v0.2d, v0.2d, v1.2d
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: concat_high_high_v2f64:
diff --git a/llvm/test/CodeGen/AArch64/highextractbitcast.ll b/llvm/test/CodeGen/AArch64/highextractbitcast.ll
index de2911b8a5127..851b7cdd13730 100644
--- a/llvm/test/CodeGen/AArch64/highextractbitcast.ll
+++ b/llvm/test/CodeGen/AArch64/highextractbitcast.ll
@@ -140,9 +140,8 @@ entry:
define <4 x i32> @test_smull_high_s16_bitcasta1_wrongindex(<2 x i64> %aa, <8 x i16> %b) #0 {
; CHECK-LE-LABEL: test_smull_high_s16_bitcasta1_wrongindex:
; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-LE-NEXT: ext v0.16b, v0.16b, v0.16b, #4
; CHECK-LE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-LE-NEXT: ext v0.8b, v0.8b, v2.8b, #4
; CHECK-LE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-LE-NEXT: ret
;
@@ -176,9 +175,8 @@ entry:
define <4 x i32> @test_smull_high_s16_bitcastb1_wrongindex(<8 x i16> %a, <16 x i8> %bb) #0 {
; CHECK-LE-LABEL: test_smull_high_s16_bitcastb1_wrongindex:
; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: ext v2.16b, v1.16b, v1.16b, #8
; CHECK-LE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-LE-NEXT: ext v1.8b, v1.8b, v2.8b, #6
+; CHECK-LE-NEXT: ext v1.16b, v1.16b, v0.16b, #6
; CHECK-LE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-LE-NEXT: ret
;
@@ -189,7 +187,7 @@ define <4 x i32> @test_smull_high_s16_bitcastb1_wrongindex(<8 x i16> %a, <16 x i
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #6
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v0.16b, #6
; CHECK-BE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
@@ -249,7 +247,7 @@ define <4 x i32> @test_smull_high_s16_bitcastb2_wrongindex(<8 x i16> %a, <16 x i
; CHECK-LE-LABEL: test_smull_high_s16_bitcastb2_wrongindex:
; CHECK-LE: // %bb.0: // %entry
; CHECK-LE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-LE-NEXT: ext v1.16b, v1.16b, v1.16b, #4
+; CHECK-LE-NEXT: ext v1.16b, v1.16b, v0.16b, #4
; CHECK-LE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-LE-NEXT: ret
;
@@ -259,7 +257,7 @@ define <4 x i32> @test_smull_high_s16_bitcastb2_wrongindex(<8 x i16> %a, <16 x i
; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #4
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v0.16b, #4
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: rev16 v1.8b, v1.8b
; CHECK-BE-NEXT: smull v0.4s, v0.4h, v1.4h
@@ -345,8 +343,8 @@ entry:
define <4 x i32> @test_smull_high_s16_splata2(<4 x i32> %a, <8 x i16> %b) #0 {
; CHECK-LE-LABEL: test_smull_high_s16_splata2:
; CHECK-LE: // %bb.0: // %entry
-; CHECK-LE-NEXT: dup v0.2s, v0.s[3]
; CHECK-LE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-LE-NEXT: dup v0.4s, v0.s[3]
; CHECK-LE-NEXT: smull v0.4s, v0.4h, v1.4h
; CHECK-LE-NEXT: ret
;
@@ -356,7 +354,7 @@ define <4 x i32> @test_smull_high_s16_splata2(<4 x i32> %a, <8 x i16> %b) #0 {
; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-BE-NEXT: dup v0.2s, v0.s[3]
+; CHECK-BE-NEXT: dup v0.4s, v0.s[3]
; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-BE-NEXT: rev32 v0.4h, v0.4h
; CHECK-BE-NEXT: smull v0.4s, v0.4h, v1.4h
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
index 6b082b1762cc8..dad67750f4e1d 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
@@ -67,10 +67,11 @@ entry:
define void @extract_v32i8_half_unaligned(ptr %in, ptr %out) #0 vscale_range(2,2) {
; CHECK-LABEL: extract_v32i8_half_unaligned:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: adrp x8, .LCPI4_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI4_0
; CHECK-NEXT: ldr z0, [x0]
-; CHECK-NEXT: movprfx z1, z0
-; CHECK-NEXT: ext z1.b, z1.b, z0.b, #16
-; CHECK-NEXT: ext v0.16b, v0.16b, v1.16b, #4
+; CHECK-NEXT: ldr z1, [x8]
+; CHECK-NEXT: tbl z0.s, { z0.s }, z1.s
; CHECK-NEXT: str q0, [x1]
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/v8.2a-neon-intrinsics-constrained.ll b/llvm/test/CodeGen/AArch64/v8.2a-neon-intrinsics-constrained.ll
index 58f1accdf91d5..79077b43794e5 100644
--- a/llvm/test/CodeGen/AArch64/v8.2a-neon-intrinsics-constrained.ll
+++ b/llvm/test/CodeGen/AArch64/v8.2a-neon-intrinsics-constrained.ll
@@ -91,7 +91,8 @@ entry:
define <4 x half> @test_vfma_laneq_f16(<4 x half> %a, <4 x half> %b, <8 x half> %c) #0 {
; CHECK-LABEL: test_vfma_laneq_f16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fmla v0.4h, v1.4h, v2.h[7]
+; CHECK-NEXT: dup v2.8h, v2.h[7]
+; CHECK-NEXT: fmla v0.4h, v1.4h, v2.4h
; CHECK-NEXT: ret
entry:
%lane = shufflevector <8 x half> %c, <8 x half> poison, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
@@ -188,7 +189,8 @@ entry:
define <4 x half> @test_vfms_laneq_f16(<4 x half> %a, <4 x half> %b, <8 x half> %c) #0 {
; CHECK-LABEL: test_vfms_laneq_f16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fmls v0.4h, v1.4h, v2.h[7]
+; CHECK-NEXT: dup v2.8h, v2.h[7]
+; CHECK-NEXT: fmls v0.4h, v2.4h, v1.4h
; CHECK-NEXT: ret
entry:
%fneg = fneg <4 x half> %b
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index 021b15ef09bee..8d18c6dbd9339 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2872,34 +2872,34 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: s_mov_b32 s4, 0xc0c0004
-; SDAG-NEXT: v_perm_b32 v8, v8, v9, s4
-; SDAG-NEXT: v_perm_b32 v9, v10, v11, s4
-; SDAG-NEXT: buffer_load_ubyte v10, off, s[0:3], s32
+; SDAG-NEXT: v_perm_b32 v20, v20, v21, s4
+; SDAG-NEXT: v_perm_b32 v21, v22, v23, s4
+; SDAG-NEXT: buffer_load_ubyte v22, off, s[0:3], s32
; SDAG-NEXT: v_perm_b32 v12, v12, v13, s4
; SDAG-NEXT: v_perm_b32 v13, v14, v15, s4
+; SDAG-NEXT: v_perm_b32 v8, v8, v9, s4
+; SDAG-NEXT: v_perm_b32 v9, v10, v11, s4
; SDAG-NEXT: v_perm_b32 v4, v4, v5, s4
; SDAG-NEXT: v_perm_b32 v5, v6, v7, s4
-; SDAG-NEXT: v_perm_b32 v0, v0, v1, s4
-; SDAG-NEXT: v_perm_b32 v6, v2, v3, s4
-; SDAG-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; SDAG-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; SDAG-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; SDAG-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; SDAG-NEXT: v_perm_b32 v7, v28, v29, s4
-; SDAG-NEXT: v_perm_b32 v11, v24, v25, s4
-; SDAG-NEXT: v_perm_b32 v14, v26, v27, s4
-; SDAG-NEXT: v_perm_b32 v15, v20, v21, s4
-; SDAG-NEXT: v_perm_b32 v20, v22, v23, s4
+; SDAG-NEXT: v_perm_b32 v10, v0, v1, s4
+; SDAG-NEXT: v_perm_b32 v3, v2, v3, s4
+; SDAG-NEXT: v_perm_b32 v24, v24, v25, s4
+; SDAG-NEXT: v_perm_b32 v25, v26, v27, s4
+; SDAG-NEXT: v_perm_b32 v23, v28, v29, s4
; SDAG-NEXT: v_perm_b32 v16, v16, v17, s4
; SDAG-NEXT: v_perm_b32 v17, v18, v19, s4
-; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; SDAG-NEXT: v_lshl_or_b32 v5, v14, 16, v11
-; SDAG-NEXT: v_lshl_or_b32 v4, v20, 16, v15
-; SDAG-NEXT: v_lshl_or_b32 v3, v17, 16, v16
-; SDAG-NEXT: s_waitcnt vmcnt(1)
-; SDAG-NEXT: v_perm_b32 v0, v30, v10, s4
-; SDAG-NEXT: v_lshl_or_b32 v6, v0, 16, v7
-; SDAG-NEXT: buffer_store_dwordx4 v[3:6], off, s[16:19], 0 offset:16
+; SDAG-NEXT: v_lshl_or_b32 v7, v13, 16, v12
+; SDAG-NEXT: v_lshl_or_b32 v6, v9, 16, v8
+; SDAG-NEXT: v_lshl_or_b32 v5, v5, 16, v4
+; SDAG-NEXT: v_lshl_or_b32 v4, v3, 16, v10
+; SDAG-NEXT: v_lshl_or_b32 v2, v25, 16, v24
+; SDAG-NEXT: v_lshl_or_b32 v1, v21, 16, v20
+; SDAG-NEXT: v_lshl_or_b32 v0, v17, 16, v16
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_perm_b32 v3, v30, v22, s4
+; SDAG-NEXT: v_lshl_or_b32 v3, v3, 16, v23
+; SDAG-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0
+; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
index 5aa2cc713a4ec..d1db68f90dcce 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -769,37 +769,45 @@ define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class_subreg(
define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class_subreg_odd(ptr addrspace(1) %arg) #0 {
; CHECK-LABEL: test_rewrite_mfma_direct_copy_from_agpr_class_subreg_odd:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_mov_b32_e32 v1, 2.0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def a[0:31]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: v_mov_b32_e32 v18, 4.0
-; CHECK-NEXT: v_accvgpr_mov_b32 a0, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a1, a2
-; CHECK-NEXT: v_accvgpr_mov_b32 a2, a3
-; CHECK-NEXT: v_accvgpr_mov_b32 a3, a4
-; CHECK-NEXT: v_accvgpr_mov_b32 a4, a5
-; CHECK-NEXT: v_accvgpr_mov_b32 a5, a6
-; CHECK-NEXT: v_accvgpr_mov_b32 a6, a7
-; CHECK-NEXT: v_accvgpr_mov_b32 a7, a8
-; CHECK-NEXT: v_accvgpr_mov_b32 a8, a9
-; CHECK-NEXT: v_accvgpr_mov_b32 a9, a10
-; CHECK-NEXT: v_accvgpr_mov_b32 a10, a11
-; CHECK-NEXT: v_accvgpr_mov_b32 a11, a12
-; CHECK-NEXT: v_accvgpr_mov_b32 a12, a13
-; CHECK-NEXT: v_accvgpr_mov_b32 a13, a14
-; CHECK-NEXT: v_accvgpr_mov_b32 a14, a15
-; CHECK-NEXT: v_accvgpr_mov_b32 a15, a16
+; CHECK-NEXT: v_mov_b32_e32 v20, 2.0
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a15
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a14
+; CHECK-NEXT: v_pk_mov_b32 v[16:17], v[4:5], v[2:3] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a13
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a12
+; CHECK-NEXT: v_pk_mov_b32 v[14:15], v[2:3], v[4:5] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a11
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a10
+; CHECK-NEXT: v_pk_mov_b32 v[12:13], v[4:5], v[2:3] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a9
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a8
+; CHECK-NEXT: v_pk_mov_b32 v[10:11], v[2:3], v[4:5] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a7
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a6
+; CHECK-NEXT: v_pk_mov_b32 v[8:9], v[4:5], v[2:3] op_sel:[1,0]
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a4
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a5
+; CHECK-NEXT: v_accvgpr_read_b32 v18, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v19, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
+; CHECK-NEXT: v_pk_mov_b32 v[6:7], v[2:3], v[4:5] op_sel:[1,0]
+; CHECK-NEXT: v_pk_mov_b32 v[4:5], v[18:19], v[2:3] op_sel:[1,0]
+; CHECK-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[18:19] op_sel:[1,0]
+; CHECK-NEXT: v_mov_b32_e32 v1, 4.0
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT: v_mfma_f32_16x16x1_4b_f32 a[0:15], v1, v18, a[0:15]
+; CHECK-NEXT: v_mfma_f32_16x16x1_4b_f32 v[2:17], v20, v1, v[2:17]
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 6, v0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_nop 7
-; CHECK-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
-; CHECK-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
-; CHECK-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
-; CHECK-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]
+; CHECK-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1] offset:48
+; CHECK-NEXT: global_store_dwordx4 v0, v[10:13], s[0:1] offset:32
+; CHECK-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
+; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; CHECK-NEXT: s_endpgm
%def = call <32 x float> asm sideeffect "; def $0", "=a"()
%src2 = shufflevector <32 x float> %def, <32 x float> poison, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll
index 441509ba01f64..d2427a8a5c9b1 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll
@@ -546,44 +546,44 @@ define amdgpu_kernel void @sdiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: v_readfirstlane_b32 s0, v2
; GFX9-NEXT: s_abs_i32 s1, s0
; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s1
-; GFX9-NEXT: v_readfirstlane_b32 s5, v0
-; GFX9-NEXT: s_xor_b32 s0, s5, s0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: s_xor_b32 s0, s4, s0
; GFX9-NEXT: s_ashr_i32 s6, s0, 31
; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v2
; GFX9-NEXT: s_sub_i32 s0, 0, s1
-; GFX9-NEXT: s_abs_i32 s5, s5
-; GFX9-NEXT: v_readfirstlane_b32 s4, v3
+; GFX9-NEXT: s_abs_i32 s4, s4
+; GFX9-NEXT: v_readfirstlane_b32 s5, v3
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v2
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
; GFX9-NEXT: s_mul_i32 s0, s0, s7
; GFX9-NEXT: s_mul_hi_u32 s0, s7, s0
; GFX9-NEXT: s_add_i32 s7, s7, s0
-; GFX9-NEXT: s_mul_hi_u32 s0, s5, s7
+; GFX9-NEXT: s_mul_hi_u32 s0, s4, s7
; GFX9-NEXT: s_mul_i32 s7, s0, s1
-; GFX9-NEXT: s_sub_i32 s5, s5, s7
+; GFX9-NEXT: s_sub_i32 s4, s4, s7
; GFX9-NEXT: s_add_i32 s10, s0, 1
-; GFX9-NEXT: s_sub_i32 s7, s5, s1
-; GFX9-NEXT: s_cmp_ge_u32 s5, s1
+; GFX9-NEXT: s_sub_i32 s7, s4, s1
+; GFX9-NEXT: s_cmp_ge_u32 s4, s1
; GFX9-NEXT: s_cselect_b32 s0, s10, s0
-; GFX9-NEXT: s_cselect_b32 s5, s7, s5
+; GFX9-NEXT: s_cselect_b32 s4, s7, s4
; GFX9-NEXT: s_add_i32 s7, s0, 1
-; GFX9-NEXT: s_cmp_ge_u32 s5, s1
-; GFX9-NEXT: s_cselect_b32 s5, s7, s0
-; GFX9-NEXT: s_abs_i32 s7, s4
+; GFX9-NEXT: s_cmp_ge_u32 s4, s1
+; GFX9-NEXT: s_cselect_b32 s4, s7, s0
+; GFX9-NEXT: s_abs_i32 s7, s5
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_xor_b32 s5, s5, s6
+; GFX9-NEXT: s_xor_b32 s4, s4, s6
; GFX9-NEXT: s_mov_b32 s1, s9
; GFX9-NEXT: s_sub_i32 s9, 0, s7
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_sub_i32 s5, s5, s6
+; GFX9-NEXT: s_sub_i32 s4, s4, s6
; GFX9-NEXT: s_mov_b32 s0, s8
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_xor_b32 s4, s8, s4
+; GFX9-NEXT: s_xor_b32 s5, s8, s5
; GFX9-NEXT: s_abs_i32 s8, s8
-; GFX9-NEXT: s_ashr_i32 s4, s4, 31
+; GFX9-NEXT: s_ashr_i32 s5, s5, 31
; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_mul_i32 s9, s9, s6
; GFX9-NEXT: s_mul_hi_u32 s9, s6, s9
@@ -599,10 +599,10 @@ define amdgpu_kernel void @sdiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: s_add_i32 s9, s6, 1
; GFX9-NEXT: s_cmp_ge_u32 s8, s7
; GFX9-NEXT: s_cselect_b32 s6, s9, s6
-; GFX9-NEXT: s_xor_b32 s6, s6, s4
-; GFX9-NEXT: s_sub_i32 s4, s6, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: s_xor_b32 s6, s6, s5
+; GFX9-NEXT: s_sub_i32 s5, s6, s5
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
index f1d147947ccdf..f17edfa268423 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2f32.v4f32.ll
@@ -879,9 +879,8 @@ define void @v_shuffle_v2f32_v4f32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +891,8 @@ define void @v_shuffle_v2f32_v4f32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x float> asm "; def $0", "=v"()
@@ -1501,8 +1499,7 @@ define void @v_shuffle_v2f32_v4f32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1511,7 @@ define void @v_shuffle_v2f32_v4f32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
index 39c6a447788e4..f942c4c540fc9 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
@@ -879,9 +879,8 @@ define void @v_shuffle_v2i32_v4i32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +891,8 @@ define void @v_shuffle_v2i32_v4i32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1501,8 +1499,7 @@ define void @v_shuffle_v2i32_v4i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1511,7 @@ define void @v_shuffle_v2i32_v4i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
index c35361721e9b0..1f8745e242865 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll
@@ -879,9 +879,8 @@ define void @v_shuffle_v2p3_v4p3__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +891,8 @@ define void @v_shuffle_v2p3_v4p3__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x ptr addrspace(3)> asm "; def $0", "=v"()
@@ -1501,8 +1499,7 @@ define void @v_shuffle_v2p3_v4p3__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1511,7 @@ define void @v_shuffle_v2p3_v4p3__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll
index 4cf74a9ba96a8..3cd5645f9a69a 100644
--- a/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -294,24 +294,24 @@ define amdgpu_kernel void @ashr_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_mov_b32 s1, s5
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s4, v0
-; VI-NEXT: v_readfirstlane_b32 s5, v2
-; VI-NEXT: v_readfirstlane_b32 s6, v1
+; VI-NEXT: v_readfirstlane_b32 s5, v1
+; VI-NEXT: v_readfirstlane_b32 s6, v2
; VI-NEXT: v_readfirstlane_b32 s7, v3
; VI-NEXT: s_lshr_b32 s8, s7, 16
-; VI-NEXT: s_ashr_i32 s9, s6, 16
-; VI-NEXT: s_sext_i32_i16 s6, s6
-; VI-NEXT: s_lshr_b32 s10, s5, 16
+; VI-NEXT: s_ashr_i32 s9, s5, 16
+; VI-NEXT: s_sext_i32_i16 s5, s5
+; VI-NEXT: s_lshr_b32 s10, s6, 16
; VI-NEXT: s_ashr_i32 s11, s4, 16
; VI-NEXT: s_sext_i32_i16 s4, s4
; VI-NEXT: s_ashr_i32 s8, s9, s8
-; VI-NEXT: s_ashr_i32 s6, s6, s7
+; VI-NEXT: s_ashr_i32 s5, s5, s7
; VI-NEXT: s_ashr_i32 s7, s11, s10
-; VI-NEXT: s_ashr_i32 s4, s4, s5
-; VI-NEXT: s_lshl_b32 s5, s8, 16
-; VI-NEXT: s_and_b32 s6, s6, 0xffff
+; VI-NEXT: s_ashr_i32 s4, s4, s6
+; VI-NEXT: s_lshl_b32 s6, s8, 16
+; VI-NEXT: s_and_b32 s5, s5, 0xffff
; VI-NEXT: s_lshl_b32 s7, s7, 16
; VI-NEXT: s_and_b32 s4, s4, 0xffff
-; VI-NEXT: s_or_b32 s5, s6, s5
+; VI-NEXT: s_or_b32 s5, s5, s6
; VI-NEXT: s_or_b32 s4, s4, s7
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
diff --git a/llvm/test/CodeGen/AMDGPU/srem.ll b/llvm/test/CodeGen/AMDGPU/srem.ll
index 089c0b2d46578..5e32815777819 100644
--- a/llvm/test/CodeGen/AMDGPU/srem.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem.ll
@@ -467,28 +467,28 @@ define amdgpu_kernel void @srem_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GCN-NEXT: v_readfirstlane_b32 s2, v2
; GCN-NEXT: s_abs_i32 s2, s2
; GCN-NEXT: v_cvt_f32_u32_e32 v2, s2
-; GCN-NEXT: v_readfirstlane_b32 s4, v0
+; GCN-NEXT: v_readfirstlane_b32 s3, v0
; GCN-NEXT: s_sub_i32 s6, 0, s2
-; GCN-NEXT: s_ashr_i32 s5, s4, 31
+; GCN-NEXT: s_ashr_i32 s5, s3, 31
; GCN-NEXT: v_rcp_iflag_f32_e32 v2, v2
-; GCN-NEXT: s_abs_i32 s4, s4
-; GCN-NEXT: v_readfirstlane_b32 s3, v3
+; GCN-NEXT: s_abs_i32 s3, s3
+; GCN-NEXT: v_readfirstlane_b32 s4, v3
; GCN-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v2
; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0
; GCN-NEXT: v_readfirstlane_b32 s7, v0
; GCN-NEXT: s_mul_i32 s6, s6, s7
; GCN-NEXT: s_mul_hi_u32 s6, s7, s6
; GCN-NEXT: s_add_i32 s7, s7, s6
-; GCN-NEXT: s_mul_hi_u32 s6, s4, s7
+; GCN-NEXT: s_mul_hi_u32 s6, s3, s7
; GCN-NEXT: s_mul_i32 s6, s6, s2
-; GCN-NEXT: s_sub_i32 s4, s4, s6
-; GCN-NEXT: s_sub_i32 s6, s4, s2
-; GCN-NEXT: s_cmp_ge_u32 s4, s2
-; GCN-NEXT: s_cselect_b32 s4, s6, s4
-; GCN-NEXT: s_sub_i32 s6, s4, s2
-; GCN-NEXT: s_cmp_ge_u32 s4, s2
-; GCN-NEXT: s_cselect_b32 s2, s6, s4
-; GCN-NEXT: s_abs_i32 s3, s3
+; GCN-NEXT: s_sub_i32 s3, s3, s6
+; GCN-NEXT: s_sub_i32 s6, s3, s2
+; GCN-NEXT: s_cmp_ge_u32 s3, s2
+; GCN-NEXT: s_cselect_b32 s3, s6, s3
+; GCN-NEXT: s_sub_i32 s6, s3, s2
+; GCN-NEXT: s_cmp_ge_u32 s3, s2
+; GCN-NEXT: s_cselect_b32 s2, s6, s3
+; GCN-NEXT: s_abs_i32 s3, s4
; GCN-NEXT: v_cvt_f32_u32_e32 v0, s3
; GCN-NEXT: s_xor_b32 s2, s2, s5
; GCN-NEXT: s_sub_i32 s7, 0, s3
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index 6bf6d540299f1..4da64d50043d9 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -5198,12 +5198,13 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GX900-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GX900-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
; GX900-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GX900-NEXT: s_mov_b32 s3, 0x5040100
; GX900-NEXT: s_movk_i32 s2, 0x7fff
-; GX900-NEXT: s_mov_b32 s3, 0x7060302
; GX900-NEXT: s_waitcnt lgkmcnt(0)
; GX900-NEXT: global_load_dwordx2 v[1:2], v0, s[0:1]
; GX900-NEXT: global_load_dwordx2 v[3:4], v0, s[4:5]
; GX900-NEXT: global_load_dwordx2 v[5:6], v0, s[6:7]
+; GX900-NEXT: s_mov_b32 s4, 0x7060302
; GX900-NEXT: s_waitcnt vmcnt(2)
; GX900-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
; GX900-NEXT: s_waitcnt vmcnt(1)
@@ -5212,67 +5213,71 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GX900-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GX900-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GX900-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GX900-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GX900-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GX900-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GX900-NEXT: v_perm_b32 v11, v4, v4, s3
; GX900-NEXT: v_fma_f32 v7, v8, v9, v7
+; GX900-NEXT: v_and_b32_e32 v12, 0xffff0000, v2
+; GX900-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GX900-NEXT: v_fma_f32 v1, v8, v5, v1
-; GX900-NEXT: v_fma_f32 v2, v12, v5, v2
-; GX900-NEXT: v_bfe_u32 v5, v7, 16, 1
-; GX900-NEXT: v_fma_f32 v8, v12, v9, v11
-; GX900-NEXT: v_or_b32_e32 v9, 0x400000, v7
-; GX900-NEXT: v_bfe_u32 v11, v1, 16, 1
-; GX900-NEXT: v_add3_u32 v5, v5, v7, s2
+; GX900-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
+; GX900-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GX900-NEXT: v_bfe_u32 v14, v7, 16, 1
+; GX900-NEXT: v_or_b32_e32 v15, 0x400000, v7
+; GX900-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GX900-NEXT: v_fma_f32 v8, v8, v9, v12
+; GX900-NEXT: v_fma_f32 v2, v11, v5, v2
+; GX900-NEXT: v_add3_u32 v5, v14, v7, s2
; GX900-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GX900-NEXT: v_bfe_u32 v13, v8, 16, 1
-; GX900-NEXT: v_add3_u32 v11, v11, v1, s2
-; GX900-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc
+; GX900-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GX900-NEXT: v_add3_u32 v9, v16, v1, s2
+; GX900-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GX900-NEXT: v_cndmask_b32_e32 v5, v5, v15, vcc
; GX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GX900-NEXT: v_or_b32_e32 v14, 0x400000, v8
-; GX900-NEXT: v_bfe_u32 v15, v2, 16, 1
-; GX900-NEXT: v_add3_u32 v13, v13, v8, s2
-; GX900-NEXT: v_cndmask_b32_e32 v1, v11, v12, vcc
+; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GX900-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GX900-NEXT: v_cndmask_b32_e32 v1, v9, v17, vcc
+; GX900-NEXT: v_add3_u32 v7, v11, v8, s2
; GX900-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
; GX900-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GX900-NEXT: v_lshlrev_b32_e32 v10, 16, v6
; GX900-NEXT: v_or_b32_e32 v16, 0x400000, v2
-; GX900-NEXT: v_add3_u32 v15, v15, v2, s2
-; GX900-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc
-; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GX900-NEXT: v_add3_u32 v9, v14, v2, s2
; GX900-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GX900-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc
+; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GX900-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GX900-NEXT: v_cndmask_b32_e32 v2, v15, v16, vcc
+; GX900-NEXT: v_perm_b32 v4, v4, v4, s4
; GX900-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; GX900-NEXT: v_cndmask_b32_e32 v2, v9, v16, vcc
; GX900-NEXT: v_fma_f32 v1, v3, v10, v1
+; GX900-NEXT: v_lshlrev_b32_e32 v13, 16, v4
; GX900-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GX900-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GX900-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GX900-NEXT: v_fma_f32 v3, v3, v6, v5
-; GX900-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GX900-NEXT: v_fma_f32 v2, v4, v10, v2
-; GX900-NEXT: v_fma_f32 v4, v4, v6, v7
-; GX900-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GX900-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GX900-NEXT: v_add3_u32 v5, v5, v1, s2
+; GX900-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GX900-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
+; GX900-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GX900-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GX900-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GX900-NEXT: v_fma_f32 v2, v13, v10, v2
+; GX900-NEXT: v_fma_f32 v4, v4, v6, v5
+; GX900-NEXT: v_add3_u32 v5, v7, v1, s2
; GX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GX900-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GX900-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GX900-NEXT: v_add3_u32 v7, v7, v3, s2
-; GX900-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc
+; GX900-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GX900-NEXT: v_add3_u32 v6, v9, v3, s2
+; GX900-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GX900-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc
; GX900-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GX900-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GX900-NEXT: v_bfe_u32 v11, v4, 16, 1
-; GX900-NEXT: v_add3_u32 v9, v9, v2, s2
-; GX900-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
+; GX900-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GX900-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GX900-NEXT: v_cndmask_b32_e32 v3, v6, v11, vcc
+; GX900-NEXT: v_add3_u32 v5, v7, v2, s2
; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GX900-NEXT: v_add3_u32 v11, v11, v4, s2
-; GX900-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GX900-NEXT: v_add3_u32 v6, v10, v4, s2
+; GX900-NEXT: v_cndmask_b32_e32 v2, v5, v9, vcc
; GX900-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GX900-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc
-; GX900-NEXT: v_perm_b32 v2, v4, v2, s3
-; GX900-NEXT: v_perm_b32 v1, v3, v1, s3
+; GX900-NEXT: v_cndmask_b32_e32 v4, v6, v12, vcc
+; GX900-NEXT: v_perm_b32 v1, v3, v1, s4
+; GX900-NEXT: v_perm_b32 v2, v4, v2, s4
; GX900-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1]
; GX900-NEXT: s_endpgm
;
@@ -5282,12 +5287,13 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v6, 3, v0
+; GFX942-NEXT: s_mov_b32 s3, 0x5040100
; GFX942-NEXT: s_movk_i32 s2, 0x7fff
-; GFX942-NEXT: s_mov_b32 s3, 0x7060302
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dwordx2 v[0:1], v6, s[8:9]
; GFX942-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
; GFX942-NEXT: global_load_dwordx2 v[4:5], v6, s[10:11]
+; GFX942-NEXT: s_mov_b32 s4, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(2)
; GFX942-NEXT: v_lshlrev_b32_e32 v7, 16, v0
; GFX942-NEXT: s_waitcnt vmcnt(1)
@@ -5296,68 +5302,72 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX942-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX942-NEXT: v_perm_b32 v11, v1, v1, s3
+; GFX942-NEXT: v_fmac_f32_e32 v8, v7, v9
; GFX942-NEXT: v_and_b32_e32 v12, 0xffff0000, v3
; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX942-NEXT: v_fmac_f32_e32 v8, v7, v9
; GFX942-NEXT: v_fmac_f32_e32 v2, v7, v4
+; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v11
+; GFX942-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX942-NEXT: v_bfe_u32 v14, v8, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v15, 0x400000, v8
+; GFX942-NEXT: v_bfe_u32 v16, v2, 16, 1
+; GFX942-NEXT: v_fmac_f32_e32 v12, v7, v9
; GFX942-NEXT: v_fmac_f32_e32 v3, v11, v4
-; GFX942-NEXT: v_bfe_u32 v4, v8, 16, 1
-; GFX942-NEXT: v_fmac_f32_e32 v12, v11, v9
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v8
-; GFX942-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX942-NEXT: v_add3_u32 v4, v4, v8, s2
+; GFX942-NEXT: v_add3_u32 v4, v14, v8, s2
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v8, v8
-; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v2
-; GFX942-NEXT: v_bfe_u32 v13, v12, 16, 1
-; GFX942-NEXT: v_add3_u32 v9, v9, v2, s2
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc
+; GFX942-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX942-NEXT: v_add3_u32 v7, v16, v2, s2
+; GFX942-NEXT: v_bfe_u32 v9, v12, 16, 1
+; GFX942-NEXT: v_cndmask_b32_e32 v4, v4, v15, vcc
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_or_b32_e32 v14, 0x400000, v12
-; GFX942-NEXT: v_bfe_u32 v15, v3, 16, 1
-; GFX942-NEXT: v_add3_u32 v13, v13, v12, s2
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc
+; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v12
+; GFX942-NEXT: v_bfe_u32 v14, v3, 16, 1
+; GFX942-NEXT: v_cndmask_b32_e32 v2, v7, v17, vcc
+; GFX942-NEXT: v_add3_u32 v7, v9, v12, s2
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v12, v12
; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX942-NEXT: v_or_b32_e32 v16, 0x400000, v3
-; GFX942-NEXT: v_add3_u32 v15, v15, v3, s2
-; GFX942-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX942-NEXT: v_add3_u32 v8, v14, v3, s2
; GFX942-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX942-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX942-NEXT: v_cndmask_b32_e32 v3, v15, v16, vcc
+; GFX942-NEXT: v_perm_b32 v1, v1, v1, s4
; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX942-NEXT: v_cndmask_b32_e32 v3, v8, v16, vcc
; GFX942-NEXT: v_fmac_f32_e32 v2, v0, v10
+; GFX942-NEXT: v_lshlrev_b32_e32 v13, 16, v1
; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; GFX942-NEXT: v_fmac_f32_e32 v4, v0, v5
-; GFX942-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX942-NEXT: v_fmac_f32_e32 v3, v1, v10
-; GFX942-NEXT: v_fmac_f32_e32 v7, v1, v5
-; GFX942-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX942-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX942-NEXT: v_add3_u32 v0, v0, v2, s2
+; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v7
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX942-NEXT: v_fmac_f32_e32 v0, v13, v10
+; GFX942-NEXT: v_fmac_f32_e32 v3, v1, v5
+; GFX942-NEXT: v_add3_u32 v1, v7, v2, s2
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX942-NEXT: v_add3_u32 v5, v5, v4, s2
-; GFX942-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX942-NEXT: v_add3_u32 v5, v9, v4, s2
+; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX942-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX942-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX942-NEXT: v_add3_u32 v9, v9, v3, s2
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX942-NEXT: v_bfe_u32 v10, v3, 16, 1
+; GFX942-NEXT: v_cndmask_b32_e32 v2, v5, v11, vcc
+; GFX942-NEXT: v_add3_u32 v4, v7, v0, s2
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX942-NEXT: v_or_b32_e32 v12, 0x400000, v3
+; GFX942-NEXT: v_add3_u32 v5, v10, v3, s2
+; GFX942-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX942-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX942-NEXT: v_add3_u32 v11, v11, v7, s2
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX942-NEXT: v_perm_b32 v0, v2, v0, s3
+; GFX942-NEXT: v_perm_b32 v0, v2, v1, s4
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v3, v11, v12, vcc
-; GFX942-NEXT: v_perm_b32 v1, v3, v1, s3
+; GFX942-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc
+; GFX942-NEXT: v_perm_b32 v1, v3, v4, s4
; GFX942-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1]
; GFX942-NEXT: s_endpgm
;
@@ -5380,67 +5390,71 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX10-NEXT: v_perm_b32 v11, v3, v3, 0x5040100
+; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX10-NEXT: v_fmac_f32_e32 v7, v8, v9
; GFX10-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
+; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT: v_fmac_f32_e32 v11, v12, v9
-; GFX10-NEXT: v_fmac_f32_e32 v1, v12, v4
-; GFX10-NEXT: v_bfe_u32 v4, v7, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v7
-; GFX10-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX10-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
-; GFX10-NEXT: v_bfe_u32 v15, v1, 16, 1
-; GFX10-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX10-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v16, 0x400000, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX10-NEXT: v_bfe_u32 v14, v7, 16, 1
+; GFX10-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX10-NEXT: v_fmac_f32_e32 v12, v8, v9
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_fmac_f32_e32 v1, v11, v4
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v9, v16, v0, 0x7fff
+; GFX10-NEXT: v_or_b32_e32 v15, 0x400000, v7
+; GFX10-NEXT: v_add3_u32 v4, v14, v7, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX10-NEXT: v_bfe_u32 v11, v12, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX10-NEXT: v_add3_u32 v8, v16, v1, 0x7fff
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX10-NEXT: v_or_b32_e32 v14, 0x400000, v11
-; GFX10-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX10-NEXT: v_or_b32_e32 v14, 0x400000, v12
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v15, vcc_lo
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_add3_u32 v9, v11, v12, 0x7fff
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_perm_b32 v3, v3, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v8, v7, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX10-NEXT: v_fmac_f32_e32 v4, v2, v5
; GFX10-NEXT: v_fmac_f32_e32 v0, v2, v10
+; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX10-NEXT: v_cndmask_b32_e32 v7, v9, v14, vcc_lo
+; GFX10-NEXT: v_fmac_f32_e32 v4, v2, v5
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX10-NEXT: v_fmac_f32_e32 v1, v3, v10
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX10-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX10-NEXT: v_fmac_f32_e32 v7, v3, v5
-; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX10-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
-; GFX10-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX10-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX10-NEXT: v_fmac_f32_e32 v1, v13, v10
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX10-NEXT: v_fmac_f32_e32 v2, v3, v5
+; GFX10-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v5, v9, v4, 0x7fff
+; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX10-NEXT: v_bfe_u32 v11, v2, 16, 1
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT: v_add3_u32 v3, v7, v1, 0x7fff
+; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX10-NEXT: v_add3_u32 v7, v11, v2, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX10-NEXT: v_perm_b32 v0, v4, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v12, vcc_lo
; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x7060302
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v0, v3, v0, 0x7060302
; GFX10-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
@@ -5457,83 +5471,87 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v6, s[0:1]
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v6, s[4:5]
; GFX11-TRUE16-NEXT: global_load_b64 v[4:5], v6, s[6:7]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_lshlrev_b32 v8, 16, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v0, v8, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v12, v8, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v16, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v1, v11, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v14, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v16, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v4, v2, v5
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v11, v12, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v14, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v0, v2, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v4, v2, v5 :: v_dual_cndmask_b32 v1, v9, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v7, v3, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v5, v8 :: v_dual_fmac_f32 v1, v3, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v9, v12 :: v_dual_fmac_f32 v0, v2, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v8 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v1, v13, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
; GFX11-TRUE16-NEXT: global_store_b64 v6, v[0:1], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -5551,85 +5569,85 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v6, s[0:1]
; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v6, s[4:5]
; GFX11-FAKE16-NEXT: global_load_b64 v[4:5], v6, s[6:7]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v11, v3, v3, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_lshlrev_b32 v8, 16, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v15, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v16, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v0, v8, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v7, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_and_b32 v8, 0xffff0000, v11
+; GFX11-FAKE16-NEXT: v_bfe_u32 v16, v0, 16, 1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v12, v8, v9
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v14, v7, 16, 1
+; GFX11-FAKE16-NEXT: v_add3_u32 v9, v16, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v15, 0x400000, v7
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v9, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v4, v2, v5
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v11, v12, v9
-; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
-; GFX11-FAKE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v1, v3, v10
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v7, v3, v5
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v0, v2, v10
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
-; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v0, v2, v10 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v1, v11, v4
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v14, v7, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v12, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, 0x400000, v12
+; GFX11-FAKE16-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v15, vcc_lo
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v9, v11, v12, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v16, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v8, v7 :: v_dual_and_b32 v4, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v4, v2, v5
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v9, v14, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v1, v13, v10
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v2, v3, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v9, v4, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v3, v7, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v11, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v7, v12, vcc_lo
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x7060302
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v3, v0, 0x7060302
; GFX11-FAKE16-NEXT: global_store_b64 v6, v[0:1], s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll b/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll
index f325fef03f936..e8e6740cf2f56 100644
--- a/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll
+++ b/llvm/test/CodeGen/ARM/arm-bf16-dotprod-intrinsics.ll
@@ -51,7 +51,8 @@ entry:
define <2 x float> @test_vbfdot_laneq_f32(<2 x float> %r, <4 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-LABEL: test_vbfdot_laneq_f32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vdot.bf16 d0, d1, d3[1]
+; CHECK-NEXT: vdup.32 q8, d3[1]
+; CHECK-NEXT: vdot.bf16 d0, d1, d16
; CHECK-NEXT: bx lr
entry:
%.cast = bitcast <8 x bfloat> %b to <4 x float>
diff --git a/llvm/test/CodeGen/ARM/bf16-create-get-set-dup.ll b/llvm/test/CodeGen/ARM/bf16-create-get-set-dup.ll
index 39416ccdd7765..aae255df2e34b 100644
--- a/llvm/test/CodeGen/ARM/bf16-create-get-set-dup.ll
+++ b/llvm/test/CodeGen/ARM/bf16-create-get-set-dup.ll
@@ -63,7 +63,8 @@ entry:
define arm_aapcs_vfpcc <4 x bfloat> @test_vdup_laneq_bf16(<8 x bfloat> %v) {
; CHECK-LABEL: test_vdup_laneq_bf16:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vdup.16 d0, d1[3]
+; CHECK-NEXT: vdup.16 q0, d1[3]
+; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: bx lr
entry:
%lane = shufflevector <8 x bfloat> %v, <8 x bfloat> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
diff --git a/llvm/test/CodeGen/ARM/bf16-shuffle.ll b/llvm/test/CodeGen/ARM/bf16-shuffle.ll
index a45ad8f698b23..e7cc3f052b39b 100644
--- a/llvm/test/CodeGen/ARM/bf16-shuffle.ll
+++ b/llvm/test/CodeGen/ARM/bf16-shuffle.ll
@@ -269,7 +269,8 @@ entry:
define dso_local <4 x bfloat> @test_vext_unaligned_bf16(<8 x bfloat> %a) {
; CHECK-LABEL: test_vext_unaligned_bf16:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vext.16 d0, d0, d1, #3
+; CHECK-NEXT: vext.16 q0, q0, q8, #3
+; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: bx lr
entry:
%vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
diff --git a/llvm/test/CodeGen/ARM/nop_concat_vectors.ll b/llvm/test/CodeGen/ARM/nop_concat_vectors.ll
index aa3cdc3495300..eb89b378143e7 100644
--- a/llvm/test/CodeGen/ARM/nop_concat_vectors.ll
+++ b/llvm/test/CodeGen/ARM/nop_concat_vectors.ll
@@ -4,6 +4,8 @@
define void @foo(ptr %J) {
; CHECK-LABEL: foo:
; CHECK: @ %bb.0:
+; CHECK-NEXT: vldr d17, [r0, #8]
+; CHECK-NEXT: vst1.32 {d16, d17}, [r0]
; CHECK-NEXT: bx lr
%A = load <16 x i8>, ptr %J
%T1 = shufflevector <16 x i8> %A, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/CodeGen/ARM/vext.ll b/llvm/test/CodeGen/ARM/vext.ll
index f1672643e4593..1a378283fe59b 100644
--- a/llvm/test/CodeGen/ARM/vext.ll
+++ b/llvm/test/CodeGen/ARM/vext.ll
@@ -239,9 +239,8 @@ define <4 x i16> @test_largespan(ptr %B) nounwind {
; CHECK-LABEL: test_largespan:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vorr d18, d16, d16
-; CHECK-NEXT: vuzp.16 d18, d17
-; CHECK-NEXT: vmov r0, r1, d18
+; CHECK-NEXT: vuzp.16 q8, q9
+; CHECK-NEXT: vmov r0, r1, d16
; CHECK-NEXT: mov pc, lr
%tmp1 = load <8 x i16>, ptr %B
%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
diff --git a/llvm/test/CodeGen/ARM/vpadd.ll b/llvm/test/CodeGen/ARM/vpadd.ll
index a98eabc63ef2a..24b6165033ee9 100644
--- a/llvm/test/CodeGen/ARM/vpadd.ll
+++ b/llvm/test/CodeGen/ARM/vpadd.ll
@@ -218,7 +218,8 @@ define void @addCombineToVPADD_i8(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADD_i8:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpadd.i8 d16, d16, d17
+; CHECK-NEXT: vuzp.8 q8, q9
+; CHECK-NEXT: vadd.i8 d16, d18, d16
; CHECK-NEXT: vstr d16, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <16 x i8>, ptr %cbcr
@@ -235,7 +236,8 @@ define void @addCombineToVPADD_i16(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADD_i16:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpadd.i16 d16, d16, d17
+; CHECK-NEXT: vuzp.16 q8, q9
+; CHECK-NEXT: vadd.i16 d16, d18, d16
; CHECK-NEXT: vstr d16, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <8 x i16>, ptr %cbcr
@@ -251,7 +253,8 @@ define void @addCombineToVPADD_i32(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADD_i32:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpadd.i32 d16, d16, d17
+; CHECK-NEXT: vuzp.32 q8, q9
+; CHECK-NEXT: vadd.i32 d16, d18, d16
; CHECK-NEXT: vstr d16, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <4 x i32>, ptr %cbcr
@@ -267,7 +270,8 @@ define void @addCombineToVPADDLq_s8(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_s8:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.s8 q8, q8
+; CHECK-NEXT: vuzp.8 q8, q9
+; CHECK-NEXT: vaddl.s8 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <16 x i8>, ptr %cbcr
@@ -308,7 +312,8 @@ define void @addCombineToVPADDLq_u8(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_u8:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.u8 q8, q8
+; CHECK-NEXT: vuzp.8 q8, q9
+; CHECK-NEXT: vaddl.u8 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <16 x i8>, ptr %cbcr
@@ -387,7 +392,8 @@ define void @addCombineToVPADDLq_s16(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_s16:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.s16 q8, q8
+; CHECK-NEXT: vuzp.16 q8, q9
+; CHECK-NEXT: vaddl.s16 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <8 x i16>, ptr %cbcr
@@ -405,7 +411,8 @@ define void @addCombineToVPADDLq_u16(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_u16:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.u16 q8, q8
+; CHECK-NEXT: vuzp.16 q8, q9
+; CHECK-NEXT: vaddl.u16 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <8 x i16>, ptr %cbcr
@@ -423,7 +430,8 @@ define void @addCombineToVPADDLq_s32(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_s32:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.s32 q8, q8
+; CHECK-NEXT: vuzp.32 q8, q9
+; CHECK-NEXT: vaddl.s32 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <4 x i32>, ptr %cbcr
@@ -441,7 +449,8 @@ define void @addCombineToVPADDLq_u32(ptr %cbcr, ptr %X) nounwind ssp {
; CHECK-LABEL: addCombineToVPADDLq_u32:
; CHECK: @ %bb.0:
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vpaddl.u32 q8, q8
+; CHECK-NEXT: vuzp.32 q8, q9
+; CHECK-NEXT: vaddl.u32 q8, d18, d16
; CHECK-NEXT: vst1.64 {d16, d17}, [r1]
; CHECK-NEXT: mov pc, lr
%tmp = load <4 x i32>, ptr %cbcr
diff --git a/llvm/test/CodeGen/ARM/vuzp.ll b/llvm/test/CodeGen/ARM/vuzp.ll
index d24dadc7fc401..daaeef5795230 100644
--- a/llvm/test/CodeGen/ARM/vuzp.ll
+++ b/llvm/test/CodeGen/ARM/vuzp.ll
@@ -522,11 +522,11 @@ define <10 x i8> @vuzp_wide_type(<10 x i8> %tr0, <10 x i8> %tr1,
define %struct.uint8x8x2_t @vuzp_extract_subvector(<16 x i8> %t) #0 {
; CHECK-LABEL: vuzp_extract_subvector:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmov d16, r2, r3
-; CHECK-NEXT: vmov d17, r0, r1
-; CHECK-NEXT: vuzp.8 d17, d16
-; CHECK-NEXT: vmov r0, r1, d17
-; CHECK-NEXT: vmov r2, r3, d16
+; CHECK-NEXT: vmov d17, r2, r3
+; CHECK-NEXT: vmov d16, r0, r1
+; CHECK-NEXT: vuzp.8 q8, q9
+; CHECK-NEXT: vmov r0, r1, d16
+; CHECK-NEXT: vmov r2, r3, d18
; CHECK-NEXT: mov pc, lr
%vuzp.i = shufflevector <16 x i8> %t, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
diff --git a/llvm/test/CodeGen/ARM/vzip.ll b/llvm/test/CodeGen/ARM/vzip.ll
index ce40a2e48b6e8..bc9aaf22e5d6b 100644
--- a/llvm/test/CodeGen/ARM/vzip.ll
+++ b/llvm/test/CodeGen/ARM/vzip.ll
@@ -351,8 +351,9 @@ define void @vzip_vext_factor(ptr %A, ptr %B) {
; CHECK-LABEL: vzip_vext_factor:
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: vld1.64 {d16, d17}, [r0]
-; CHECK-NEXT: vext.16 d18, d16, d17, #1
-; CHECK-NEXT: vext.16 d16, d18, d17, #2
+; CHECK-NEXT: vext.16 d16, d16, d17, #3
+; CHECK-NEXT: vext.16 d17, d16, d16, #1
+; CHECK-NEXT: vzip.16 d16, d17
; CHECK-NEXT: vext.16 d16, d16, d16, #1
; CHECK-NEXT: vstr d16, [r1]
; CHECK-NEXT: mov pc, lr
diff --git a/llvm/test/CodeGen/LoongArch/lasx/and-not-combine.ll b/llvm/test/CodeGen/LoongArch/lasx/and-not-combine.ll
index 5ed49d959bf33..c354de72ac4ae 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/and-not-combine.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/and-not-combine.ll
@@ -414,8 +414,9 @@ define void @and_extract_subvector_not_combine_v32i8(ptr %pa, ptr %dst) nounwind
; CHECK-LABEL: and_extract_subvector_not_combine_v32i8:
; CHECK: # %bb.0:
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
-; CHECK-NEXT: vnori.b $vr0, $vr0, 251
+; CHECK-NEXT: xvxori.b $xr0, $xr0, 255
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
+; CHECK-NEXT: vandi.b $vr0, $vr0, 4
; CHECK-NEXT: vst $vr0, $a1, 0
; CHECK-NEXT: ret
%a = load volatile <32 x i8>, ptr %pa
@@ -432,9 +433,10 @@ define void @and_extract_subvector_not_combine_v16i16(ptr %pa, ptr %dst) nounwin
; CHECK-LABEL: and_extract_subvector_not_combine_v16i16:
; CHECK: # %bb.0:
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvxori.b $xr0, $xr0, 255
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: vrepli.h $vr1, 4
-; CHECK-NEXT: vandn.v $vr0, $vr0, $vr1
+; CHECK-NEXT: vand.v $vr0, $vr0, $vr1
; CHECK-NEXT: vst $vr0, $a1, 0
; CHECK-NEXT: ret
%a = load volatile <16 x i16>, ptr %pa
@@ -450,9 +452,10 @@ define void @and_extract_subvector_not_combine_v8i32(ptr %pa, ptr %dst) nounwind
; CHECK-LABEL: and_extract_subvector_not_combine_v8i32:
; CHECK: # %bb.0:
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvxori.b $xr0, $xr0, 255
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: vrepli.w $vr1, 4
-; CHECK-NEXT: vandn.v $vr0, $vr0, $vr1
+; CHECK-NEXT: vand.v $vr0, $vr0, $vr1
; CHECK-NEXT: vst $vr0, $a1, 0
; CHECK-NEXT: ret
%a = load volatile <8 x i32>, ptr %pa
@@ -467,9 +470,10 @@ define void @and_extract_subvector_not_combine_v4i64(ptr %pa, ptr %dst) nounwind
; CHECK-LABEL: and_extract_subvector_not_combine_v4i64:
; CHECK: # %bb.0:
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvxori.b $xr0, $xr0, 255
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: vrepli.d $vr1, 4
-; CHECK-NEXT: vandn.v $vr0, $vr0, $vr1
+; CHECK-NEXT: vand.v $vr0, $vr0, $vr1
; CHECK-NEXT: vst $vr0, $a1, 0
; CHECK-NEXT: ret
%a = load volatile <4 x i64>, ptr %pa
diff --git a/llvm/test/CodeGen/LoongArch/lasx/insert-extract-subvector.ll b/llvm/test/CodeGen/LoongArch/lasx/insert-extract-subvector.ll
index 48d6e0130105e..04bbfd14c632a 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/insert-extract-subvector.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/insert-extract-subvector.ll
@@ -411,7 +411,7 @@ entry:
define <4 x i32> @extract_hi128_v8i32_1(<8 x i32> %a) {
; CHECK-LABEL: extract_hi128_v8i32_1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -433,7 +433,7 @@ entry:
define <4 x i32> @extract_hi128_v8i32_2(<8 x i32> %a, <8 x i32> %b) {
; CHECK-LABEL: extract_hi128_v8i32_2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr1, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr1, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -454,7 +454,7 @@ entry:
define <4 x float> @extract_hi128_v8f32_1(<8 x float> %a) {
; CHECK-LABEL: extract_hi128_v8f32_1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -476,7 +476,7 @@ entry:
define <4 x float> @extract_hi128_v8f32_2(<8 x float> %a, <8 x float> %b) {
; CHECK-LABEL: extract_hi128_v8f32_2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr1, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr1, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -497,7 +497,7 @@ entry:
define <2 x i64> @extract_hi128_v4i64_1(<4 x i64> %a) {
; CHECK-LABEL: extract_hi128_v4i64_1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -519,7 +519,7 @@ entry:
define <2 x i64> @extract_hi128_v4i64_2(<4 x i64> %a, <4 x i64> %b) {
; CHECK-LABEL: extract_hi128_v4i64_2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr1, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr1, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -540,7 +540,7 @@ entry:
define <2 x double> @extract_hi128_v4f64_1(<4 x double> %a) {
; CHECK-LABEL: extract_hi128_v4f64_1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -562,7 +562,7 @@ entry:
define <2 x double> @extract_hi128_v4f64_2(<4 x double> %a, <4 x double> %b) {
; CHECK-LABEL: extract_hi128_v4f64_2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr1, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr1, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -584,7 +584,7 @@ entry:
define <8 x i16> @extract_hi128_v16i16_1(<16 x i16> %a) {
; CHECK-LABEL: extract_hi128_v16i16_1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -608,7 +608,7 @@ entry:
define <8 x i16> @extract_hi128_v16i16_2(<16 x i16> %a, <16 x i16> %b) {
; CHECK-LABEL: extract_hi128_v16i16_2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr1, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr1, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -632,7 +632,7 @@ entry:
define <16 x i8> @extract_hi128_v32i8_1(<32 x i8> %a) {
; CHECK-LABEL: extract_hi128_v32i8_1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr0, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
@@ -658,7 +658,7 @@ entry:
define <16 x i8> @extract_hi128_v32i8_2(<32 x i8> %a, <32 x i8> %b) {
; CHECK-LABEL: extract_hi128_v32i8_2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: xvpermi.q $xr0, $xr1, 1
+; CHECK-NEXT: xvpermi.d $xr0, $xr1, 14
; CHECK-NEXT: # kill: def $vr0 killed $vr0 killed $xr0
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
index c05f306424519..6708bd2528d87 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-changes-length.ll
@@ -6,89 +6,57 @@
define <8 x i1> @v8i1_v16i1(<16 x i1>) {
; RV32-LABEL: v8i1_v16i1:
; RV32: # %bb.0:
-; RV32-NEXT: vsetivli zero, 1, e16, m1, ta, ma
-; RV32-NEXT: vmv.x.s a0, v0
-; RV32-NEXT: slli a1, a0, 18
-; RV32-NEXT: srli a2, a0, 31
-; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; RV32-NEXT: vmv.v.x v8, a2
-; RV32-NEXT: slli a2, a0, 27
-; RV32-NEXT: srli a1, a1, 31
-; RV32-NEXT: vslide1down.vx v8, v8, a1
-; RV32-NEXT: slli a1, a0, 26
-; RV32-NEXT: srli a1, a1, 31
-; RV32-NEXT: vmv.v.x v9, a1
-; RV32-NEXT: slli a1, a0, 28
-; RV32-NEXT: srli a2, a2, 31
-; RV32-NEXT: vslide1down.vx v8, v8, a2
-; RV32-NEXT: slli a2, a0, 19
-; RV32-NEXT: srli a2, a2, 31
-; RV32-NEXT: vslide1down.vx v9, v9, a2
-; RV32-NEXT: slli a2, a0, 24
-; RV32-NEXT: slli a0, a0, 29
-; RV32-NEXT: srli a1, a1, 31
-; RV32-NEXT: srli a2, a2, 31
-; RV32-NEXT: srli a0, a0, 31
-; RV32-NEXT: vslide1down.vx v8, v8, a1
-; RV32-NEXT: vslide1down.vx v9, v9, a2
-; RV32-NEXT: vmv.v.i v0, 15
-; RV32-NEXT: vslide1down.vx v9, v9, a0
-; RV32-NEXT: vslidedown.vi v8, v9, 4, v0.t
-; RV32-NEXT: vand.vi v8, v8, 1
-; RV32-NEXT: vmsne.vi v0, v8, 0
+; RV32-NEXT: lui a0, %hi(.LCPI0_0)
+; RV32-NEXT: addi a0, a0, %lo(.LCPI0_0)
+; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; RV32-NEXT: vle8.v v8, (a0)
+; RV32-NEXT: vmv.v.i v9, 0
+; RV32-NEXT: vmerge.vim v9, v9, 1, v0
+; RV32-NEXT: vrgather.vv v10, v9, v8
+; RV32-NEXT: vmsne.vi v0, v10, 0
; RV32-NEXT: ret
;
; RV64-LABEL: v8i1_v16i1:
; RV64: # %bb.0:
-; RV64-NEXT: vsetivli zero, 1, e16, m1, ta, ma
-; RV64-NEXT: vmv.x.s a0, v0
-; RV64-NEXT: slli a1, a0, 50
-; RV64-NEXT: srli a2, a0, 63
-; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; RV64-NEXT: vmv.v.x v8, a2
-; RV64-NEXT: slli a2, a0, 59
-; RV64-NEXT: srli a1, a1, 63
-; RV64-NEXT: vslide1down.vx v8, v8, a1
-; RV64-NEXT: slli a1, a0, 58
-; RV64-NEXT: srli a1, a1, 63
-; RV64-NEXT: vmv.v.x v9, a1
-; RV64-NEXT: slli a1, a0, 60
-; RV64-NEXT: srli a2, a2, 63
-; RV64-NEXT: vslide1down.vx v8, v8, a2
-; RV64-NEXT: slli a2, a0, 51
-; RV64-NEXT: srli a2, a2, 63
-; RV64-NEXT: vslide1down.vx v9, v9, a2
-; RV64-NEXT: slli a2, a0, 56
-; RV64-NEXT: slli a0, a0, 61
-; RV64-NEXT: srli a1, a1, 63
-; RV64-NEXT: srli a2, a2, 63
-; RV64-NEXT: srli a0, a0, 63
-; RV64-NEXT: vslide1down.vx v8, v8, a1
-; RV64-NEXT: vslide1down.vx v9, v9, a2
-; RV64-NEXT: vmv.v.i v0, 15
-; RV64-NEXT: vslide1down.vx v9, v9, a0
-; RV64-NEXT: vslidedown.vi v8, v9, 4, v0.t
-; RV64-NEXT: vand.vi v8, v8, 1
-; RV64-NEXT: vmsne.vi v0, v8, 0
+; RV64-NEXT: lui a0, %hi(.LCPI0_0)
+; RV64-NEXT: ld a0, %lo(.LCPI0_0)(a0)
+; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; RV64-NEXT: vmv.v.i v8, 0
+; RV64-NEXT: vmerge.vim v8, v8, 1, v0
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vmv.v.x v9, a0
+; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; RV64-NEXT: vrgather.vv v10, v8, v9
+; RV64-NEXT: vmsne.vi v0, v10, 0
; RV64-NEXT: ret
%2 = shufflevector <16 x i1> %0, <16 x i1> poison, <8 x i32> <i32 5, i32 12, i32 7, i32 2, i32 15, i32 13, i32 4, i32 3>
ret <8 x i1> %2
}
define <4 x i32> @v4i32_v8i32(<8 x i32>) {
-; CHECK-LABEL: v4i32_v8i32:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT: vmv.v.i v0, 8
-; CHECK-NEXT: vslidedown.vi v10, v8, 2
-; CHECK-NEXT: vslideup.vi v10, v8, 1, v0.t
-; CHECK-NEXT: vmv.v.i v0, 5
-; CHECK-NEXT: vsetivli zero, 4, e32, m2, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 4
-; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, mu
-; CHECK-NEXT: vslidedown.vi v10, v8, 1, v0.t
-; CHECK-NEXT: vmv.v.v v8, v10
-; CHECK-NEXT: ret
+; RV32-LABEL: v4i32_v8i32:
+; RV32: # %bb.0:
+; RV32-NEXT: lui a0, %hi(.LCPI1_0)
+; RV32-NEXT: addi a0, a0, %lo(.LCPI1_0)
+; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT: vle16.v v12, (a0)
+; RV32-NEXT: vrgatherei16.vv v10, v8, v12
+; RV32-NEXT: vmv1r.v v8, v10
+; RV32-NEXT: ret
+;
+; RV64-LABEL: v4i32_v8i32:
+; RV64: # %bb.0:
+; RV64-NEXT: lui a0, 131079
+; RV64-NEXT: slli a0, a0, 4
+; RV64-NEXT: addi a0, a0, 3
+; RV64-NEXT: slli a0, a0, 16
+; RV64-NEXT: addi a0, a0, 5
+; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT: vmv.v.x v12, a0
+; RV64-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT: vrgatherei16.vv v10, v8, v12
+; RV64-NEXT: vmv1r.v v8, v10
+; RV64-NEXT: ret
%2 = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> <i32 5, i32 3, i32 7, i32 2>
ret <4 x i32> %2
}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
index 63b96a96a284a..2b2a5a291c1f9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
@@ -9,19 +9,14 @@
define void @deinterleave3_0_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave3_0_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: lui a0, %hi(.LCPI0_0)
-; CHECK-NEXT: addi a0, a0, %lo(.LCPI0_0)
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vle8.v v9, (a0)
-; CHECK-NEXT: li a0, 73
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v8, 8
+; CHECK-NEXT: lui a0, 9
+; CHECK-NEXT: addi a0, a0, 585
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT: vrgather.vv v10, v8, v9
; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
@@ -34,19 +29,14 @@ entry:
define void @deinterleave3_8_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave3_8_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: lui a0, %hi(.LCPI1_0)
-; CHECK-NEXT: addi a0, a0, %lo(.LCPI1_0)
+; CHECK-NEXT: lui a0, 2
+; CHECK-NEXT: addi a0, a0, 1170
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vle8.v v9, (a0)
-; CHECK-NEXT: li a0, 146
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0
-; CHECK-NEXT: vrgather.vv v10, v8, v9
; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
@@ -97,17 +87,15 @@ entry:
define void @deinterleave5_0_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave5_0_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: li a0, 33
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v9, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0
-; CHECK-NEXT: vmv.v.i v0, 10
-; CHECK-NEXT: vslidedown.vi v8, v8, 4, v0.t
-; CHECK-NEXT: vse8.v v8, (a1)
+; CHECK-NEXT: lui a0, 8
+; CHECK-NEXT: addi a0, a0, 1057
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -119,18 +107,15 @@ entry:
define void @deinterleave5_8_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave5_8_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v9, v8, 1
-; CHECK-NEXT: vslidedown.vi v9, v8, 5, v0.t
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vrgather.vi v9, v8, 3, v0.t
-; CHECK-NEXT: vse8.v v9, (a1)
+; CHECK-NEXT: lui a0, 1
+; CHECK-NEXT: addi a0, a0, -1982
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -142,16 +127,15 @@ entry:
define void @deinterleave6_0_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave6_0_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v9, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v8, v8, 5, v0.t
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vrgather.vi v8, v9, 4, v0.t
-; CHECK-NEXT: vse8.v v8, (a1)
+; CHECK-NEXT: lui a0, 1
+; CHECK-NEXT: addi a0, a0, 65
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -163,18 +147,15 @@ entry:
define void @deinterleave6_8_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave6_8_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v9, v8, 1
-; CHECK-NEXT: vslidedown.vi v9, v8, 6, v0.t
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vrgather.vi v9, v8, 5, v0.t
-; CHECK-NEXT: vse8.v v9, (a1)
+; CHECK-NEXT: lui a0, 2
+; CHECK-NEXT: addi a0, a0, 130
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -186,16 +167,15 @@ entry:
define void @deinterleave7_0_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave7_0_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v9, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v8, v8, 6, v0.t
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vrgather.vi v8, v9, 6, v0.t
-; CHECK-NEXT: vse8.v v8, (a1)
+; CHECK-NEXT: lui a0, 4
+; CHECK-NEXT: addi a0, a0, 129
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -207,18 +187,15 @@ entry:
define void @deinterleave7_8_i8(ptr %in, ptr %out) {
; CHECK-LABEL: deinterleave7_8_i8:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
-; CHECK-NEXT: vslidedown.vi v9, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT: vslidedown.vi v10, v9, 4
-; CHECK-NEXT: vslideup.vi v10, v9, 1, v0.t
-; CHECK-NEXT: vmv.v.i v0, 6
-; CHECK-NEXT: vrgather.vi v9, v8, 1
-; CHECK-NEXT: vmerge.vvm v8, v9, v10, v0
-; CHECK-NEXT: vse8.v v8, (a1)
+; CHECK-NEXT: lui a0, 4
+; CHECK-NEXT: addi a0, a0, 258
+; CHECK-NEXT: vmv.s.x v9, a0
+; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT: vcompress.vm v10, v8, v9
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -275,17 +252,11 @@ define void @deinterleave7_0_i64(ptr %in, ptr %out) {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma
; CHECK-NEXT: vle64.v v8, (a0)
-; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT: vmv.v.i v0, 2
-; CHECK-NEXT: vmv4r.v v16, v8
-; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, mu
-; CHECK-NEXT: vslidedown.vi v16, v8, 6, v0.t
-; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT: vmv.v.i v0, 4
-; CHECK-NEXT: vsetivli zero, 8, e64, m8, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, mu
-; CHECK-NEXT: vrgather.vi v16, v8, 6, v0.t
+; CHECK-NEXT: lui a0, 4
+; CHECK-NEXT: addi a0, a0, 129
+; CHECK-NEXT: vmv.s.x v24, a0
+; CHECK-NEXT: vcompress.vm v16, v8, v24
+; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
; CHECK-NEXT: vse64.v v16, (a1)
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
index 175693318500b..a2296abc01804 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
@@ -230,9 +230,9 @@ define void @vnsrl_0_i32(ptr %in, ptr %out) {
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, ma
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; ZVE32F-NEXT: vslideup.vi v8, v9, 1
+; ZVE32F-NEXT: vmv.v.i v0, 2
+; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, mu
+; ZVE32F-NEXT: vslidedown.vi v8, v8, 1, v0.t
; ZVE32F-NEXT: vse32.v v8, (a1)
; ZVE32F-NEXT: ret
;
@@ -275,10 +275,10 @@ define void @vnsrl_32_i32(ptr %in, ptr %out) {
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vmv.v.i v0, 1
+; ZVE32F-NEXT: vmv.v.i v0, 2
+; ZVE32F-NEXT: vslidedown.vi v9, v8, 1
; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, mu
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 1, v0.t
+; ZVE32F-NEXT: vslidedown.vi v9, v8, 2, v0.t
; ZVE32F-NEXT: vse32.v v9, (a1)
; ZVE32F-NEXT: ret
;
@@ -322,9 +322,9 @@ define void @vnsrl_0_float(ptr %in, ptr %out) {
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, ma
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; ZVE32F-NEXT: vslideup.vi v8, v9, 1
+; ZVE32F-NEXT: vmv.v.i v0, 2
+; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, mu
+; ZVE32F-NEXT: vslidedown.vi v8, v8, 1, v0.t
; ZVE32F-NEXT: vse32.v v8, (a1)
; ZVE32F-NEXT: ret
;
@@ -367,10 +367,10 @@ define void @vnsrl_32_float(ptr %in, ptr %out) {
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vmv.v.i v0, 1
+; ZVE32F-NEXT: vmv.v.i v0, 2
+; ZVE32F-NEXT: vslidedown.vi v9, v8, 1
; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, mu
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 2
-; ZVE32F-NEXT: vslidedown.vi v9, v8, 1, v0.t
+; ZVE32F-NEXT: vslidedown.vi v9, v8, 2, v0.t
; ZVE32F-NEXT: vse32.v v9, (a1)
; ZVE32F-NEXT: ret
;
@@ -405,9 +405,9 @@ define void @vnsrl_0_i64(ptr %in, ptr %out) {
; V: # %bb.0: # %entry
; V-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; V-NEXT: vle64.v v8, (a0)
-; V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; V-NEXT: vslidedown.vi v9, v8, 2
-; V-NEXT: vslideup.vi v8, v9, 1
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT: vslidedown.vi v8, v8, 1, v0.t
; V-NEXT: vse64.v v8, (a1)
; V-NEXT: ret
;
@@ -432,9 +432,9 @@ define void @vnsrl_0_i64(ptr %in, ptr %out) {
; ZVZIP: # %bb.0: # %entry
; ZVZIP-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; ZVZIP-NEXT: vle64.v v8, (a0)
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vslidedown.vi v9, v8, 2
-; ZVZIP-NEXT: vslideup.vi v8, v9, 1
+; ZVZIP-NEXT: vmv.v.i v0, 2
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; ZVZIP-NEXT: vslidedown.vi v8, v8, 1, v0.t
; ZVZIP-NEXT: vse64.v v8, (a1)
; ZVZIP-NEXT: ret
entry:
@@ -449,10 +449,10 @@ define void @vnsrl_64_i64(ptr %in, ptr %out) {
; V: # %bb.0: # %entry
; V-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; V-NEXT: vle64.v v8, (a0)
-; V-NEXT: vmv.v.i v0, 1
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vslidedown.vi v9, v8, 1
; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
-; V-NEXT: vslidedown.vi v9, v8, 2
-; V-NEXT: vslidedown.vi v9, v8, 1, v0.t
+; V-NEXT: vslidedown.vi v9, v8, 2, v0.t
; V-NEXT: vse64.v v9, (a1)
; V-NEXT: ret
;
@@ -477,10 +477,11 @@ define void @vnsrl_64_i64(ptr %in, ptr %out) {
; ZVZIP: # %bb.0: # %entry
; ZVZIP-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; ZVZIP-NEXT: vle64.v v8, (a0)
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vslidedown.vi v9, v8, 2
-; ZVZIP-NEXT: vpairo.vv v10, v8, v9
-; ZVZIP-NEXT: vse64.v v10, (a1)
+; ZVZIP-NEXT: vmv.v.i v0, 2
+; ZVZIP-NEXT: vslidedown.vi v9, v8, 1
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; ZVZIP-NEXT: vslidedown.vi v9, v8, 2, v0.t
+; ZVZIP-NEXT: vse64.v v9, (a1)
; ZVZIP-NEXT: ret
entry:
%0 = load <4 x i64>, ptr %in, align 8
@@ -494,9 +495,9 @@ define void @vnsrl_0_double(ptr %in, ptr %out) {
; V: # %bb.0: # %entry
; V-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; V-NEXT: vle64.v v8, (a0)
-; V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; V-NEXT: vslidedown.vi v9, v8, 2
-; V-NEXT: vslideup.vi v8, v9, 1
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; V-NEXT: vslidedown.vi v8, v8, 1, v0.t
; V-NEXT: vse64.v v8, (a1)
; V-NEXT: ret
;
@@ -521,9 +522,9 @@ define void @vnsrl_0_double(ptr %in, ptr %out) {
; ZVZIP: # %bb.0: # %entry
; ZVZIP-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; ZVZIP-NEXT: vle64.v v8, (a0)
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vslidedown.vi v9, v8, 2
-; ZVZIP-NEXT: vslideup.vi v8, v9, 1
+; ZVZIP-NEXT: vmv.v.i v0, 2
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; ZVZIP-NEXT: vslidedown.vi v8, v8, 1, v0.t
; ZVZIP-NEXT: vse64.v v8, (a1)
; ZVZIP-NEXT: ret
entry:
@@ -538,10 +539,10 @@ define void @vnsrl_64_double(ptr %in, ptr %out) {
; V: # %bb.0: # %entry
; V-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; V-NEXT: vle64.v v8, (a0)
-; V-NEXT: vmv.v.i v0, 1
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vslidedown.vi v9, v8, 1
; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
-; V-NEXT: vslidedown.vi v9, v8, 2
-; V-NEXT: vslidedown.vi v9, v8, 1, v0.t
+; V-NEXT: vslidedown.vi v9, v8, 2, v0.t
; V-NEXT: vse64.v v9, (a1)
; V-NEXT: ret
;
@@ -566,10 +567,11 @@ define void @vnsrl_64_double(ptr %in, ptr %out) {
; ZVZIP: # %bb.0: # %entry
; ZVZIP-NEXT: vsetivli zero, 4, e64, m1, ta, ma
; ZVZIP-NEXT: vle64.v v8, (a0)
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vslidedown.vi v9, v8, 2
-; ZVZIP-NEXT: vpairo.vv v10, v8, v9
-; ZVZIP-NEXT: vse64.v v10, (a1)
+; ZVZIP-NEXT: vmv.v.i v0, 2
+; ZVZIP-NEXT: vslidedown.vi v9, v8, 1
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, mu
+; ZVZIP-NEXT: vslidedown.vi v9, v8, 2, v0.t
+; ZVZIP-NEXT: vse64.v v9, (a1)
; ZVZIP-NEXT: ret
entry:
%0 = load <4 x double>, ptr %in, align 8
@@ -628,27 +630,59 @@ entry:
; Not a vnsrl (checking for a prior pattern matching bug)
define void @vnsrl_0_i8_undef_negative(ptr %in, ptr %out) {
-; CHECK-LABEL: vnsrl_0_i8_undef_negative:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
-; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: li a0, 32
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: lui a0, %hi(.LCPI17_0)
-; CHECK-NEXT: addi a0, a0, %lo(.LCPI17_0)
-; CHECK-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
-; CHECK-NEXT: vle8.v v9, (a0)
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vslidedown.vi v10, v8, 8
-; CHECK-NEXT: vsetivli zero, 8, e8, mf4, ta, mu
-; CHECK-NEXT: vslideup.vi v11, v10, 4
-; CHECK-NEXT: vslideup.vi v11, v10, 3, v0.t
-; CHECK-NEXT: li a0, 48
-; CHECK-NEXT: vmv.s.x v0, a0
-; CHECK-NEXT: vrgather.vv v10, v8, v9
-; CHECK-NEXT: vmerge.vvm v8, v10, v11, v0
-; CHECK-NEXT: vse8.v v8, (a1)
-; CHECK-NEXT: ret
+; V-LABEL: vnsrl_0_i8_undef_negative:
+; V: # %bb.0: # %entry
+; V-NEXT: lui a2, %hi(.LCPI17_0)
+; V-NEXT: ld a2, %lo(.LCPI17_0)(a2)
+; V-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; V-NEXT: vle8.v v8, (a0)
+; V-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; V-NEXT: vmv.v.x v9, a2
+; V-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; V-NEXT: vrgather.vv v10, v8, v9
+; V-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
+; V-NEXT: vse8.v v10, (a1)
+; V-NEXT: ret
+;
+; ZVE32F-LABEL: vnsrl_0_i8_undef_negative:
+; ZVE32F: # %bb.0: # %entry
+; ZVE32F-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZVE32F-NEXT: vle8.v v8, (a0)
+; ZVE32F-NEXT: lui a0, %hi(.LCPI17_0)
+; ZVE32F-NEXT: addi a0, a0, %lo(.LCPI17_0)
+; ZVE32F-NEXT: vle8.v v9, (a0)
+; ZVE32F-NEXT: vrgather.vv v10, v8, v9
+; ZVE32F-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
+; ZVE32F-NEXT: vse8.v v10, (a1)
+; ZVE32F-NEXT: ret
+;
+; ZIP-LABEL: vnsrl_0_i8_undef_negative:
+; ZIP: # %bb.0: # %entry
+; ZIP-NEXT: lui a2, %hi(.LCPI17_0)
+; ZIP-NEXT: ld a2, %lo(.LCPI17_0)(a2)
+; ZIP-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZIP-NEXT: vle8.v v8, (a0)
+; ZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; ZIP-NEXT: vmv.v.x v9, a2
+; ZIP-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZIP-NEXT: vrgather.vv v10, v8, v9
+; ZIP-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
+; ZIP-NEXT: vse8.v v10, (a1)
+; ZIP-NEXT: ret
+;
+; ZVZIP-LABEL: vnsrl_0_i8_undef_negative:
+; ZVZIP: # %bb.0: # %entry
+; ZVZIP-NEXT: lui a2, %hi(.LCPI17_0)
+; ZVZIP-NEXT: ld a2, %lo(.LCPI17_0)(a2)
+; ZVZIP-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZVZIP-NEXT: vle8.v v8, (a0)
+; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
+; ZVZIP-NEXT: vmv.v.x v9, a2
+; ZVZIP-NEXT: vsetivli zero, 16, e8, mf2, ta, ma
+; ZVZIP-NEXT: vrgather.vv v10, v8, v9
+; ZVZIP-NEXT: vsetivli zero, 8, e8, mf4, ta, ma
+; ZVZIP-NEXT: vse8.v v10, (a1)
+; ZVZIP-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
%shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 poison, i32 1>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
index 9887c7ff5b182..eff8047d4372f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
@@ -555,7 +555,12 @@ define <4 x i16> @shuffle_shuffle_vslidedown(<16 x i16> %0) {
; CHECK-LABEL: shuffle_shuffle_vslidedown:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT: vslidedown.vi v8, v8, 5
+; CHECK-NEXT: vrgather.vi v10, v8, 8
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT: vslidedown.vi v8, v8, 4
+; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT: vslidedown.vi v8, v8, 1
+; CHECK-NEXT: vslideup.vi v8, v10, 3
; CHECK-NEXT: ret
entry:
%1 = shufflevector <16 x i16> %0, <16 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index 7f624cffa014c..390b2650cbeb5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -337,11 +337,11 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle2step_i16(<16 x i16> %src) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u32 q2, q1, #16
; CHECK-NEXT: vstrh.32 q2, [r0, #8]
; CHECK-NEXT: vshr.u32 q2, q0, #16
-; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: vstrh.32 q2, [r0]
; CHECK-NEXT: vstrh.32 q1, [r1, #8]
; CHECK-NEXT: vstrh.32 q0, [r1]
@@ -627,11 +627,11 @@ define arm_aapcs_vfpcc <16 x i8> @shuffle2step_i8(<32 x i8> %src) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u16 q2, q1, #8
; CHECK-NEXT: vstrb.16 q2, [r0, #8]
; CHECK-NEXT: vshr.u16 q2, q0, #8
-; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: vstrb.16 q2, [r0]
; CHECK-NEXT: vstrb.16 q1, [r1, #8]
; CHECK-NEXT: vstrb.16 q0, [r1]
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index ac2f283166a93..c14d95839f481 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -5,10 +5,10 @@
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
@@ -1959,80 +1959,42 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT: vpbroadcastd %xmm1, %ymm1
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
-; AVX512F-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT: vpbroadcastd %xmm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
-; AVX512DQ-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
-; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512BW-SLOW: # %bb.0:
-; AVX512BW-SLOW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-SLOW-NEXT: vpbroadcastd %xmm0, %ymm0
-; AVX512BW-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
-; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rcx)
-; AVX512BW-SLOW-NEXT: vzeroupper
-; AVX512BW-SLOW-NEXT: retq
+; AVX512F-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
+; AVX512F-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
;
-; AVX512BW-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512BW-FAST: # %bb.0:
-; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,9,0,11,0,13,0,15]
-; AVX512BW-FAST-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vpermd %zmm0, %zmm1, %zmm0
-; AVX512BW-FAST-NEXT: vpaddb (%rdx), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, (%rcx)
-; AVX512BW-FAST-NEXT: vzeroupper
-; AVX512BW-FAST-NEXT: retq
+; AVX512DQ-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
+; AVX512DQ-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
+;
+; AVX512BW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,9,0,11,0,13,0,15]
+; AVX512BW-NEXT: vpermd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rcx)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64
%in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64
%in.vec = add <64 x i8> %in.vec.base, %in.vec.bias
@@ -2111,57 +2073,31 @@ define void @vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [8,1,2,3,8,5,6,7]
-; AVX512F-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [8,1,2,3,8,5,6,7]
-; AVX512DQ-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
+; AVX512F-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,10,11,0,13,14,15]
+; AVX512F-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512DQ-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,10,11,0,13,14,15]
+; AVX512DQ-NEXT: vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
;
; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
; AVX512BW-SLOW: # %bb.0:
@@ -2263,57 +2199,31 @@ define void @vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1
-; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
-; AVX512F-FAST-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
-; AVX512DQ-FAST-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
+; AVX512F-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
+; AVX512F-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512DQ-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
+; AVX512DQ-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rcx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
;
; AVX512BW-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
; AVX512BW-SLOW: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
index 4a542949c7859..89ae037bb64ae 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -5,10 +5,10 @@
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
@@ -1569,8 +1569,8 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
;
; AVX512F-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastd (%rdi), %ymm0
-; AVX512F-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,0,11,0,13,0,15]
+; AVX512F-NEXT: vpermd (%rdi), %zmm0, %zmm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512F-NEXT: vzeroupper
@@ -1578,8 +1578,8 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
;
; AVX512DQ-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastd (%rdi), %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,0,11,0,13,0,15]
+; AVX512DQ-NEXT: vpermd (%rdi), %zmm0, %zmm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
@@ -1587,8 +1587,8 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
;
; AVX512BW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpbroadcastd (%rdi), %ymm0
-; AVX512BW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1],ymm0[2],mem[3],ymm0[4],mem[5],ymm0[6],mem[7]
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,0,11,0,13,0,15]
+; AVX512BW-NEXT: vpermd (%rdi), %zmm0, %zmm0
; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -1651,59 +1651,32 @@ define void @vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5,6,7]
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
-; AVX512F-FAST-NEXT: vpermd (%rdi), %zmm0, %zmm0
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5,6,7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
-; AVX512DQ-FAST-NEXT: vpermd (%rdi), %zmm0, %zmm0
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
-; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512BW-SLOW: # %bb.0:
-; AVX512BW-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX512BW-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5,6,7]
-; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-SLOW-NEXT: vzeroupper
-; AVX512BW-SLOW-NEXT: retq
+; AVX512F-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
+; AVX512F-NEXT: vpermd (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
;
-; AVX512BW-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512BW-FAST: # %bb.0:
-; AVX512BW-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
-; AVX512BW-FAST-NEXT: vpermd (%rdi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-FAST-NEXT: vzeroupper
-; AVX512BW-FAST-NEXT: retq
+; AVX512DQ-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
+; AVX512DQ-NEXT: vpermd (%rdi), %zmm0, %zmm0
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
+;
+; AVX512BW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
+; AVX512BW-NEXT: vpermd (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%in.vec = load <64 x i8>, ptr %in.elt.ptr, align 64
%in.vec.cast = bitcast <64 x i8> %in.vec to <16 x i32>
%broadcast.of.zextinreg = shufflevector <16 x i32> %in.vec.cast, <16 x i32> poison, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 0, i32 13, i32 14, i32 15>
@@ -1761,59 +1734,32 @@ define void @vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
-; AVX512F-FAST-NEXT: vpermq (%rdi), %zmm0, %zmm0
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
-; AVX512DQ-FAST-NEXT: vpermq (%rdi), %zmm0, %zmm0
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
-; AVX512BW-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512BW-SLOW: # %bb.0:
-; AVX512BW-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX512BW-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-SLOW-NEXT: vzeroupper
-; AVX512BW-SLOW-NEXT: retq
+; AVX512F-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
+; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
;
-; AVX512BW-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512BW-FAST: # %bb.0:
-; AVX512BW-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
-; AVX512BW-FAST-NEXT: vpermq (%rdi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-FAST-NEXT: vzeroupper
-; AVX512BW-FAST-NEXT: retq
+; AVX512DQ-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
+; AVX512DQ-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
+;
+; AVX512BW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
+; AVX512BW-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
%in.vec = load <64 x i8>, ptr %in.elt.ptr, align 64
%in.vec.cast = bitcast <64 x i8> %in.vec to <8 x i64>
%broadcast.of.zextinreg = shufflevector <8 x i64> %in.vec.cast, <8 x i64> poison, <4 x i32> <i32 0, i32 5, i32 0, i32 7>
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index da0cef0e4e99b..373c434efbf4b 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -259,7 +259,10 @@ define i8 @shuf_test1(i16 %v) nounwind {
;
; X86-LABEL: shuf_test1:
; X86: ## %bb.0:
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k0
+; X86-NEXT: kshiftrw $8, %k0, %k0
+; X86-NEXT: kmovd %k0, %eax
+; X86-NEXT: ## kill: def $al killed $al killed $eax
; X86-NEXT: retl
%v1 = bitcast i16 %v to <16 x i1>
%mask = shufflevector <16 x i1> %v1, <16 x i1> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
index b3bf464b529d0..8751109957527 100644
--- a/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
+++ b/llvm/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
@@ -225,25 +225,15 @@ define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask1(ptr %vp, <8 x i16
}
define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; CHECK-FAST-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
-; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]
-; CHECK-FAST-NEXT: vptestnmw %xmm1, %xmm1, %k1
-; CHECK-FAST-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-FAST-NEXT: vzeroupper
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm2
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm3
-; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]
-; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3,4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm1, %xmm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm2, %xmm0 {%k1}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]
+; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1
+; CHECK-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
%vec = load <16 x i16>, ptr %vp
%shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
%cmp = icmp eq <8 x i16> %mask, zeroinitializer
@@ -252,24 +242,14 @@ define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16>
}
define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]
-; CHECK-FAST-NEXT: vptestnmw %xmm0, %xmm0, %k1
-; CHECK-FAST-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-FAST-NEXT: vzeroupper
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm1
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm2
-; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]
-; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm0, %xmm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} {z}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]
+; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1
+; CHECK-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
%vec = load <16 x i16>, ptr %vp
%shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>
%cmp = icmp eq <8 x i16> %mask, zeroinitializer
@@ -1050,23 +1030,41 @@ define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec, <4 x i
ret <4 x i32> %res
}
define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask0(ptr %vp) {
-; CHECK-LABEL: test_8xi32_to_4xi32_perm_mem_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps 16(%rdi), %xmm0
-; CHECK-NEXT: vshufps $7, (%rdi), %xmm0, %xmm0 # xmm0 = xmm0[3,1],mem[0,0]
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [7,5,0,0,0,0,0,0]
+; CHECK-FAST-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vmovaps 16(%rdi), %xmm0
+; CHECK-FAST-PERLANE-NEXT: vshufps $7, (%rdi), %xmm0, %xmm0 # xmm0 = xmm0[3,1],mem[0,0]
+; CHECK-FAST-PERLANE-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>
ret <4 x i32> %res
}
define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {
-; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps 16(%rdi), %xmm2
-; CHECK-NEXT: vshufps $7, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[3,1],mem[0,0]
-; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k1
-; CHECK-NEXT: vmovdqa32 %xmm2, %xmm0 {%k1}
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [7,5,0,0,0,0,0,0]
+; CHECK-FAST-NEXT: vptestnmd %xmm1, %xmm1, %k1
+; CHECK-FAST-NEXT: vpermd (%rdi), %ymm2, %ymm0 {%k1}
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vmovaps 16(%rdi), %xmm2
+; CHECK-FAST-PERLANE-NEXT: vshufps $7, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[3,1],mem[0,0]
+; CHECK-FAST-PERLANE-NEXT: vptestnmd %xmm1, %xmm1, %k1
+; CHECK-FAST-PERLANE-NEXT: vmovdqa32 %xmm2, %xmm0 {%k1}
+; CHECK-FAST-PERLANE-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>
%cmp = icmp eq <4 x i32> %mask, zeroinitializer
@@ -1075,13 +1073,22 @@ define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %
}
define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %mask) {
-; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps 16(%rdi), %xmm1
-; CHECK-NEXT: vshufps $7, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[3,1],mem[0,0]
-; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k1
-; CHECK-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} {z}
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [7,5,0,0,0,0,0,0]
+; CHECK-FAST-NEXT: vptestnmd %xmm0, %xmm0, %k1
+; CHECK-FAST-NEXT: vpermd (%rdi), %ymm1, %ymm0 {%k1} {z}
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vmovaps 16(%rdi), %xmm1
+; CHECK-FAST-PERLANE-NEXT: vshufps $7, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[3,1],mem[0,0]
+; CHECK-FAST-PERLANE-NEXT: vptestnmd %xmm0, %xmm0, %k1
+; CHECK-FAST-PERLANE-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} {z}
+; CHECK-FAST-PERLANE-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>
%cmp = icmp eq <4 x i32> %mask, zeroinitializer
@@ -1158,9 +1165,10 @@ define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask2(ptr %vp, <4 x i32>
define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask3(ptr %vp) {
; CHECK-LABEL: test_8xi32_to_4xi32_perm_mem_mask3:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm1
-; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [5,1,2,7]
-; CHECK-NEXT: vpermi2d 16(%rdi), %xmm1, %xmm0
+; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [5,3,2,7]
+; CHECK-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>
@@ -1169,11 +1177,12 @@ define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask3(ptr %vp) {
define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {
; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask3:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm2
-; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [5,1,2,7]
-; CHECK-NEXT: vpermi2d 16(%rdi), %xmm2, %xmm3
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [5,3,2,7]
; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k1
-; CHECK-NEXT: vmovdqa32 %xmm3, %xmm0 {%k1}
+; CHECK-NEXT: vpermd (%rdi), %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>
@@ -1185,11 +1194,11 @@ define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %
define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %mask) {
; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask3:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm2
-; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [5,1,2,7]
+; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [5,3,2,7]
; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k1
-; CHECK-NEXT: vpermi2d 16(%rdi), %xmm2, %xmm1 {%k1} {z}
-; CHECK-NEXT: vmovdqa %xmm1, %xmm0
+; CHECK-NEXT: vpermd (%rdi), %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <8 x i32>, ptr %vp
%shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>
@@ -1477,8 +1486,9 @@ define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mask3(<16 x i32> %vec, <4 x
define <8 x i32> @test_16xi32_to_8xi32_perm_mem_mask0(ptr %vp) {
; CHECK-LABEL: test_16xi32_to_8xi32_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm0 = [7,0,6,0,1,2,4,4]
-; CHECK-NEXT: vpermps 32(%rdi), %ymm0, %ymm0
+; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm0 = [15,8,14,8,9,10,12,12]
+; CHECK-NEXT: vpermps (%rdi), %zmm0, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <16 x i32>, ptr %vp
%res = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 15, i32 8, i32 14, i32 8, i32 9, i32 10, i32 12, i32 12>
@@ -1487,9 +1497,11 @@ define <8 x i32> @test_16xi32_to_8xi32_perm_mem_mask0(ptr %vp) {
define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mem_mask0(ptr %vp, <8 x i32> %vec2, <8 x i32> %mask) {
; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [7,0,6,0,1,2,4,4]
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [15,8,14,8,9,10,12,12]
; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k1
-; CHECK-NEXT: vpermd 32(%rdi), %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: vpermd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <16 x i32>, ptr %vp
%shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 15, i32 8, i32 14, i32 8, i32 9, i32 10, i32 12, i32 12>
@@ -1501,9 +1513,10 @@ define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mem_mask0(ptr %vp, <8 x i32>
define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mem_mask0(ptr %vp, <8 x i32> %mask) {
; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [7,0,6,0,1,2,4,4]
+; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [15,8,14,8,9,10,12,12]
; CHECK-NEXT: vptestnmd %ymm0, %ymm0, %k1
-; CHECK-NEXT: vpermd 32(%rdi), %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: vpermd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <16 x i32>, ptr %vp
%shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 15, i32 8, i32 14, i32 8, i32 9, i32 10, i32 12, i32 12>
@@ -1864,8 +1877,9 @@ define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mask1(<4 x i64> %vec, <2 x i
define <2 x i64> @test_4xi64_to_2xi64_perm_mem_mask0(ptr %vp) {
; CHECK-LABEL: test_4xi64_to_2xi64_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps (%rdi), %xmm0
-; CHECK-NEXT: vunpckhpd 16(%rdi), %xmm0, %xmm0 # xmm0 = xmm0[1],mem[1]
+; CHECK-NEXT: vpermpd $237, (%rdi), %ymm0 # ymm0 = mem[1,3,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%res = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
@@ -1874,9 +1888,11 @@ define <2 x i64> @test_4xi64_to_2xi64_perm_mem_mask0(ptr %vp) {
define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %vec2, <2 x i64> %mask) {
; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqa (%rdi), %xmm2
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k1
-; CHECK-NEXT: vpunpckhqdq 16(%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[1],mem[1]
+; CHECK-NEXT: vpermq $237, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[1,3,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
@@ -1888,9 +1904,10 @@ define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %
define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %mask) {
; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqa (%rdi), %xmm1
; CHECK-NEXT: vptestnmq %xmm0, %xmm0, %k1
-; CHECK-NEXT: vpunpckhqdq 16(%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[1],mem[1]
+; CHECK-NEXT: vpermq $237, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[1,3,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
@@ -1902,10 +1919,11 @@ define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64>
define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %vec2, <2 x i64> %mask) {
; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mem_mask1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqa 16(%rdi), %xmm2
-; CHECK-NEXT: vpblendd $12, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k1
-; CHECK-NEXT: vmovdqa64 %xmm2, %xmm0 {%k1}
+; CHECK-NEXT: vpermq $230, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 1>
@@ -1917,10 +1935,10 @@ define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %
define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %mask) {
; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mem_mask1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovdqa 16(%rdi), %xmm1
-; CHECK-NEXT: vpblendd $12, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[0,1],mem[2,3]
; CHECK-NEXT: vptestnmq %xmm0, %xmm0, %k1
-; CHECK-NEXT: vmovdqa64 %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT: vpermq $230, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x i64>, ptr %vp
%shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 1>
@@ -1932,8 +1950,9 @@ define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64>
define <4 x i64> @test_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec) {
; CHECK-LABEL: test_8xi64_to_4xi64_perm_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm0
-; CHECK-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,3,2,1]
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [6,7,6,5]
+; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 7, i32 6, i32 5>
ret <4 x i64> %res
@@ -1941,9 +1960,10 @@ define <4 x i64> @test_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec) {
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,7,6,5]
; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,3,2,1]
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
; CHECK-NEXT: vmovdqa %ymm1, %ymm0
; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 7, i32 6, i32 5>
@@ -1955,9 +1975,10 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec, <4 x i64
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec, <4 x i64> %mask) {
; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,7,6,5]
; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,2,1]
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 7, i32 6, i32 5>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -1965,23 +1986,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec, <4 x i
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask1(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,4,6,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,2,1]
-; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,4,6,1]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 4, i32 6, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -1989,44 +2001,27 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask1(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask1(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,4,6,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,2,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,4,6,1]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 4, i32 6, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask2(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,3,6,3]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,3,2,3]
-; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,3,6,3]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 3, i32 6, i32 3>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2034,61 +2029,37 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask2(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask2(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,3,6,3]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,2,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,3,6,3]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 3, i32 6, i32 3>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
ret <4 x i64> %res
}
define <4 x i64> @test_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec) {
-; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [6,0,0,7]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm0
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; CHECK-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,0,0,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_8xi64_to_4xi64_perm_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [6,0,0,7]
+; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,0,0,7]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,0,3]
-; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,0,0,7]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2096,44 +2067,27 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,0,0,7]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,0,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,0,0,7]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask4(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask4:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,7,7,5]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask4:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,3,3,1]
-; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,7,7,5]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 3, i32 7, i32 7, i32 5>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2141,21 +2095,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask4(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask4(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask4:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,7,7,5]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask4:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm2[1],ymm0[3],ymm2[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,3,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,7,7,5]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 3, i32 7, i32 7, i32 5>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
@@ -2190,40 +2136,24 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask5(<8 x i64> %vec, <4 x i
ret <4 x i64> %res
}
define <4 x i64> @test_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec) {
-; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,6,5,3]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm0
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[3,2,1,3]
-; CHECK-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_8xi64_to_4xi64_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,6,5,3]
+; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,6,5,3]
-; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm3 = ymm3[3,2,1,3]
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpblendmq %ymm0, %ymm1, %ymm0 {%k1}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,6,5,3]
+; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqa %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2231,22 +2161,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i64
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,6,5,3]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm2 = ymm2[3,2,1,3]
-; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vmovdqa64 %ymm0, %ymm0 {%k1} {z}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,6,5,3]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
@@ -2383,22 +2304,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask0(ptr %vp, <4 x i64>
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,7,6,0]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd $15, (%rdi), %ymm2, %ymm2 # ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[0,3,2,0]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,7,6,0]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 7, i32 6, i32 0>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2407,21 +2320,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,6,0]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpblendd $15, (%rdi), %ymm1, %ymm1 # ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[0,3,2,0]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,6,0]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 7, i32 6, i32 0>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2430,23 +2335,15 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64>
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,1,1,5]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm2, %ymm2 # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,1,1,0]
-; CHECK-FAST-PERLANE-NEXT: retq
- %vec = load <8 x i64>, ptr %vp
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,1,1,5]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
+ %vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 1, i32 1, i32 5>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
@@ -2454,21 +2351,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,1,1,5]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm1, %ymm1 # ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,1,1,0]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,1,1,5]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 1, i32 1, i32 5>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2477,40 +2366,25 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64>
}
define <4 x i64> @test_8xi64_to_4xi64_perm_mem_mask3(ptr %vp) {
-; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [7,0,0,2]
-; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm0, %zmm0
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm0
-; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm0, %ymm0 # ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,1,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_8xi64_to_4xi64_perm_mem_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm0 = [7,0,0,2]
+; CHECK-NEXT: vpermpd (%rdi), %zmm0, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>
ret <4 x i64> %res
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask3(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,0,0,2]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm2, %ymm2 # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,1,1,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,0,0,2]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2519,21 +2393,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask3(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask3(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,0,0,2]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask3:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm1, %ymm1 # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,1,1,3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,0,0,2]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2573,22 +2439,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask4(ptr %vp, <4 x i64>
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask5(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask5:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,2,7,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask5:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpblendd $192, 32(%rdi), %ymm2, %ymm2 # ymm2 = ymm2[0,1,2,3,4,5],mem[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[0,2,3,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,2,7,1]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 7, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2597,21 +2455,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask5(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask5(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask5:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,7,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask5:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpblendd $192, 32(%rdi), %ymm1, %ymm1 # ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[0,2,3,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,7,1]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 7, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2662,22 +2512,14 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask6(ptr %vp, <4 x i64>
}
define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask7(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask7:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,7,5,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask7:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm2, %ymm2 # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,2,0,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,7,5,1]
+; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 7, i32 5, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2686,21 +2528,13 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask7(ptr %vp, <4 x i64> %
}
define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask7(ptr %vp, <4 x i64> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask7:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,7,5,1]
-; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask7:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm1, %ymm1 # ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
-; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,2,0,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,7,5,1]
+; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k1
+; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x i64>, ptr %vp
%shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 7, i32 5, i32 1>
%cmp = icmp eq <4 x i64> %mask, zeroinitializer
@@ -2804,25 +2638,44 @@ define <2 x i64> @test_masked_z_8xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64>
}
define <4 x float> @test_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec) {
-; CHECK-LABEL: test_8xfloat_to_4xfloat_perm_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm1[0,1]
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,3,4,5,0,0,0,0]
+; CHECK-FAST-NEXT: vpermps %ymm0, %ymm1, %ymm0
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm1
+; CHECK-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm1[0,1]
+; CHECK-FAST-PERLANE-NEXT: vzeroupper
+; CHECK-FAST-PERLANE-NEXT: retq
%res = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>
ret <4 x float> %res
}
define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {
-; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm4
-; CHECK-NEXT: vcmpeqps %xmm4, %xmm2, %k1
-; CHECK-NEXT: vshufps {{.*#+}} xmm1 {%k1} = xmm0[0,3],xmm3[0,1]
-; CHECK-NEXT: vmovaps %xmm1, %xmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,3,4,5,0,0,0,0]
+; CHECK-FAST-NEXT: vxorps %xmm4, %xmm4, %xmm4
+; CHECK-FAST-NEXT: vcmpeqps %xmm4, %xmm2, %k1
+; CHECK-FAST-NEXT: vpermps %ymm0, %ymm3, %ymm1 {%k1}
+; CHECK-FAST-NEXT: vmovaps %xmm1, %xmm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm3
+; CHECK-FAST-PERLANE-NEXT: vxorps %xmm4, %xmm4, %xmm4
+; CHECK-FAST-PERLANE-NEXT: vcmpeqps %xmm4, %xmm2, %k1
+; CHECK-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm1 {%k1} = xmm0[0,3],xmm3[0,1]
+; CHECK-FAST-PERLANE-NEXT: vmovaps %xmm1, %xmm0
+; CHECK-FAST-PERLANE-NEXT: vzeroupper
+; CHECK-FAST-PERLANE-NEXT: retq
%shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>
%cmp = fcmp oeq <4 x float> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec2
@@ -2830,14 +2683,24 @@ define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec,
}
define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec, <4 x float> %mask) {
-; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask0:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm2
-; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k1
-; CHECK-NEXT: vshufps {{.*#+}} xmm0 {%k1} {z} = xmm0[0,3],xmm2[0,1]
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; CHECK-FAST-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST: # %bb.0:
+; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,3,4,5,0,0,0,0]
+; CHECK-FAST-NEXT: vxorps %xmm3, %xmm3, %xmm3
+; CHECK-FAST-NEXT: vcmpeqps %xmm3, %xmm1, %k1
+; CHECK-FAST-NEXT: vpermps %ymm0, %ymm2, %ymm0 {%k1} {z}
+; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-FAST-NEXT: vzeroupper
+; CHECK-FAST-NEXT: retq
+;
+; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask0:
+; CHECK-FAST-PERLANE: # %bb.0:
+; CHECK-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm2
+; CHECK-FAST-PERLANE-NEXT: vxorps %xmm3, %xmm3, %xmm3
+; CHECK-FAST-PERLANE-NEXT: vcmpeqps %xmm3, %xmm1, %k1
+; CHECK-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 {%k1} {z} = xmm0[0,3],xmm2[0,1]
+; CHECK-FAST-PERLANE-NEXT: vzeroupper
+; CHECK-FAST-PERLANE-NEXT: retq
%shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>
%cmp = fcmp oeq <4 x float> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
@@ -3505,25 +3368,15 @@ define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mem_mask1(ptr %vp, <8
}
define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mem_mask2(ptr %vp, <8 x float> %vec2, <8 x float> %mask) {
-; CHECK-FAST-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,13,10,11,10,0,0,9]
-; CHECK-FAST-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqps %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm2
-; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,13,10,11,10,0,0,9]
-; CHECK-FAST-PERLANE-NEXT: vpermi2ps 32(%rdi), %ymm2, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; CHECK-FAST-PERLANE-NEXT: vcmpeqps %ymm2, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vmovaps %ymm3, %ymm0 {%k1}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,13,10,11,10,0,0,9]
+; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <16 x float>, ptr %vp
%shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 1, i32 13, i32 10, i32 11, i32 10, i32 0, i32 0, i32 9>
%cmp = fcmp oeq <8 x float> %mask, zeroinitializer
@@ -3532,24 +3385,14 @@ define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mem_mask2(ptr %vp, <8 x
}
define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2(ptr %vp, <8 x float> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,10,11,10,0,0,9]
-; CHECK-FAST-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; CHECK-FAST-NEXT: vcmpeqps %ymm2, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm2
-; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,10,11,10,0,0,9]
-; CHECK-FAST-PERLANE-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqps %ymm3, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermi2ps 32(%rdi), %ymm2, %ymm1 {%k1} {z}
-; CHECK-FAST-PERLANE-NEXT: vmovaps %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,10,11,10,0,0,9]
+; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqps %ymm2, %ymm0, %k1
+; CHECK-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <16 x float>, ptr %vp
%shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 1, i32 13, i32 10, i32 11, i32 10, i32 0, i32 0, i32 9>
%cmp = fcmp oeq <8 x float> %mask, zeroinitializer
@@ -3839,8 +3682,9 @@ define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mask1(<4 x double>
define <2 x double> @test_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp) {
; CHECK-LABEL: test_4xdouble_to_2xdouble_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovaps (%rdi), %xmm0
-; CHECK-NEXT: vblendps $3, 16(%rdi), %xmm0, %xmm0 # xmm0 = mem[0,1],xmm0[2,3]
+; CHECK-NEXT: vpermpd $230, (%rdi), %ymm0 # ymm0 = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%res = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>
@@ -3849,11 +3693,12 @@ define <2 x double> @test_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp) {
define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2 x double> %vec2, <2 x double> %mask) {
; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %xmm2
-; CHECK-NEXT: vblendpd $1, 16(%rdi), %xmm2, %xmm2 # xmm2 = mem[0],xmm2[1]
-; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k1
-; CHECK-NEXT: vmovapd %xmm2, %xmm0 {%k1}
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqpd %xmm2, %xmm1, %k1
+; CHECK-NEXT: vpermpd $230, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>
@@ -3865,11 +3710,11 @@ define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2
define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2 x double> %mask) {
; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mem_mask0:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %xmm1
-; CHECK-NEXT: vblendpd $1, 16(%rdi), %xmm1, %xmm1 # xmm1 = mem[0],xmm1[1]
-; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT: vcmpeqpd %xmm2, %xmm0, %k1
-; CHECK-NEXT: vmovapd %xmm1, %xmm0 {%k1} {z}
+; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vcmpeqpd %xmm1, %xmm0, %k1
+; CHECK-NEXT: vpermpd $230, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[2,1,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>
@@ -3881,10 +3726,12 @@ define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp,
define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2 x double> %vec2, <2 x double> %mask) {
; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mem_mask1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd 16(%rdi), %xmm2
-; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k1
-; CHECK-NEXT: vunpcklpd (%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[0],mem[0]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqpd %xmm2, %xmm1, %k1
+; CHECK-NEXT: vpermpd $226, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[2,0,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>
@@ -3896,10 +3743,11 @@ define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2
define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2 x double> %mask) {
; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mem_mask1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd 16(%rdi), %xmm1
-; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT: vcmpeqpd %xmm2, %xmm0, %k1
-; CHECK-NEXT: vunpcklpd (%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[0],mem[0]
+; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vcmpeqpd %xmm1, %xmm0, %k1
+; CHECK-NEXT: vpermpd $226, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[2,0,2,3]
+; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%vec = load <4 x double>, ptr %vp
%shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>
@@ -4045,25 +3893,15 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask3(<8 x double>
ret <4 x double> %res
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask4(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm3 = [1,5]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm0
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[0,0,1,1]
-; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $2, %zmm0, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[0,0,1,1]
-; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,1,5,5]
+; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm4
+; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovapd %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 1, i32 5, i32 5>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
@@ -4071,23 +3909,14 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask4(<8 x double> %v
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask4(<8 x double> %vec, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm2 = [1,5]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0
-; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,0,1,1]
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $2, %zmm0, %xmm2
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm2, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,0,1,1]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,1,5,5]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 1, i32 5, i32 5>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
@@ -4124,42 +3953,25 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask5(<8 x double>
ret <4 x double> %res
}
define <4 x double> @test_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec) {
-; CHECK-FAST-LABEL: test_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [5,0,7,0]
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm0
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm0
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [5,0,7,0]
+; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
ret <4 x double> %res
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [5,0,7,0]
-; CHECK-FAST-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm0
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm1 {%k1} = ymm3[1],ymm0[1],ymm3[3],ymm0[3]
-; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [5,0,7,0]
+; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm4
+; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovapd %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
@@ -4167,48 +3979,29 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %v
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,0,7,0]
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm0
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm2[1],ymm0[1],ymm2[3],ymm0[3]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,0,7,0]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
ret <4 x double> %res
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask7(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask7:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,5,0,6]
-; CHECK-FAST-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
-; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask7:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm3
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,3]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm1 {%k1} = ymm0[1],ymm3[1],ymm0[2],ymm3[2]
-; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,5,0,6]
+; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm4
+; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}
+; CHECK-NEXT: vmovapd %ymm1, %ymm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 5, i32 0, i32 6>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
@@ -4216,23 +4009,14 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask7(<8 x double> %v
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask7(<8 x double> %vec, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask7:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,5,0,6]
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask7:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,3]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm2[1],ymm0[2],ymm2[2]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,5,0,6]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 5, i32 0, i32 6>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
%res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
@@ -4384,23 +4168,15 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask0(ptr %vp,
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vbroadcastsd 32(%rdi), %ymm2
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,0,6,2]
-; CHECK-FAST-NEXT: vpermi2pd (%rdi), %ymm2, %ymm3
-; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm1, %k1
-; CHECK-FAST-NEXT: vmovapd %ymm3, %ymm0 {%k1}
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vpermpd $236, (%rdi), %ymm2 # ymm2 = mem[0,3,2,3]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd $1, 32(%rdi){1to4}, %ymm2, %ymm0 {%k1}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,4,2,4]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 4, i32 2, i32 4>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -4409,23 +4185,14 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp, <4
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vbroadcastsd 32(%rdi), %ymm2
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,0,6,2]
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermi2pd (%rdi), %ymm2, %ymm1 {%k1} {z}
-; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vpermpd $236, (%rdi), %ymm1 # ymm1 = mem[0,3,2,3]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd $1, 32(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [3,4,2,4]
+; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
+; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 4, i32 2, i32 4>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -4434,24 +4201,15 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp,
}
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask2(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,2,3,4]
-; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovapd (%rdi), %ymm2
-; CHECK-FAST-PERLANE-NEXT: vperm2f128 $33, 32(%rdi), %ymm2, %ymm3 # ymm3 = ymm2[2,3],mem[0,1]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm1, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} = ymm2[1],ymm3[0],ymm2[3],ymm3[2]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,2,3,4]
+; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
+; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -4460,23 +4218,14 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask2(ptr %vp, <4
}
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2(ptr %vp, <4 x double> %mask) {
-; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2:
-; CHECK-FAST: # %bb.0:
-; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,2,3,4]
-; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
-; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
-; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; CHECK-FAST-NEXT: retq
-;
-; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2:
-; CHECK-FAST-PERLANE: # %bb.0:
-; CHECK-FAST-PERLANE-NEXT: vmovapd (%rdi), %ymm1
-; CHECK-FAST-PERLANE-NEXT: vperm2f128 $33, 32(%rdi), %ymm1, %ymm2 # ymm2 = ymm1[2,3],mem[0,1]
-; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm0, %k1
-; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} {z} = ymm1[1],ymm2[0],ymm1[3],ymm2[2]
-; CHECK-FAST-PERLANE-NEXT: retq
+; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,2,3,4]
+; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
+; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
%cmp = fcmp oeq <4 x double> %mask, zeroinitializer
@@ -4564,11 +4313,12 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask4(ptr %vp,
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask5(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {
; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask5:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %ymm2
-; CHECK-NEXT: vperm2f128 $33, 32(%rdi), %ymm2, %ymm2 # ymm2 = ymm2[2,3],mem[0,1]
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [2,5,5,5]
; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-NEXT: vshufpd $14, 40(%rdi){1to4}, %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 5, i32 5, i32 5>
@@ -4580,11 +4330,11 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask5(ptr %vp, <4
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask5(ptr %vp, <4 x double> %mask) {
; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask5:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %ymm1
-; CHECK-NEXT: vperm2f128 $33, 32(%rdi), %ymm1, %ymm1 # ymm1 = ymm1[2,3],mem[0,1]
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [2,5,5,5]
; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
-; CHECK-NEXT: vshufpd $14, 40(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 5, i32 5, i32 5>
@@ -4640,10 +4390,12 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask6(ptr %vp,
define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask7(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {
; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask7:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %ymm2
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,2,5]
; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm3
; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k1
-; CHECK-NEXT: vunpcklpd 40(%rdi){1to4}, %ymm2, %ymm0 {%k1}
+; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 5>
@@ -4655,10 +4407,11 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask7(ptr %vp, <4
define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask7(ptr %vp, <4 x double> %mask) {
; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask7:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovapd (%rdi), %ymm1
+; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,5,2,5]
; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k1
-; CHECK-NEXT: vunpcklpd 40(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}
+; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}
+; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; CHECK-NEXT: retq
%vec = load <8 x double>, ptr %vp
%shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 5>
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
index 873083a0703da..0095113d44b1a 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
@@ -191,7 +191,7 @@ define i32 @PR37890_v8i32(<8 x i32> %a) {
; AVX1-FAST-LABEL: PR37890_v8i32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -278,7 +278,7 @@ define i16 @PR37890_v16i16(<16 x i16> %a) {
; AVX1-FAST-LABEL: PR37890_v16i16:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
@@ -369,11 +369,11 @@ define i32 @PR37890_v16i32(<16 x i32> %a) {
;
; AVX1-FAST-LABEL: PR37890_v16i32:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm2
-; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm2
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll b/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
index 15cbd013981eb..b6604b8f4eaee 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
@@ -99,7 +99,7 @@ define double @PR37890_v4f64(<4 x double> %a) {
; AVX1-FAST-LABEL: PR37890_v4f64:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -164,7 +164,7 @@ define float @PR37890_v8f32(<8 x float> %a) {
; AVX1-FAST-LABEL: PR37890_v8f32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
@@ -235,7 +235,7 @@ define double @PR37890_v8f64(<8 x double> %a) {
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -314,7 +314,7 @@ define float @PR37890_v16f32(<16 x float> %a) {
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 7c748439b1730..3eb5d73f78ec0 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -1827,10 +1827,27 @@ define <4 x i32> @pmaddwd_8(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_8:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%b = sext <8 x i16> %B to <8 x i32>
%m = mul nsw <8 x i32> %a, %b
@@ -1846,10 +1863,27 @@ define <4 x i32> @pmaddwd_8_swapped(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_8_swapped:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_8_swapped:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_8_swapped:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%b = sext <8 x i16> %B to <8 x i32>
%m = mul nsw <8 x i32> %a, %b
@@ -1921,10 +1955,21 @@ define <8 x i32> @pmaddwd_16(<16 x i16> %A, <16 x i16> %B) {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddwd_16:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddwd_16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: pmaddwd_16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,3,5,7,9,11,13,15]
+; AVX512-NEXT: vpermd %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT: retq
%a = sext <16 x i16> %A to <16 x i32>
%b = sext <16 x i16> %B to <16 x i32>
%m = mul nsw <16 x i32> %a, %b
@@ -1991,10 +2036,24 @@ define <4 x i32> @pmaddwd_const(<8 x i16> %A) {
; SSE2-NEXT: pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [32767,32768,0,0,1,7,42,32]
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_const:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32767,32768,0,0,1,7,42,32]
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_const:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32767,0,32768,0,0,0,0,0]
+; AVX1-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [1,0,7,0,42,0,32,0]
+; AVX1-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_const:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX256-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [32767,0,32768,0,0,0,0,0,1,0,7,0,42,0,32,0]
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%m = mul nsw <8 x i32> %a, <i32 32767, i32 -32768, i32 0, i32 0, i32 1, i32 7, i32 42, i32 32>
%odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -2098,10 +2157,29 @@ define <4 x i32> @jumbled_indices4(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: jumbled_indices4:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: jumbled_indices4:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,3]
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: jumbled_indices4:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,3]
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%exta = sext <8 x i16> %A to <8 x i32>
%extb = sext <8 x i16> %B to <8 x i32>
%m = mul <8 x i32> %exta, %extb
@@ -2127,10 +2205,22 @@ define <8 x i32> @jumbled_indices8(<16 x i16> %A, <16 x i16> %B) {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: jumbled_indices8:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: jumbled_indices8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: jumbled_indices8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,7,4,11,8,15,12]
+; AVX512-NEXT: vpermd %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,3,6,5,10,9,14,13]
+; AVX512-NEXT: vpermd %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT: retq
%exta = sext <16 x i16> %A to <16 x i32>
%extb = sext <16 x i16> %B to <16 x i32>
%m = mul <16 x i32> %exta, %extb
@@ -2306,10 +2396,44 @@ define <4 x i32> @pmaddwd_128(ptr %Aptr, ptr %Bptr) {
define <8 x i32> @pmaddwd_256(ptr %Aptr, ptr %Bptr) {
; SSE2-LABEL: pmaddwd_256:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: movdqa 16(%rdi), %xmm1
-; SSE2-NEXT: pmaddwd (%rsi), %xmm0
-; SSE2-NEXT: pmaddwd 16(%rsi), %xmm1
+; SSE2-NEXT: movdqa (%rdi), %xmm1
+; SSE2-NEXT: movdqa 16(%rdi), %xmm2
+; SSE2-NEXT: movdqa (%rsi), %xmm0
+; SSE2-NEXT: movdqa 16(%rsi), %xmm3
+; SSE2-NEXT: movdqa %xmm2, %xmm4
+; SSE2-NEXT: pslld $16, %xmm4
+; SSE2-NEXT: psrad $16, %xmm4
+; SSE2-NEXT: movdqa %xmm1, %xmm5
+; SSE2-NEXT: pslld $16, %xmm5
+; SSE2-NEXT: psrad $16, %xmm5
+; SSE2-NEXT: packssdw %xmm4, %xmm5
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: psrad $16, %xmm1
+; SSE2-NEXT: packssdw %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: pslld $16, %xmm4
+; SSE2-NEXT: psrad $16, %xmm4
+; SSE2-NEXT: packssdw %xmm2, %xmm4
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: packssdw %xmm3, %xmm0
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pmulhw %xmm4, %xmm2
+; SSE2-NEXT: pmullw %xmm5, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: pmulhw %xmm0, %xmm2
+; SSE2-NEXT: pmullw %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm4, %xmm0
; SSE2-NEXT: retq
;
; AVX1-LABEL: pmaddwd_256:
@@ -2321,11 +2445,52 @@ define <8 x i32> @pmaddwd_256(ptr %Aptr, ptr %Bptr) {
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddwd_256:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vmovdqa (%rdi), %ymm0
-; AVX256-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddwd_256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rsi), %xmm1
+; AVX2-NEXT: vmovdqa 16(%rsi), %xmm4
+; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm2[1],xmm4[2],xmm2[3],xmm4[4],xmm2[5],xmm4[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm5, %xmm2, %xmm2
+; AVX2-NEXT: vpsrld $16, %xmm4, %xmm4
+; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
+; AVX2-NEXT: vpmovsxwd %xmm3, %ymm3
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX2-NEXT: vpmulld %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: pmaddwd_256:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512-NEXT: vpmovdw %zmm0, %ymm2
+; AVX512-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX512-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512-NEXT: vpmovdw %zmm1, %ymm3
+; AVX512-NEXT: vpsrld $16, %ymm1, %ymm1
+; AVX512-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX512-NEXT: vpmovsxwd %xmm3, %ymm3
+; AVX512-NEXT: vpmulld %ymm3, %ymm2, %ymm2
+; AVX512-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
%A = load <16 x i16>, ptr %Aptr
%B = load <16 x i16>, ptr %Bptr
%A_even = shufflevector <16 x i16> %A, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -2373,23 +2538,84 @@ define <16 x i32> @pmaddwd_512(ptr %Aptr, ptr %Bptr) {
; AVX2: # %bb.0:
; AVX2-NEXT: vmovdqa (%rdi), %ymm0
; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX2-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX2-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
+; AVX2-NEXT: vmovdqa (%rsi), %ymm2
+; AVX2-NEXT: vmovdqa 32(%rsi), %ymm3
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm4 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vshufps {{.*#+}} ymm4 = ymm5[0,2],ymm4[0,2],ymm5[4,6],ymm4[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm5, %ymm0, %ymm0
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm1 = ymm3[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm6 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm6[0,2],ymm1[0,2],ymm6[4,6],ymm1[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpshufb %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vpshufb %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT: vpmovsxwd %xmm4, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm4
+; AVX2-NEXT: vpmovsxwd %xmm4, %ymm4
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm5
+; AVX2-NEXT: vpmulld %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT: vpmulld %ymm1, %ymm4, %ymm1
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm4
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm5
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm0
+; AVX2-NEXT: vpmulld %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX2-NEXT: vpmulld %ymm2, %ymm5, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
; AVX2-NEXT: retq
;
; AVX512F-LABEL: pmaddwd_512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT: vmovdqa64 (%rsi), %zmm1
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm2
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm3
+; AVX512F-NEXT: vpsrld $16, %zmm1, %zmm1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512F-NEXT: vpmulld %zmm3, %zmm2, %zmm2
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddwd_512:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm1
+; AVX512BW-NEXT: vpmovdw %zmm0, %ymm2
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-NEXT: vpermw %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT: vpmovdw %zmm1, %ymm4
+; AVX512BW-NEXT: vpermw %zmm1, %zmm3, %zmm1
+; AVX512BW-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512BW-NEXT: vpmovsxwd %ymm4, %zmm3
+; AVX512BW-NEXT: vpmulld %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512BW-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512BW-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: retq
%A = load <32 x i16>, ptr %Aptr
%B = load <32 x i16>, ptr %Bptr
@@ -2475,24 +2701,69 @@ define <32 x i32> @pmaddwd_1024(ptr %Aptr, ptr %Bptr) {
;
; AVX512F-LABEL: pmaddwd_1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm2
-; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm3
-; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpmaddwd 96(%rsi), %ymm3, %ymm1
-; AVX512F-NEXT: vpmaddwd 64(%rsi), %ymm2, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT: vmovdqa64 64(%rdi), %zmm1
+; AVX512F-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512F-NEXT: vmovdqa64 64(%rsi), %zmm3
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm4
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm5
+; AVX512F-NEXT: vpsrld $16, %zmm1, %zmm1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: vpmovdw %zmm3, %ymm6
+; AVX512F-NEXT: vpmovdw %zmm2, %ymm7
+; AVX512F-NEXT: vpsrld $16, %zmm3, %zmm3
+; AVX512F-NEXT: vpmovdw %zmm3, %ymm3
+; AVX512F-NEXT: vpsrld $16, %zmm2, %zmm2
+; AVX512F-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512F-NEXT: vpmovsxwd %ymm5, %zmm5
+; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm4
+; AVX512F-NEXT: vpmovsxwd %ymm7, %zmm7
+; AVX512F-NEXT: vpmulld %zmm7, %zmm5, %zmm5
+; AVX512F-NEXT: vpmovsxwd %ymm6, %zmm6
+; AVX512F-NEXT: vpmulld %zmm6, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512F-NEXT: vpmulld %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm5, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm2
+; AVX512F-NEXT: vpmulld %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vpaddd %zmm1, %zmm4, %zmm1
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddwd_1024:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
-; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmaddwd 64(%rsi), %zmm1, %zmm1
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm3
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
+; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm5
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm6, %zmm0
+; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm2
+; AVX512BW-NEXT: vpmovsxwd %ymm5, %zmm1
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm5, %ymm3
+; AVX512BW-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512BW-NEXT: vpmovsxwd %ymm4, %zmm5
+; AVX512BW-NEXT: vpmulld %zmm5, %zmm1, %zmm1
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm4, %ymm4
+; AVX512BW-NEXT: vpmovsxwd %ymm4, %zmm4
+; AVX512BW-NEXT: vpmulld %zmm4, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovsxwd %ymm0, %zmm4
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512BW-NEXT: vpmovsxwd %ymm0, %zmm5
+; AVX512BW-NEXT: vpmovsxwd %ymm2, %zmm0
+; AVX512BW-NEXT: vpmulld %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm2, %ymm1
+; AVX512BW-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512BW-NEXT: vpmulld %zmm1, %zmm5, %zmm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm3, %zmm1
; AVX512BW-NEXT: retq
%A = load <64 x i16>, ptr %Aptr
%B = load <64 x i16>, ptr %Bptr
diff --git a/llvm/test/CodeGen/X86/matrix-multiply.ll b/llvm/test/CodeGen/X86/matrix-multiply.ll
index f38b769fe4987..a39c11546cf8c 100644
--- a/llvm/test/CodeGen/X86/matrix-multiply.ll
+++ b/llvm/test/CodeGen/X86/matrix-multiply.ll
@@ -119,35 +119,42 @@ define <4 x double> @test_mul2x2_f64(<4 x double> %a0, <4 x double> %a1) nounwin
;
; AVX1-LABEL: test_mul2x2_f64:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3]
-; AVX1-NEXT: vshufpd {{.*#+}} ymm3 = ymm1[1,1,3,3]
-; AVX1-NEXT: vmulpd %ymm3, %ymm2, %ymm2
-; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX1-NEXT: vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2]
+; AVX1-NEXT: vmovddup {{.*#+}} xmm2 = xmm1[0,0]
+; AVX1-NEXT: vmulpd %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = xmm3[0,0]
+; AVX1-NEXT: vmulpd %xmm3, %xmm0, %xmm3
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX1-NEXT: vshufpd {{.*#+}} ymm1 = ymm1[1,1,3,3]
; AVX1-NEXT: vmulpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT: vaddpd %ymm2, %ymm0, %ymm0
+; AVX1-NEXT: vaddpd %ymm0, %ymm2, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: test_mul2x2_f64:
; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: vshufpd {{.*#+}} ymm2 = ymm1[1,1,3,3]
-; AVX2-NEXT: vpermpd {{.*#+}} ymm3 = ymm0[2,3,2,3]
-; AVX2-NEXT: vmulpd %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2]
-; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-NEXT: vmovddup {{.*#+}} xmm2 = xmm1[0,0]
+; AVX2-NEXT: vmulpd %xmm2, %xmm0, %xmm2
+; AVX2-NEXT: vpermpd {{.*#+}} ymm3 = ymm1[2,2,2,2]
+; AVX2-NEXT: vmulpd %xmm3, %xmm0, %xmm3
+; AVX2-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX2-NEXT: vshufpd {{.*#+}} ymm1 = ymm1[1,1,3,3]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,3,2,3]
; AVX2-NEXT: vmulpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vaddpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vaddpd %ymm0, %ymm2, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_mul2x2_f64:
; AVX512: # %bb.0: # %entry
-; AVX512-NEXT: vshufpd {{.*#+}} ymm2 = ymm1[1,1,3,3]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm3 = ymm0[2,3,2,3]
-; AVX512-NEXT: vmulpd %ymm2, %ymm3, %ymm2
-; AVX512-NEXT: vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512-NEXT: vmovddup {{.*#+}} xmm2 = xmm1[0,0]
+; AVX512-NEXT: vmulpd %xmm2, %xmm0, %xmm2
+; AVX512-NEXT: vpermpd {{.*#+}} ymm3 = ymm1[2,2,2,2]
+; AVX512-NEXT: vmulpd %xmm3, %xmm0, %xmm3
+; AVX512-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT: vshufpd {{.*#+}} ymm1 = ymm1[1,1,3,3]
+; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,3,2,3]
; AVX512-NEXT: vmulpd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vaddpd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vaddpd %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
entry:
%split = shufflevector <4 x double> %a0, <4 x double> poison, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/CodeGen/X86/pclmulqdq.ll b/llvm/test/CodeGen/X86/pclmulqdq.ll
index 92df2d9987dbe..a95e65ef0b4f3 100644
--- a/llvm/test/CodeGen/X86/pclmulqdq.ll
+++ b/llvm/test/CodeGen/X86/pclmulqdq.ll
@@ -163,10 +163,19 @@ define <4 x i64> @pclmul256_hi_hi_vector(<4 x i64> %a0, <4 x i64> %a1) {
; AVX-PCLMUL-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX-PCLMUL-NEXT: retq
;
-; AVX-VPCLMULQDQ-LABEL: pclmul256_hi_hi_vector:
-; AVX-VPCLMULQDQ: # %bb.0:
-; AVX-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm0
-; AVX-VPCLMULQDQ-NEXT: retq
+; AVX2-VPCLMULQDQ-LABEL: pclmul256_hi_hi_vector:
+; AVX2-VPCLMULQDQ: # %bb.0:
+; AVX2-VPCLMULQDQ-NEXT: vpclmulqdq $17, %ymm1, %ymm0, %ymm0
+; AVX2-VPCLMULQDQ-NEXT: retq
+;
+; AVX512-VPCLMULQDQ-LABEL: pclmul256_hi_hi_vector:
+; AVX512-VPCLMULQDQ: # %bb.0:
+; AVX512-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,2,3]
+; AVX512-VPCLMULQDQ-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm2
+; AVX512-VPCLMULQDQ-NEXT: vpclmulqdq $0, %xmm1, %xmm0, %xmm0
+; AVX512-VPCLMULQDQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512-VPCLMULQDQ-NEXT: retq
%s0 = shufflevector <4 x i64> %a0, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
%s1 = shufflevector <4 x i64> %a1, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
%x0 = zext <2 x i64> %s0 to <2 x i128>
@@ -365,12 +374,9 @@ define <8 x i64> @pclmul512_lo_lo(<8 x i64> %a0, <8 x i64> %a1) {
;
; AVX512-VPCLMULQDQ-LABEL: pclmul512_lo_lo:
; AVX512-VPCLMULQDQ: # %bb.0:
-; AVX512-VPCLMULQDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; AVX512-VPCLMULQDQ-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
-; AVX512-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512-VPCLMULQDQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512-VPCLMULQDQ-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
-; AVX512-VPCLMULQDQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512-VPCLMULQDQ-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,2,4,6]
+; AVX512-VPCLMULQDQ-NEXT: vpermq %zmm0, %zmm2, %zmm0
+; AVX512-VPCLMULQDQ-NEXT: vpermq %zmm1, %zmm2, %zmm1
; AVX512-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX512-VPCLMULQDQ-NEXT: vextracti128 $1, %ymm1, %xmm3
; AVX512-VPCLMULQDQ-NEXT: vpclmulqdq $17, %xmm1, %xmm0, %xmm4
diff --git a/llvm/test/CodeGen/X86/pmaddubsw.ll b/llvm/test/CodeGen/X86/pmaddubsw.ll
index d6c9877cd99b6..deb8a60e529fb 100644
--- a/llvm/test/CodeGen/X86/pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/pmaddubsw.ll
@@ -43,26 +43,183 @@ define <8 x i16> @pmaddubsw_128(ptr %Aptr, ptr %Bptr) {
define <16 x i16> @pmaddubsw_256(ptr %Aptr, ptr %Bptr) {
; SSE-LABEL: pmaddubsw_256:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa (%rsi), %xmm0
-; SSE-NEXT: movdqa 16(%rsi), %xmm1
-; SSE-NEXT: pmaddubsw (%rdi), %xmm0
-; SSE-NEXT: pmaddubsw 16(%rdi), %xmm1
+; SSE-NEXT: movdqa (%rdi), %xmm2
+; SSE-NEXT: movdqa 16(%rdi), %xmm0
+; SSE-NEXT: movdqa (%rsi), %xmm3
+; SSE-NEXT: movdqa 16(%rsi), %xmm7
+; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT: movdqa %xmm7, %xmm1
+; SSE-NEXT: pand %xmm5, %xmm1
+; SSE-NEXT: pand %xmm3, %xmm5
+; SSE-NEXT: packuswb %xmm1, %xmm5
+; SSE-NEXT: pxor %xmm6, %xmm6
+; SSE-NEXT: movdqa %xmm5, %xmm4
+; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; SSE-NEXT: movdqa %xmm3, %xmm1
+; SSE-NEXT: psrlw $8, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE-NEXT: movdqa %xmm7, %xmm1
+; SSE-NEXT: psrlw $8, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; SSE-NEXT: movdqa %xmm2, %xmm8
+; SSE-NEXT: psllw $8, %xmm8
+; SSE-NEXT: psraw $8, %xmm8
+; SSE-NEXT: movdqa %xmm0, %xmm9
+; SSE-NEXT: psllw $8, %xmm9
+; SSE-NEXT: psraw $8, %xmm9
+; SSE-NEXT: psraw $8, %xmm2
+; SSE-NEXT: psraw $8, %xmm0
+; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; SSE-NEXT: movdqa %xmm9, %xmm1
+; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT: pmaddwd %xmm7, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
+; SSE-NEXT: pmaddwd %xmm5, %xmm9
+; SSE-NEXT: packssdw %xmm9, %xmm1
+; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; SSE-NEXT: movdqa %xmm8, %xmm0
+; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE-NEXT: pmaddwd %xmm3, %xmm0
+; SSE-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
+; SSE-NEXT: pmaddwd %xmm4, %xmm8
+; SSE-NEXT: packssdw %xmm8, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: pmaddubsw_256:
; AVX1: # %bb.0:
+; AVX1-NEXT: vmovdqa (%rdi), %xmm0
+; AVX1-NEXT: vmovdqa 16(%rdi), %xmm2
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm1
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm4
+; AVX1-NEXT: vpackuswb %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vmovq {{.*#+}} xmm4 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpshufb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm0[0],xmm2[0]
; AVX1-NEXT: vmovdqa (%rsi), %xmm0
-; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1
-; AVX1-NEXT: vpmaddubsw 16(%rdi), %xmm1, %xmm1
-; AVX1-NEXT: vpmaddubsw (%rdi), %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vmovdqa 16(%rsi), %xmm5
+; AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm3
+; AVX1-NEXT: vpackuswb %xmm6, %xmm3, %xmm3
+; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm2[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[2,3,2,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm2[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; AVX1-NEXT: vpmovsxbw %xmm11, %xmm11
+; AVX1-NEXT: vpmovsxbw %xmm8, %xmm8
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm11[0],xmm8[1],xmm11[1],xmm8[2],xmm11[2],xmm8[3],xmm11[3]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm8, %xmm3
+; AVX1-NEXT: vpmovsxbw %xmm10, %xmm4
+; AVX1-NEXT: vpmovsxbw %xmm7, %xmm7
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX1-NEXT: vpmaddwd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpackssdw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpmovsxbw %xmm2, %xmm2
+; AVX1-NEXT: vpmovsxbw %xmm1, %xmm1
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT: vpmaddwd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpmovsxbw %xmm9, %xmm2
+; AVX1-NEXT: vpmovsxbw %xmm6, %xmm4
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; AVX1-NEXT: vpmaddwd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vpackssdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddubsw_256:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vmovdqa (%rsi), %ymm0
-; AVX256-NEXT: vpmaddubsw (%rdi), %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddubsw_256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vmovdqa (%rsi), %ymm1
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,2,3]
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm1[0,2,2,3]
+; AVX2-NEXT: vpmovsxbd %xmm3, %ymm6
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
+; AVX2-NEXT: vpmovsxbd %xmm3, %ymm3
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm2, %ymm6, %ymm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vpmovsxbd %xmm0, %ymm5
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovsxbd %xmm0, %ymm0
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm5, %ymm1
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: vpackssdw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: pmaddubsw_256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vpshufb %ymm2, %ymm0, %ymm3
+; AVX512F-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm4 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512F-NEXT: vpshufb %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX512F-NEXT: vpshufb %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm2, %zmm3, %zmm2
+; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT: retq
+;
+; AVX512BW-LABEL: pmaddubsw_256:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512BW-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm2
+; AVX512BW-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512BW-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512BW-NEXT: vpmovwb %zmm1, %ymm4
+; AVX512BW-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512BW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512BW-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm3, %zmm2, %zmm2
+; AVX512BW-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512BW-NEXT: retq
%A = load <32 x i8>, ptr %Aptr
%B = load <32 x i8>, ptr %Bptr
%A_even = shufflevector <32 x i8> %A, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -152,24 +309,167 @@ define <64 x i16> @pmaddubsw_512(ptr %Aptr, ptr %Bptr) {
;
; AVX512F-LABEL: pmaddubsw_512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rsi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1
-; AVX512F-NEXT: vmovdqa 64(%rsi), %ymm2
-; AVX512F-NEXT: vmovdqa 96(%rsi), %ymm3
-; AVX512F-NEXT: vpmaddubsw 32(%rdi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddubsw (%rdi), %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm3
+; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm4
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm5 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vpshufb %ymm5, %ymm4, %ymm2
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3],ymm7[4,5],ymm2[6,7]
+; AVX512F-NEXT: vpshufb %ymm5, %ymm1, %ymm7
+; AVX512F-NEXT: vpshufb %ymm6, %ymm0, %ymm8
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm7, %zmm2
+; AVX512F-NEXT: vpermq {{.*#+}} zmm2 = zmm2[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm7 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm8 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm8, %ymm3, %ymm3
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm1, %ymm1
+; AVX512F-NEXT: vpshufb %ymm8, %ymm0, %ymm0
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; AVX512F-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm3
+; AVX512F-NEXT: vmovdqa 64(%rsi), %ymm4
+; AVX512F-NEXT: vmovdqa 96(%rsi), %ymm9
+; AVX512F-NEXT: vpshufb %ymm5, %ymm9, %ymm10
+; AVX512F-NEXT: vpshufb %ymm6, %ymm4, %ymm11
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,3],ymm11[4,5],ymm10[6,7]
+; AVX512F-NEXT: vpshufb %ymm5, %ymm3, %ymm5
+; AVX512F-NEXT: vpshufb %ymm6, %ymm1, %ymm6
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm10, %zmm5, %zmm5
+; AVX512F-NEXT: vpermq {{.*#+}} zmm5 = zmm5[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm9, %ymm6
+; AVX512F-NEXT: vpshufb %ymm8, %ymm4, %ymm4
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3],ymm4[4,5],ymm6[6,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT: vpshufb %ymm8, %ymm1, %ymm1
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; AVX512F-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm3
+; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX512F-NEXT: vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm6
+; AVX512F-NEXT: vpmovsxbd %xmm6, %zmm6
+; AVX512F-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm5, %ymm7
+; AVX512F-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero,xmm8[4],zero,zero,zero,xmm8[5],zero,zero,zero,xmm8[6],zero,zero,zero,xmm8[7],zero,zero,zero,xmm8[8],zero,zero,zero,xmm8[9],zero,zero,zero,xmm8[10],zero,zero,zero,xmm8[11],zero,zero,zero,xmm8[12],zero,zero,zero,xmm8[13],zero,zero,zero,xmm8[14],zero,zero,zero,xmm8[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm8, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm5, %xmm7
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm6, %zmm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm5, %zmm2, %zmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm5
+; AVX512F-NEXT: vextracti128 $1, %ymm5, %xmm7
+; AVX512F-NEXT: vpmovsxbd %xmm7, %zmm7
+; AVX512F-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm8
+; AVX512F-NEXT: vpmovsxbd %xmm8, %zmm8
+; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm9
+; AVX512F-NEXT: vextracti128 $1, %ymm9, %xmm10
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero,xmm10[8],zero,zero,zero,xmm10[9],zero,zero,zero,xmm10[10],zero,zero,zero,xmm10[11],zero,zero,zero,xmm10[12],zero,zero,zero,xmm10[13],zero,zero,zero,xmm10[14],zero,zero,zero,xmm10[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm10, %zmm7, %zmm7
+; AVX512F-NEXT: vpaddd %zmm7, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero,xmm9[8],zero,zero,zero,xmm9[9],zero,zero,zero,xmm9[10],zero,zero,zero,xmm9[11],zero,zero,zero,xmm9[12],zero,zero,zero,xmm9[13],zero,zero,zero,xmm9[14],zero,zero,zero,xmm9[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm5, %zmm5
+; AVX512F-NEXT: vpaddd %zmm5, %zmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm5, %zmm8, %zmm5
+; AVX512F-NEXT: vpaddd %zmm5, %zmm6, %zmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT: vpmovsdw %zmm5, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpmaddubsw 96(%rdi), %ymm3, %ymm1
-; AVX512F-NEXT: vpmaddubsw 64(%rdi), %ymm2, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT: vpmovsdw %zmm3, %ymm1
+; AVX512F-NEXT: vpmovsdw %zmm4, %ymm2
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddubsw_512:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0
-; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm1
-; AVX512BW-NEXT: vpmaddubsw (%rdi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmaddubsw 64(%rdi), %zmm1, %zmm1
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm3
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,0,2,4,6,8,10,12,14,128,128,128,128,128,128,128,128,16,18,20,22,24,26,28,30,128,128,128,128,128,128,128,128,32,34,36,38,40,42,44,46,128,128,128,128,128,128,128,128,48,50,52,54,56,58,60,62]
+; AVX512BW-NEXT: vpshufb %zmm4, %zmm1, %zmm5
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm6 = [0,2,4,6,8,10,12,14,128,128,128,128,128,128,128,128,16,18,20,22,24,26,28,30,128,128,128,128,128,128,128,128,32,34,36,38,40,42,44,46,128,128,128,128,128,128,128,128,48,50,52,54,56,58,60,62,128,128,128,128,128,128,128,128]
+; AVX512BW-NEXT: vpshufb %zmm6, %zmm0, %zmm7
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm8 = [0,2,4,6,9,11,13,15]
+; AVX512BW-NEXT: vpermt2q %zmm5, %zmm8, %zmm7
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm5 = [128,128,128,128,128,128,128,128,1,3,5,7,9,11,13,15,128,128,128,128,128,128,128,128,17,19,21,23,25,27,29,31,128,128,128,128,128,128,128,128,33,35,37,39,41,43,45,47,128,128,128,128,128,128,128,128,49,51,53,55,57,59,61,63]
+; AVX512BW-NEXT: vpshufb %zmm5, %zmm1, %zmm9
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm10 = [1,3,5,7,9,11,13,15,128,128,128,128,128,128,128,128,17,19,21,23,25,27,29,31,128,128,128,128,128,128,128,128,33,35,37,39,41,43,45,47,128,128,128,128,128,128,128,128,49,51,53,55,57,59,61,63,128,128,128,128,128,128,128,128]
+; AVX512BW-NEXT: vpshufb %zmm10, %zmm0, %zmm1
+; AVX512BW-NEXT: vpermt2q %zmm9, %zmm8, %zmm1
+; AVX512BW-NEXT: vpshufb %zmm4, %zmm3, %zmm0
+; AVX512BW-NEXT: vpshufb %zmm6, %zmm2, %zmm4
+; AVX512BW-NEXT: vpermt2q %zmm0, %zmm8, %zmm4
+; AVX512BW-NEXT: vpshufb %zmm5, %zmm3, %zmm3
+; AVX512BW-NEXT: vpshufb %zmm10, %zmm2, %zmm0
+; AVX512BW-NEXT: vpermt2q %zmm3, %zmm8, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm7, %ymm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX512BW-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512BW-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm7, %xmm5
+; AVX512BW-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512BW-NEXT: vpmovsxbd %xmm7, %zmm6
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm4, %ymm7
+; AVX512BW-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero,xmm8[4],zero,zero,zero,xmm8[5],zero,zero,zero,xmm8[6],zero,zero,zero,xmm8[7],zero,zero,zero,xmm8[8],zero,zero,zero,xmm8[9],zero,zero,zero,xmm8[10],zero,zero,zero,xmm8[11],zero,zero,zero,xmm8[12],zero,zero,zero,xmm8[13],zero,zero,zero,xmm8[14],zero,zero,zero,xmm8[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm8, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm4, %xmm7
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm5, %zmm5
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm4, %zmm6, %zmm4
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm1, %ymm6
+; AVX512BW-NEXT: vextracti128 $1, %ymm6, %xmm7
+; AVX512BW-NEXT: vpmovsxbd %xmm7, %zmm7
+; AVX512BW-NEXT: vpmovsxbd %xmm6, %zmm6
+; AVX512BW-NEXT: vextracti128 $1, %ymm1, %xmm8
+; AVX512BW-NEXT: vpmovsxbd %xmm8, %zmm8
+; AVX512BW-NEXT: vpmovsxbd %xmm1, %zmm1
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm9
+; AVX512BW-NEXT: vextracti128 $1, %ymm9, %xmm10
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero,xmm10[8],zero,zero,zero,xmm10[9],zero,zero,zero,xmm10[10],zero,zero,zero,xmm10[11],zero,zero,zero,xmm10[12],zero,zero,zero,xmm10[13],zero,zero,zero,xmm10[14],zero,zero,zero,xmm10[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm10, %zmm7, %zmm7
+; AVX512BW-NEXT: vpaddd %zmm7, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero,xmm9[8],zero,zero,zero,xmm9[9],zero,zero,zero,xmm9[10],zero,zero,zero,xmm9[11],zero,zero,zero,xmm9[12],zero,zero,zero,xmm9[13],zero,zero,zero,xmm9[14],zero,zero,zero,xmm9[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm6, %zmm6
+; AVX512BW-NEXT: vpaddd %zmm6, %zmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm6
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm6, %zmm8, %zmm6
+; AVX512BW-NEXT: vpaddd %zmm6, %zmm5, %zmm5
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,6,7],zmm5[2,3,6,7]
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm5[0,1,4,5]
+; AVX512BW-NEXT: vpackssdw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[2,3,6,7],zmm3[2,3,6,7]
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,4,5],zmm3[0,1,4,5]
+; AVX512BW-NEXT: vpackssdw %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: retq
%A = load <128 x i8>, ptr %Aptr
%B = load <128 x i8>, ptr %Bptr
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index 625d18bcf4edf..3b4f657b90a31 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -1183,9 +1183,9 @@ define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) {
; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512-NEXT: vpermd %zmm0, %zmm1, %zmm0
+; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512-NEXT: retq
entry:
%val1a = zext <8 x i32> %val1 to <8 x i64>
diff --git a/llvm/test/CodeGen/X86/pr167793.ll b/llvm/test/CodeGen/X86/pr167793.ll
index 9b394bfddc396..995ef831657fd 100644
--- a/llvm/test/CodeGen/X86/pr167793.ll
+++ b/llvm/test/CodeGen/X86/pr167793.ll
@@ -6,11 +6,14 @@ define <4 x double> @PR167793(<4 x double> %a0, <4 x double> %a1) {
; CHECK: # %bb.0:
; CHECK-NEXT: vhaddpd %ymm0, %ymm0, %ymm0
; CHECK-NEXT: vhaddpd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm2
+; CHECK-NEXT: vaddpd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT: vextractf128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vaddpd %xmm1, %xmm2, %xmm1
; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3],ymm1[2,3]
+; CHECK-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0]
+; CHECK-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-NEXT: vaddpd %ymm0, %ymm3, %ymm0
-; CHECK-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm2[1],ymm0[3],ymm2[3]
; CHECK-NEXT: retq
%i5 = shufflevector <4 x double> %a0, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
%i6 = fadd <4 x double> %a0, %i5
diff --git a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
index 2ac2be5545dfd..d3493733124d9 100644
--- a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
+++ b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
@@ -61,22 +61,22 @@ define void @failing(ptr %0, ptr %1) nounwind {
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
; CHECK-NEXT: movdqu 1024(%rdx,%rsi), %xmm5
; CHECK-NEXT: movdqu 1040(%rdx,%rsi), %xmm6
-; CHECK-NEXT: movq %xmm5, %rdi
-; CHECK-NEXT: movq %xmm6, %r8
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm5[2,3,2,3]
+; CHECK-NEXT: movq %xmm7, %rdi
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; CHECK-NEXT: movq %xmm7, %r8
; CHECK-NEXT: movq %xmm5, %r9
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm6[2,3,2,3]
-; CHECK-NEXT: movq %xmm5, %r10
-; CHECK-NEXT: negq %r8
-; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: sbbq %r10, %r8
+; CHECK-NEXT: movq %xmm6, %r10
+; CHECK-NEXT: negq %r10
+; CHECK-NEXT: movq %rcx, %r10
+; CHECK-NEXT: sbbq %r8, %r10
; CHECK-NEXT: setge %r8b
; CHECK-NEXT: movzbl %r8b, %r8d
; CHECK-NEXT: negq %r8
; CHECK-NEXT: movq %r8, %xmm5
-; CHECK-NEXT: negq %rdi
-; CHECK-NEXT: movq %rcx, %rdi
-; CHECK-NEXT: sbbq %r9, %rdi
+; CHECK-NEXT: negq %r9
+; CHECK-NEXT: movq %rcx, %r8
+; CHECK-NEXT: sbbq %rdi, %r8
; CHECK-NEXT: setge %dil
; CHECK-NEXT: movzbl %dil, %edi
; CHECK-NEXT: negq %rdi
@@ -119,12 +119,13 @@ define void @failing(ptr %0, ptr %1) nounwind {
; CHECK-AVX2-NEXT: .LBB0_2: # %vector.body
; CHECK-AVX2-NEXT: # Parent Loop BB0_1 Depth=1
; CHECK-AVX2-NEXT: # => This Inner Loop Header: Depth=2
-; CHECK-AVX2-NEXT: vmovdqu 1024(%rdx,%rsi), %xmm5
-; CHECK-AVX2-NEXT: vmovdqu 1040(%rdx,%rsi), %xmm6
-; CHECK-AVX2-NEXT: vpextrq $1, %xmm5, %rdi
-; CHECK-AVX2-NEXT: vpextrq $1, %xmm6, %r8
-; CHECK-AVX2-NEXT: vmovq %xmm5, %r9
-; CHECK-AVX2-NEXT: vmovq %xmm6, %r10
+; CHECK-AVX2-NEXT: vmovdqu 1024(%rdx,%rsi), %ymm5
+; CHECK-AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm5[0,2,2,3]
+; CHECK-AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[1,3,2,3]
+; CHECK-AVX2-NEXT: vmovq %xmm5, %rdi
+; CHECK-AVX2-NEXT: vpextrq $1, %xmm5, %r8
+; CHECK-AVX2-NEXT: vmovq %xmm6, %r9
+; CHECK-AVX2-NEXT: vpextrq $1, %xmm6, %r10
; CHECK-AVX2-NEXT: negq %r10
; CHECK-AVX2-NEXT: movq %rcx, %r10
; CHECK-AVX2-NEXT: sbbq %r8, %r10
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
index f632654f89e04..024ce23d9ba4c 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
@@ -1,65 +1,45 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST,AVX2-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST,AVX2-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512VL-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512VL-FAST-PERLANE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BW-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BW-FAST-PERLANE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512BWVL-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512BWVL-FAST-PERLANE
define void @shuffle_v32i8_to_v16i8_1(ptr %L, ptr %S) nounwind {
-; AVX-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovdqa (%rdi), %xmm0
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
-; AVX-NEXT: vmovq {{.*#+}} xmm2 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX-NEXT: vpshufb %xmm2, %xmm1, %xmm1
-; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm0
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: retq
-;
-; AVX512F-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT: vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT: vmovq {{.*#+}} xmm2 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512F-NEXT: vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT: vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512F-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT: vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm2 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512VL-NEXT: vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT: vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512VL-NEXT: retq
+; AVX1-LABEL: shuffle_v32i8_to_v16i8_1:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovdqa (%rdi), %xmm0
+; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT: vmovq {{.*#+}} xmm2 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX1-NEXT: retq
;
-; AVX512BW-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
-; AVX512BW-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
+; AVX2-LABEL: shuffle_v32i8_to_v16i8_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
-; AVX512BWVL-LABEL: shuffle_v32i8_to_v16i8_1:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $8, (%rdi), %ymm0
-; AVX512BWVL-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: shuffle_v32i8_to_v16i8_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%vec = load <32 x i8>, ptr %L
%strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
store <16 x i8> %strided.vec, ptr %S
@@ -115,19 +95,87 @@ define void @shuffle_v16i16_to_v8i16_1(ptr %L, ptr %S) nounwind {
}
define void @shuffle_v8i32_to_v4i32_1(ptr %L, ptr %S) nounwind {
-; AVX-LABEL: shuffle_v8i32_to_v4i32_1:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovaps (%rdi), %xmm0
-; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
-; AVX-NEXT: vmovaps %xmm0, (%rsi)
-; AVX-NEXT: retq
+; AVX1-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovaps (%rdi), %xmm0
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX1-NEXT: vmovaps %xmm0, (%rsi)
+; AVX1-NEXT: retq
;
-; AVX512-LABEL: shuffle_v8i32_to_v4i32_1:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovaps (%rdi), %xmm0
-; AVX512-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
-; AVX512-NEXT: vmovaps %xmm0, (%rsi)
-; AVX512-NEXT: retq
+; AVX2-SLOW-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX2-SLOW-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX2-FAST-ALL: # %bb.0:
+; AVX2-FAST-ALL-NEXT: vbroadcastf128 {{.*#+}} ymm0 = [1,3,5,7,1,3,5,7]
+; AVX2-FAST-ALL-NEXT: # ymm0 = mem[0,1,0,1]
+; AVX2-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX2-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-FAST-ALL-NEXT: vzeroupper
+; AVX2-FAST-ALL-NEXT: retq
+;
+; AVX2-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX2-FAST-PERLANE: # %bb.0:
+; AVX2-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX2-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-FAST-PERLANE-NEXT: retq
+;
+; AVX512F-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovaps (%rdi), %xmm0
+; AVX512F-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX512F-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512VL-FAST-ALL: # %bb.0:
+; AVX512VL-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,0,0,0,0]
+; AVX512VL-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX512VL-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512VL-FAST-ALL-NEXT: vzeroupper
+; AVX512VL-FAST-ALL-NEXT: retq
+;
+; AVX512VL-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512VL-FAST-PERLANE: # %bb.0:
+; AVX512VL-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX512VL-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX512VL-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512VL-FAST-PERLANE-NEXT: retq
+;
+; AVX512BW-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512BW-FAST-ALL: # %bb.0:
+; AVX512BW-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,0,0,0,0]
+; AVX512BW-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX512BW-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BW-FAST-ALL-NEXT: vzeroupper
+; AVX512BW-FAST-ALL-NEXT: retq
+;
+; AVX512BW-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512BW-FAST-PERLANE: # %bb.0:
+; AVX512BW-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX512BW-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX512BW-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BW-FAST-PERLANE-NEXT: retq
+;
+; AVX512BWVL-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512BWVL-FAST-ALL: # %bb.0:
+; AVX512BWVL-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,0,0,0,0]
+; AVX512BWVL-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX512BWVL-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BWVL-FAST-ALL-NEXT: vzeroupper
+; AVX512BWVL-FAST-ALL-NEXT: retq
+;
+; AVX512BWVL-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32_1:
+; AVX512BWVL-FAST-PERLANE: # %bb.0:
+; AVX512BWVL-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX512BWVL-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],mem[1,3]
+; AVX512BWVL-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BWVL-FAST-PERLANE-NEXT: retq
%vec = load <8 x i32>, ptr %L
%strided.vec = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
store <4 x i32> %strided.vec, ptr %S
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
index 571915b47d297..bc94ba6782c3c 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-512.ll
@@ -48,15 +48,21 @@ define void @shuffle_v64i8_to_v32i8_1(ptr %L, ptr %S) nounwind {
;
; AVX512BW-LABEL: shuffle_v64i8_to_v32i8_1:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsrlw $8, (%rdi), %zmm0
-; AVX512BW-NEXT: vpmovwb %zmm0, (%rsi)
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vmovdqa %ymm0, (%rsi)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: shuffle_v64i8_to_v32i8_1:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $8, (%rdi), %zmm0
-; AVX512BWVL-NEXT: vpmovwb %zmm0, (%rsi)
+; AVX512BWVL-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BWVL-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BWVL-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BWVL-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512BWVL-NEXT: vmovdqa %ymm0, (%rsi)
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
%vec = load <64 x i8>, ptr %L
@@ -66,12 +72,35 @@ define void @shuffle_v64i8_to_v32i8_1(ptr %L, ptr %S) nounwind {
}
define void @shuffle_v32i16_to_v16i16_1(ptr %L, ptr %S) nounwind {
-; AVX512-LABEL: shuffle_v32i16_to_v16i16_1:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrld $16, (%rdi), %zmm0
-; AVX512-NEXT: vpmovdw %zmm0, (%rsi)
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX512F-LABEL: shuffle_v32i16_to_v16i16_1:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpsrld $16, (%rdi), %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: shuffle_v32i16_to_v16i16_1:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpsrld $16, (%rdi), %zmm0
+; AVX512VL-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512BW-LABEL: shuffle_v32i16_to_v16i16_1:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-NEXT: vpermw (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BWVL-LABEL: shuffle_v32i16_to_v16i16_1:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpmovsxbw {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BWVL-NEXT: vpermw (%rdi), %zmm0, %zmm0
+; AVX512BWVL-NEXT: vmovdqa %ymm0, (%rsi)
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
%vec = load <32 x i16>, ptr %L
%strided.vec = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
store <16 x i16> %strided.vec, ptr %S
@@ -79,65 +108,13 @@ define void @shuffle_v32i16_to_v16i16_1(ptr %L, ptr %S) nounwind {
}
define void @shuffle_v16i32_to_v8i32_1(ptr %L, ptr %S) nounwind {
-; AVX512F-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovaps (%rdi), %ymm0
-; AVX512F-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],mem[1,3],ymm0[5,7],mem[5,7]
-; AVX512F-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512F-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512VL-FAST-ALL-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512VL-FAST-ALL: # %bb.0:
-; AVX512VL-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15]
-; AVX512VL-FAST-ALL-NEXT: vpermps (%rdi), %zmm0, %zmm0
-; AVX512VL-FAST-ALL-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512VL-FAST-ALL-NEXT: vzeroupper
-; AVX512VL-FAST-ALL-NEXT: retq
-;
-; AVX512VL-FAST-PERLANE-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512VL-FAST-PERLANE: # %bb.0:
-; AVX512VL-FAST-PERLANE-NEXT: vmovaps (%rdi), %ymm0
-; AVX512VL-FAST-PERLANE-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],mem[1,3],ymm0[5,7],mem[5,7]
-; AVX512VL-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VL-FAST-PERLANE-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512VL-FAST-PERLANE-NEXT: vzeroupper
-; AVX512VL-FAST-PERLANE-NEXT: retq
-;
-; AVX512BW-FAST-ALL-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512BW-FAST-ALL: # %bb.0:
-; AVX512BW-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15]
-; AVX512BW-FAST-ALL-NEXT: vpermps (%rdi), %zmm0, %zmm0
-; AVX512BW-FAST-ALL-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BW-FAST-ALL-NEXT: vzeroupper
-; AVX512BW-FAST-ALL-NEXT: retq
-;
-; AVX512BW-FAST-PERLANE-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512BW-FAST-PERLANE: # %bb.0:
-; AVX512BW-FAST-PERLANE-NEXT: vmovaps (%rdi), %ymm0
-; AVX512BW-FAST-PERLANE-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],mem[1,3],ymm0[5,7],mem[5,7]
-; AVX512BW-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BW-FAST-PERLANE-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BW-FAST-PERLANE-NEXT: vzeroupper
-; AVX512BW-FAST-PERLANE-NEXT: retq
-;
-; AVX512BWVL-FAST-ALL-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512BWVL-FAST-ALL: # %bb.0:
-; AVX512BWVL-FAST-ALL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15]
-; AVX512BWVL-FAST-ALL-NEXT: vpermps (%rdi), %zmm0, %zmm0
-; AVX512BWVL-FAST-ALL-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BWVL-FAST-ALL-NEXT: vzeroupper
-; AVX512BWVL-FAST-ALL-NEXT: retq
-;
-; AVX512BWVL-FAST-PERLANE-LABEL: shuffle_v16i32_to_v8i32_1:
-; AVX512BWVL-FAST-PERLANE: # %bb.0:
-; AVX512BWVL-FAST-PERLANE-NEXT: vmovaps (%rdi), %ymm0
-; AVX512BWVL-FAST-PERLANE-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],mem[1,3],ymm0[5,7],mem[5,7]
-; AVX512BWVL-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BWVL-FAST-PERLANE-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BWVL-FAST-PERLANE-NEXT: vzeroupper
-; AVX512BWVL-FAST-PERLANE-NEXT: retq
+; AVX512-LABEL: shuffle_v16i32_to_v8i32_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15]
+; AVX512-NEXT: vpermps (%rdi), %zmm0, %zmm0
+; AVX512-NEXT: vmovaps %ymm0, (%rsi)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%vec = load <16 x i32>, ptr %L
%strided.vec = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
store <8 x i32> %strided.vec, ptr %S
@@ -314,4 +291,7 @@ define void @shuffle_v64i8_to_v8i8_7(ptr %L, ptr %S) nounwind {
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512VL: {{.*}}
+; AVX512BW-FAST-ALL: {{.*}}
+; AVX512BW-FAST-PERLANE: {{.*}}
+; AVX512BWVL-FAST-ALL: {{.*}}
+; AVX512BWVL-FAST-PERLANE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
index 26af46263c0e2..63d0de1522c2e 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
@@ -889,253 +889,54 @@ define <16 x i8> @evenelts_v32i16_trunc_v16i16_to_v16i8(<32 x i16> %n2) nounwind
;
; AVX1-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %rbx
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpextrw $6, %xmm2, %eax
-; AVX1-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX1-NEXT: vpextrw $2, %xmm2, %edx
-; AVX1-NEXT: vmovd %xmm2, %esi
-; AVX1-NEXT: vpextrw $6, %xmm1, %edi
-; AVX1-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX1-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX1-NEXT: vmovd %xmm1, %r10d
+; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3],xmm1[4],xmm3[5],xmm1[6],xmm3[7]
+; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX1-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX1-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX1-NEXT: vmovd %xmm1, %r14d
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrw $6, %xmm2, %eax
-; AVX2-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm2, %edx
-; AVX2-NEXT: vmovd %xmm2, %esi
-; AVX2-NEXT: vpextrw $6, %xmm1, %edi
-; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX2-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX2-NEXT: vmovd %xmm1, %r10d
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX2-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX2-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX2-NEXT: vmovd %xmm1, %r14d
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
-;
-; AVX512F-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vpmovdb %zmm0, %xmm1
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrw $6, %xmm2, %eax
-; AVX512F-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX512F-NEXT: vpextrw $2, %xmm2, %edx
-; AVX512F-NEXT: vmovd %xmm2, %esi
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrw $6, %xmm2, %edi
-; AVX512F-NEXT: vpextrw $4, %xmm2, %r8d
-; AVX512F-NEXT: vpextrw $2, %xmm2, %r9d
-; AVX512F-NEXT: vmovd %xmm2, %r10d
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512F-NEXT: vpextrw $4, %xmm0, %ebx
-; AVX512F-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512F-NEXT: vpinsrb $5, %ebp, %xmm1, %xmm0
-; AVX512F-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %rbp
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbp
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512VL-NEXT: vmovd %xmm1, %esi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %edi
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX512VL-NEXT: vmovd %xmm1, %r10d
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX512VL-NEXT: vmovd %xmm1, %r14d
-; AVX512VL-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %rbp
-; AVX512VL-NEXT: vzeroupper
-; AVX512VL-NEXT: retq
+; AVX2-SLOW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-SLOW-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vzeroupper
+; AVX2-SLOW-NEXT: retq
;
-; AVX512BW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: pushq %rbp
-; AVX512BW-NEXT: pushq %rbx
-; AVX512BW-NEXT: vpmovdb %zmm0, %xmm1
-; AVX512BW-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512BW-NEXT: vpextrw $6, %xmm2, %eax
-; AVX512BW-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX512BW-NEXT: vpextrw $2, %xmm2, %edx
-; AVX512BW-NEXT: vmovd %xmm2, %esi
-; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX512BW-NEXT: vpextrw $6, %xmm2, %edi
-; AVX512BW-NEXT: vpextrw $4, %xmm2, %r8d
-; AVX512BW-NEXT: vpextrw $2, %xmm2, %r9d
-; AVX512BW-NEXT: vmovd %xmm2, %r10d
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512BW-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512BW-NEXT: vpextrw $4, %xmm0, %ebx
-; AVX512BW-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512BW-NEXT: vpinsrb $5, %ebp, %xmm1, %xmm0
-; AVX512BW-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BW-NEXT: popq %rbx
-; AVX512BW-NEXT: popq %rbp
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
+; AVX2-FAST-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpmovsxdq {{.*#+}} ymm2 = [84148480,218892552,353636624,488380696]
+; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX2-FAST-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-FAST-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
;
-; AVX512BWVL-ONLY-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BWVL-ONLY: # %bb.0:
-; AVX512BWVL-ONLY-NEXT: pushq %rbp
-; AVX512BWVL-ONLY-NEXT: pushq %r14
-; AVX512BWVL-ONLY-NEXT: pushq %rbx
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %esi
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %edi
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %r10d
-; AVX512BWVL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %r14d
-; AVX512BWVL-ONLY-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: popq %rbx
-; AVX512BWVL-ONLY-NEXT: popq %r14
-; AVX512BWVL-ONLY-NEXT: popq %rbp
-; AVX512BWVL-ONLY-NEXT: vzeroupper
-; AVX512BWVL-ONLY-NEXT: retq
-;
-; AVX512VBMI-FAST-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-FAST: # %bb.0:
-; AVX512VBMI-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,79]
-; AVX512VBMI-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI-FAST-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
-; AVX512VBMI-FAST-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-FAST-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512VBMI-FAST-NEXT: vpinsrb $15, %eax, %xmm1, %xmm0
-; AVX512VBMI-FAST-NEXT: vzeroupper
-; AVX512VBMI-FAST-NEXT: retq
-;
-; AVX512VBMI-SLOW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-SLOW: # %bb.0:
-; AVX512VBMI-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [0,4,8,12,16,20,24,28,32,36,40,44,48,77,78,79]
-; AVX512VBMI-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI-SLOW-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
-; AVX512VBMI-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512VBMI-SLOW-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512VBMI-SLOW-NEXT: vpextrw $2, %xmm0, %edx
-; AVX512VBMI-SLOW-NEXT: vpinsrb $13, %edx, %xmm1, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vzeroupper
-; AVX512VBMI-SLOW-NEXT: retq
+; AVX512-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%n0 = shufflevector <32 x i16> %n2, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%n1 = trunc <16 x i16> %n0 to <16 x i8>
ret <16 x i8> %n1
@@ -1230,260 +1031,70 @@ define <16 x i8> @oddelts_v32i16_trunc_v16i16_to_v16i8(<32 x i16> %n2) nounwind
;
; AVX1-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %rbx
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpextrw $7, %xmm2, %eax
-; AVX1-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX1-NEXT: vpextrw $3, %xmm2, %edx
-; AVX1-NEXT: vpextrw $1, %xmm2, %esi
-; AVX1-NEXT: vpextrw $7, %xmm1, %edi
-; AVX1-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX1-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX1-NEXT: vpextrw $1, %xmm1, %r10d
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX1-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX1-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX1-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrw $7, %xmm2, %eax
-; AVX2-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX2-NEXT: vpextrw $3, %xmm2, %edx
-; AVX2-NEXT: vpextrw $1, %xmm2, %esi
-; AVX2-NEXT: vpextrw $7, %xmm1, %edi
-; AVX2-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX2-NEXT: vpextrw $1, %xmm1, %r10d
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX2-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX2-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX2-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: pushq %r14
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512F-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512F-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512F-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512F-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512F-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512F-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512F-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512F-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512F-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %r14
-; AVX512F-NEXT: popq %rbp
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbp
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512VL-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %rbp
+; AVX512VL-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: pushq %rbp
-; AVX512BW-NEXT: pushq %r14
-; AVX512BW-NEXT: pushq %rbx
-; AVX512BW-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BW-NEXT: popq %rbx
-; AVX512BW-NEXT: popq %r14
-; AVX512BW-NEXT: popq %rbp
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-NEXT: vpermw %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
-; AVX512BWVL-ONLY-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BWVL-ONLY: # %bb.0:
-; AVX512BWVL-ONLY-NEXT: pushq %rbp
-; AVX512BWVL-ONLY-NEXT: pushq %r14
-; AVX512BWVL-ONLY-NEXT: pushq %rbx
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512BWVL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512BWVL-ONLY-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BWVL-ONLY-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: popq %rbx
-; AVX512BWVL-ONLY-NEXT: popq %r14
-; AVX512BWVL-ONLY-NEXT: popq %rbp
-; AVX512BWVL-ONLY-NEXT: vzeroupper
-; AVX512BWVL-ONLY-NEXT: retq
-;
-; AVX512VBMI-FAST-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-FAST: # %bb.0:
-; AVX512VBMI-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]
-; AVX512VBMI-FAST-NEXT: vpermb %zmm0, %zmm1, %zmm0
-; AVX512VBMI-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512VBMI-FAST-NEXT: vzeroupper
-; AVX512VBMI-FAST-NEXT: retq
-;
-; AVX512VBMI-SLOW-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-SLOW: # %bb.0:
-; AVX512VBMI-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [2,6,10,14,18,22,26,30,34,38,42,46,50,u,u,u]
-; AVX512VBMI-SLOW-NEXT: vpermb %zmm0, %zmm1, %zmm1
-; AVX512VBMI-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512VBMI-SLOW-NEXT: vpextrw $5, %xmm0, %ecx
-; AVX512VBMI-SLOW-NEXT: vpextrw $3, %xmm0, %edx
-; AVX512VBMI-SLOW-NEXT: vpinsrb $13, %edx, %xmm1, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vzeroupper
-; AVX512VBMI-SLOW-NEXT: retq
+; AVX512BWVL-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BWVL-NEXT: vpermw %zmm0, %zmm1, %zmm0
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
%n0 = shufflevector <32 x i16> %n2, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
%n1 = trunc <16 x i16> %n0 to <16 x i8>
ret <16 x i8> %n1
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512BWVL-ONLY: {{.*}}
; AVX512VBMI: {{.*}}
+; AVX512VBMI-FAST: {{.*}}
+; AVX512VBMI-SLOW: {{.*}}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 05c855ed90b3f..cd86b7b91505b 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -6,8 +6,8 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BWVL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BWVL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VBMIVL
@@ -29,29 +29,29 @@ define void @shuffle_v32i8_to_v16i8(ptr %L, ptr %S) nounwind {
;
; AVX2-LABEL: shuffle_v32i8_to_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT: vpand 16(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vpand (%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: shuffle_v32i8_to_v16i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512F-NEXT: vpand 16(%rdi), %xmm0, %xmm1
-; AVX512F-NEXT: vpand (%rdi), %xmm0, %xmm0
-; AVX512F-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
; AVX512F-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: shuffle_v32i8_to_v16i8:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT: vpand 16(%rdi), %xmm0, %xmm1
-; AVX512VL-NEXT: vpand (%rdi), %xmm0, %xmm0
-; AVX512VL-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
; AVX512VL-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: shuffle_v32i8_to_v16i8:
@@ -260,12 +260,35 @@ define void @trunc_v8i32_to_v8i16(ptr %L, ptr %S) nounwind {
}
define void @shuffle_v8i32_to_v4i32(ptr %L, ptr %S) nounwind {
-; AVX-LABEL: shuffle_v8i32_to_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovaps (%rdi), %xmm0
-; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX-NEXT: vmovaps %xmm0, (%rsi)
-; AVX-NEXT: retq
+; AVX1-LABEL: shuffle_v8i32_to_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovaps (%rdi), %xmm0
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX1-NEXT: vmovaps %xmm0, (%rsi)
+; AVX1-NEXT: retq
+;
+; AVX2-SLOW-LABEL: shuffle_v8i32_to_v4i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX2-SLOW-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-ALL-LABEL: shuffle_v8i32_to_v4i32:
+; AVX2-FAST-ALL: # %bb.0:
+; AVX2-FAST-ALL-NEXT: vbroadcastf128 {{.*#+}} ymm0 = [0,2,4,6,0,2,4,6]
+; AVX2-FAST-ALL-NEXT: # ymm0 = mem[0,1,0,1]
+; AVX2-FAST-ALL-NEXT: vpermps (%rdi), %ymm0, %ymm0
+; AVX2-FAST-ALL-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-FAST-ALL-NEXT: vzeroupper
+; AVX2-FAST-ALL-NEXT: retq
+;
+; AVX2-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32:
+; AVX2-FAST-PERLANE: # %bb.0:
+; AVX2-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX2-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX2-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX2-FAST-PERLANE-NEXT: retq
;
; AVX512F-LABEL: shuffle_v8i32_to_v4i32:
; AVX512F: # %bb.0:
@@ -281,12 +304,20 @@ define void @shuffle_v8i32_to_v4i32(ptr %L, ptr %S) nounwind {
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
-; AVX512BW-LABEL: shuffle_v8i32_to_v4i32:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovaps (%rdi), %xmm0
-; AVX512BW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512BW-NEXT: vmovaps %xmm0, (%rsi)
-; AVX512BW-NEXT: retq
+; AVX512BW-ALL-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512BW-ALL: # %bb.0:
+; AVX512BW-ALL-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512BW-ALL-NEXT: vpmovqd %zmm0, %ymm0
+; AVX512BW-ALL-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX512BW-ALL-NEXT: vzeroupper
+; AVX512BW-ALL-NEXT: retq
+;
+; AVX512BW-FAST-PERLANE-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512BW-FAST-PERLANE: # %bb.0:
+; AVX512BW-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm0
+; AVX512BW-FAST-PERLANE-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX512BW-FAST-PERLANE-NEXT: vmovaps %xmm0, (%rsi)
+; AVX512BW-FAST-PERLANE-NEXT: retq
;
; AVX512BWVL-LABEL: shuffle_v8i32_to_v4i32:
; AVX512BWVL: # %bb.0:
@@ -1301,73 +1332,65 @@ define void @trunc_v4i64_to_v4i8(ptr %L, ptr %S) nounwind {
define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
; AVX1-LABEL: negative:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[u,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u],zero,zero,zero,zero,zero,zero,zero,xmm0[0,2,4,6,8,10,12,14]
-; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615]
-; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vmovd %xmm1, %eax
+; AVX1-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: negative:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX2-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vmovd %xmm1, %eax
+; AVX2-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: negative:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512F-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX512F-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: negative:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm0[2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm1 & ~ymm2) | ymm0
-; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm2[0,3,2,3]
-; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: negative:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BW-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX512BW-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vmovd %xmm1, %eax
+; AVX512BW-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: negative:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BWVL-NEXT: movl $65537, %eax # imm = 0x10001
-; AVX512BWVL-NEXT: kmovd %eax, %k1
-; AVX512BWVL-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
-; AVX512BWVL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512BWVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vmovd %xmm1, %eax
+; AVX512BWVL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
;
; AVX512VBMIVL-LABEL: negative:
; AVX512VBMIVL: # %bb.0:
-; AVX512VBMIVL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [32,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
-; AVX512VBMIVL-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512VBMIVL-NEXT: vpermt2b %ymm1, %ymm2, %ymm0
-; AVX512VBMIVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMIVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512VBMIVL-NEXT: vmovd %xmm1, %eax
+; AVX512VBMIVL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512VBMIVL-NEXT: vzeroupper
; AVX512VBMIVL-NEXT: retq
%strided.vec = shufflevector <32 x i8> %v, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
index e27a77ed2293d..0275c14bacc19 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
@@ -359,20 +359,25 @@ define <32 x i8> @trunc_shuffle_v32i16_v32i8_ofs1(<32 x i16> %a0) {
;
; AVX512BW-LABEL: trunc_shuffle_v32i16_v32i8_ofs1:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: trunc_shuffle_v32i16_v32i8_ofs1:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BWVL-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BWVL-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BWVL-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512BWVL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512BWVL-NEXT: retq
;
; AVX512VBMI-LABEL: trunc_shuffle_v32i16_v32i8_ofs1:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512VBMI-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512VBMI-NEXT: vmovdqa {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
+; AVX512VBMI-NEXT: vpermb %zmm0, %zmm1, %zmm0
+; AVX512VBMI-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512VBMI-NEXT: retq
%bc = bitcast <32 x i16> %a0 to <64 x i8>
%res = shufflevector <64 x i8> %bc, <64 x i8> poison, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>
diff --git a/llvm/test/CodeGen/X86/sse2.ll b/llvm/test/CodeGen/X86/sse2.ll
index 6e77d3e4fd134..1eb27b88a6c8e 100644
--- a/llvm/test/CodeGen/X86/sse2.ll
+++ b/llvm/test/CodeGen/X86/sse2.ll
@@ -571,12 +571,20 @@ define <2 x double> @test16(ptr nocapture %srcA, ptr nocapture %dst) {
; X86-SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
; X86-SSE-NEXT: retl
;
-; X86-AVX-LABEL: test16:
-; X86-AVX: # %bb.0:
-; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-AVX-NEXT: vmovaps 96(%eax), %xmm0
-; X86-AVX-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
-; X86-AVX-NEXT: retl
+; X86-AVX1-LABEL: test16:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-AVX1-NEXT: vmovaps 96(%eax), %xmm0
+; X86-AVX1-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
+; X86-AVX1-NEXT: retl
+;
+; X86-AVX512-LABEL: test16:
+; X86-AVX512: # %bb.0:
+; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512-NEXT: vpermpd {{.*#+}} ymm0 = mem[0,2,2,3]
+; X86-AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X86-AVX512-NEXT: vzeroupper
+; X86-AVX512-NEXT: retl
;
; X64-SSE-LABEL: test16:
; X64-SSE: # %bb.0:
@@ -584,11 +592,18 @@ define <2 x double> @test16(ptr nocapture %srcA, ptr nocapture %dst) {
; X64-SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
; X64-SSE-NEXT: retq
;
-; X64-AVX-LABEL: test16:
-; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: vmovaps 96(%rdi), %xmm0
-; X64-AVX-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
-; X64-AVX-NEXT: retq
+; X64-AVX1-LABEL: test16:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vmovaps 96(%rdi), %xmm0
+; X64-AVX1-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0]
+; X64-AVX1-NEXT: retq
+;
+; X64-AVX512-LABEL: test16:
+; X64-AVX512: # %bb.0:
+; X64-AVX512-NEXT: vpermpd {{.*#+}} ymm0 = mem[0,2,2,3]
+; X64-AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X64-AVX512-NEXT: vzeroupper
+; X64-AVX512-NEXT: retq
%i5 = getelementptr inbounds <4 x double>, ptr %srcA, i32 3
%i6 = load <4 x double>, ptr %i5, align 32
%i7 = shufflevector <4 x double> %i6, <4 x double> undef, <2 x i32> <i32 0, i32 2>
@@ -702,8 +717,3 @@ define <4 x i32> @test_mul(<4 x i32> %x, <4 x i32> %y) {
%m = mul <4 x i32> %x, %y
ret <4 x i32> %m
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; X64-AVX1: {{.*}}
-; X64-AVX512: {{.*}}
-; X86-AVX1: {{.*}}
-; X86-AVX512: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
index e0410ae0cc5cb..eeef87606376e 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
@@ -164,62 +164,62 @@ define void @load_i16_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i16_stride2_vf8:
; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i16_stride2_vf8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i16_stride2_vf8:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-FP-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FP-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-FP-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FP-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-FP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i16_stride2_vf8:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FCP-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-FCP-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FCP-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i16_stride2_vf8:
@@ -273,27 +273,27 @@ define void @load_i16_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) no
;
; AVX-LABEL: load_i16_stride2_vf16:
; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
-; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
-; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
-; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vmovdqa 32(%rdi), %xmm2
+; AVX-NEXT: vmovdqa 48(%rdi), %xmm3
+; AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm3[0],xmm4[1],xmm3[2],xmm4[3],xmm3[4],xmm4[5],xmm3[6],xmm4[7]
+; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm4[1],xmm2[2],xmm4[3],xmm2[4],xmm4[5],xmm2[6],xmm4[7]
; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm5
-; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
-; AVX-NEXT: vpsrld $16, %xmm4, %xmm4
+; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm1[0],xmm4[1],xmm1[2],xmm4[3],xmm1[4],xmm4[5],xmm1[6],xmm4[7]
+; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm4[1],xmm0[2],xmm4[3],xmm0[4],xmm4[5],xmm0[6],xmm4[7]
+; AVX-NEXT: vpackusdw %xmm6, %xmm4, %xmm4
; AVX-NEXT: vpsrld $16, %xmm3, %xmm3
-; AVX-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa %xmm4, (%rsi)
; AVX-NEXT: vmovdqa %xmm5, 16(%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
-; AVX-NEXT: vmovdqa %xmm3, 16(%rdx)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX-NEXT: vmovdqa %xmm2, 16(%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i16_stride2_vf16:
@@ -354,14 +354,81 @@ define void @load_i16_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) no
; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
-; AVX512-LABEL: load_i16_stride2_vf16:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512-NEXT: vpsrld $16, %zmm0, %zmm1
-; AVX512-NEXT: vpmovdw %zmm0, (%rsi)
-; AVX512-NEXT: vpmovdw %zmm1, (%rdx)
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
+; AVX512-VL-LABEL: load_i16_stride2_vf16:
+; AVX512-VL: # %bb.0:
+; AVX512-VL-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512-VL-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512-VL-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512-VL-NEXT: vpmovdw %zmm1, (%rdx)
+; AVX512-VL-NEXT: vzeroupper
+; AVX512-VL-NEXT: retq
+;
+; AVX512-FCP-LABEL: load_i16_stride2_vf16:
+; AVX512-FCP: # %bb.0:
+; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512-FCP-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512-FCP-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512-FCP-NEXT: vpmovdw %zmm1, (%rdx)
+; AVX512-FCP-NEXT: vzeroupper
+; AVX512-FCP-NEXT: retq
+;
+; AVX512DQ-LABEL: load_i16_stride2_vf16:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512DQ-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512DQ-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512DQ-NEXT: vpmovdw %zmm1, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
+;
+; AVX512DQ-FCP-LABEL: load_i16_stride2_vf16:
+; AVX512DQ-FCP: # %bb.0:
+; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512DQ-FCP-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512DQ-FCP-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512DQ-FCP-NEXT: vpmovdw %zmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vzeroupper
+; AVX512DQ-FCP-NEXT: retq
+;
+; AVX512BW-LABEL: load_i16_stride2_vf16:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-NEXT: vpermw %zmm0, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512BW-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BW-FCP-LABEL: load_i16_stride2_vf16:
+; AVX512BW-FCP: # %bb.0:
+; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512BW-FCP-NEXT: vzeroupper
+; AVX512BW-FCP-NEXT: retq
+;
+; AVX512DQ-BW-LABEL: load_i16_stride2_vf16:
+; AVX512DQ-BW: # %bb.0:
+; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512DQ-BW-NEXT: vpermw %zmm0, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-BW-NEXT: vzeroupper
+; AVX512DQ-BW-NEXT: retq
+;
+; AVX512DQ-BW-FCP-LABEL: load_i16_stride2_vf16:
+; AVX512DQ-BW-FCP: # %bb.0:
+; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
+; AVX512DQ-BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT: vpmovdw %zmm0, (%rsi)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vzeroupper
+; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <32 x i16>, ptr %in.vec, align 64
%strided.vec0 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%strided.vec1 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-2.ll
index 995d641644dfa..9f5beac10018b 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-2.ll
@@ -184,12 +184,16 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX2-FCP-LABEL: load_i32_stride2_vf4:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX2-FCP-NEXT: vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,2]
-; AVX2-FCP-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
-; AVX2-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX2-FCP-NEXT: vbroadcastf128 {{.*#+}} ymm1 = [0,2,4,6,0,2,4,6]
+; AVX2-FCP-NEXT: # ymm1 = mem[0,1,0,1]
+; AVX2-FCP-NEXT: vpermps %ymm0, %ymm1, %ymm1
+; AVX2-FCP-NEXT: vbroadcastf128 {{.*#+}} ymm2 = [1,3,5,7,1,3,5,7]
+; AVX2-FCP-NEXT: # ymm2 = mem[0,1,0,1]
+; AVX2-FCP-NEXT: vpermps %ymm0, %ymm2, %ymm0
+; AVX2-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX2-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i32_stride2_vf4:
@@ -205,10 +209,10 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512-FCP-LABEL: load_i32_stride2_vf4:
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512-FCP-NEXT: vmovaps (%rdi), %xmm1
-; AVX512-FCP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,3],mem[1,3]
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,0,0,0,0]
+; AVX512-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512-FCP-NEXT: vpmovqd %ymm0, (%rsi)
-; AVX512-FCP-NEXT: vmovaps %xmm1, (%rdx)
+; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -225,10 +229,10 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512DQ-FCP-LABEL: load_i32_stride2_vf4:
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,3],mem[1,3]
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,0,0,0,0]
+; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512DQ-FCP-NEXT: vpmovqd %ymm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovaps %xmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
@@ -245,10 +249,10 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512BW-FCP-LABEL: load_i32_stride2_vf4:
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vmovaps (%rdi), %xmm1
-; AVX512BW-FCP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,3],mem[1,3]
+; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,0,0,0,0]
+; AVX512BW-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512BW-FCP-NEXT: vpmovqd %ymm0, (%rsi)
-; AVX512BW-FCP-NEXT: vmovaps %xmm1, (%rdx)
+; AVX512BW-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
@@ -265,10 +269,10 @@ define void @load_i32_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512DQ-BW-FCP-LABEL: load_i32_stride2_vf4:
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,3],mem[1,3]
+; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,0,0,0,0]
+; AVX512DQ-BW-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm1
; AVX512DQ-BW-FCP-NEXT: vpmovqd %ymm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <8 x i32>, ptr %in.vec, align 64
@@ -352,11 +356,10 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512-LABEL: load_i32_stride2_vf8:
; AVX512: # %bb.0:
; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512-NEXT: vmovaps (%rdi), %ymm1
-; AVX512-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,3],mem[1,3],ymm1[5,7],mem[5,7]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -364,20 +367,19 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT: vpermps (%rdi), %zmm1, %zmm1
+; AVX512-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512-FCP-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i32_stride2_vf8:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-NEXT: vmovaps (%rdi), %ymm1
-; AVX512DQ-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,3],mem[1,3],ymm1[5,7],mem[5,7]
-; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512DQ-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512DQ-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512DQ-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -385,20 +387,19 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT: vpermps (%rdi), %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512DQ-FCP-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i32_stride2_vf8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT: vmovaps (%rdi), %ymm1
-; AVX512BW-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,3],mem[1,3],ymm1[5,7],mem[5,7]
-; AVX512BW-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512BW-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512BW-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512BW-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512BW-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -406,20 +407,19 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; AVX512BW-FCP-NEXT: vpermps (%rdi), %zmm1, %zmm1
+; AVX512BW-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512BW-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i32_stride2_vf8:
; AVX512DQ-BW: # %bb.0:
; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-BW-NEXT: vmovaps (%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,3],mem[1,3],ymm1[5,7],mem[5,7]
-; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX512DQ-BW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; AVX512DQ-BW-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512DQ-BW-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512DQ-BW-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
@@ -427,9 +427,9 @@ define void @load_i32_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; AVX512DQ-BW-FCP-NEXT: vpermps (%rdi), %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT: vpermd %zmm0, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpmovqd %zmm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <16 x i32>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-2.ll
index aa7d8ceb14950..6833bc4063d5f 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-2.ll
@@ -40,112 +40,112 @@ define void @load_i64_stride2_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX2-LABEL: load_i64_stride2_vf2:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX2-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX2-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX2-NEXT: vmovaps %xmm2, (%rsi)
+; AVX2-NEXT: vmovaps (%rdi), %ymm0
+; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX2-NEXT: vmovaps %xmm1, (%rsi)
; AVX2-NEXT: vmovaps %xmm0, (%rdx)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i64_stride2_vf2:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-FP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX2-FP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX2-FP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX2-FP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX2-FP-NEXT: vmovaps (%rdi), %ymm0
+; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX2-FP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX2-FP-NEXT: vmovaps %xmm1, (%rsi)
; AVX2-FP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX2-FP-NEXT: vzeroupper
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i64_stride2_vf2:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX2-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX2-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX2-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX2-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX2-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i64_stride2_vf2:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovaps (%rdi), %xmm0
-; AVX512-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512-NEXT: vmovaps (%rdi), %ymm0
+; AVX512-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512-NEXT: vmovaps %xmm1, (%rsi)
; AVX512-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i64_stride2_vf2:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX512-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX512-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX512-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i64_stride2_vf2:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vmovaps (%rdi), %xmm0
-; AVX512DQ-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512DQ-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512DQ-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512DQ-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512DQ-NEXT: vmovaps (%rdi), %ymm0
+; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512DQ-NEXT: vmovaps %xmm1, (%rsi)
; AVX512DQ-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i64_stride2_vf2:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512DQ-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512DQ-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512DQ-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512DQ-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX512DQ-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i64_stride2_vf2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovaps (%rdi), %xmm0
-; AVX512BW-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512BW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512BW-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512BW-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512BW-NEXT: vmovaps (%rdi), %ymm0
+; AVX512BW-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512BW-NEXT: vmovaps %xmm1, (%rsi)
; AVX512BW-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: load_i64_stride2_vf2:
; AVX512BW-FCP: # %bb.0:
-; AVX512BW-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX512BW-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512BW-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512BW-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512BW-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512BW-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX512BW-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512BW-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512BW-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX512BW-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i64_stride2_vf2:
; AVX512DQ-BW: # %bb.0:
-; AVX512DQ-BW-NEXT: vmovaps (%rdi), %xmm0
-; AVX512DQ-BW-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512DQ-BW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512DQ-BW-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512DQ-BW-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512DQ-BW-NEXT: vmovaps (%rdi), %ymm0
+; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512DQ-BW-NEXT: vmovaps %xmm1, (%rsi)
; AVX512DQ-BW-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: load_i64_stride2_vf2:
; AVX512DQ-BW-FCP: # %bb.0:
-; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %xmm0
-; AVX512DQ-BW-FCP-NEXT: vmovaps 16(%rdi), %xmm1
-; AVX512DQ-BW-FCP-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm1[0]
-; AVX512DQ-BW-FCP-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm2, (%rsi)
+; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %ymm0
+; AVX512DQ-BW-FCP-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,2,2,3]
+; AVX512DQ-BW-FCP-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm1, (%rsi)
; AVX512DQ-BW-FCP-NEXT: vmovaps %xmm0, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <4 x i64>, ptr %in.vec, align 64
%strided.vec0 = shufflevector <4 x i64> %wide.vec, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
@@ -231,101 +231,97 @@ define void @load_i64_stride2_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512-LABEL: load_i64_stride2_vf4:
; AVX512: # %bb.0:
-; AVX512-NEXT: vmovaps (%rdi), %ymm0
-; AVX512-NEXT: vmovaps 32(%rdi), %ymm1
-; AVX512-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512-NEXT: vmovaps %ymm2, (%rsi)
+; AVX512-NEXT: vmovaps (%rdi), %zmm0
+; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512-NEXT: vpermpd %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
+; AVX512-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: vmovaps %ymm1, (%rsi)
; AVX512-NEXT: vmovaps %ymm0, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i64_stride2_vf4:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,2,4,6]
-; AVX512-FCP-NEXT: vmovaps (%rdi), %zmm1
-; AVX512-FCP-NEXT: vpermpd %zmm1, %zmm0, %zmm0
+; AVX512-FCP-NEXT: vmovaps (%rdi), %zmm0
+; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm1
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
-; AVX512-FCP-NEXT: vpermpd %zmm1, %zmm2, %zmm1
-; AVX512-FCP-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512-FCP-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512-FCP-NEXT: vmovaps %ymm1, (%rsi)
+; AVX512-FCP-NEXT: vmovaps %ymm0, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i64_stride2_vf4:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vmovaps (%rdi), %ymm0
-; AVX512DQ-NEXT: vmovaps 32(%rdi), %ymm1
-; AVX512DQ-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512DQ-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512DQ-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512DQ-NEXT: vmovaps %ymm2, (%rsi)
+; AVX512DQ-NEXT: vmovaps (%rdi), %zmm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-NEXT: vpermpd %zmm0, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
+; AVX512DQ-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512DQ-NEXT: vmovaps %ymm1, (%rsi)
; AVX512DQ-NEXT: vmovaps %ymm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i64_stride2_vf4:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,2,4,6]
-; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %zmm1
-; AVX512DQ-FCP-NEXT: vpermpd %zmm1, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT: vmovaps (%rdi), %zmm0
+; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm1
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
-; AVX512DQ-FCP-NEXT: vpermpd %zmm1, %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512DQ-FCP-NEXT: vmovaps %ymm1, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovaps %ymm0, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i64_stride2_vf4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovaps (%rdi), %ymm0
-; AVX512BW-NEXT: vmovaps 32(%rdi), %ymm1
-; AVX512BW-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512BW-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512BW-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BW-NEXT: vmovaps %ymm2, (%rsi)
+; AVX512BW-NEXT: vmovaps (%rdi), %zmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-NEXT: vpermpd %zmm0, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
+; AVX512BW-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vmovaps %ymm1, (%rsi)
; AVX512BW-NEXT: vmovaps %ymm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: load_i64_stride2_vf4:
; AVX512BW-FCP: # %bb.0:
-; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,2,4,6]
-; AVX512BW-FCP-NEXT: vmovaps (%rdi), %zmm1
-; AVX512BW-FCP-NEXT: vpermpd %zmm1, %zmm0, %zmm0
+; AVX512BW-FCP-NEXT: vmovaps (%rdi), %zmm0
+; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
-; AVX512BW-FCP-NEXT: vpermpd %zmm1, %zmm2, %zmm1
-; AVX512BW-FCP-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512BW-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512BW-FCP-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512BW-FCP-NEXT: vmovaps %ymm1, (%rsi)
+; AVX512BW-FCP-NEXT: vmovaps %ymm0, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i64_stride2_vf4:
; AVX512DQ-BW: # %bb.0:
-; AVX512DQ-BW-NEXT: vmovaps (%rdi), %ymm0
-; AVX512DQ-BW-NEXT: vmovaps 32(%rdi), %ymm1
-; AVX512DQ-BW-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
-; AVX512DQ-BW-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512DQ-BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512DQ-BW-NEXT: vmovaps %ymm2, (%rsi)
+; AVX512DQ-BW-NEXT: vmovaps (%rdi), %zmm0
+; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-BW-NEXT: vpermpd %zmm0, %zmm1, %zmm1
+; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
+; AVX512DQ-BW-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512DQ-BW-NEXT: vmovaps %ymm1, (%rsi)
; AVX512DQ-BW-NEXT: vmovaps %ymm0, (%rdx)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: load_i64_stride2_vf4:
; AVX512DQ-BW-FCP: # %bb.0:
-; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,2,4,6]
-; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %zmm1
-; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm1, %zmm0, %zmm0
+; AVX512DQ-BW-FCP-NEXT: vmovaps (%rdi), %zmm0
+; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm0, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,3,5,7]
-; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm1, %zmm2, %zmm1
-; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vpermpd %zmm0, %zmm2, %zmm0
+; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm1, (%rsi)
+; AVX512DQ-BW-FCP-NEXT: vmovaps %ymm0, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <8 x i64>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
index b609299e5f757..b765cda89da01 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
@@ -416,165 +416,141 @@ define void @load_i8_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i8_stride2_vf16:
; AVX: # %bb.0:
-; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i8_stride2_vf16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i8_stride2_vf16:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-FP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-FP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX2-FP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-FP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-FP-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX2-FP-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-FP-NEXT: vzeroupper
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i8_stride2_vf16:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i8_stride2_vf16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i8_stride2_vf16:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i8_stride2_vf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512DQ-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512DQ-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512DQ-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512DQ-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX512DQ-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i8_stride2_vf16:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512DQ-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512DQ-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
+; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rdx)
+; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i8_stride2_vf16:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT: vpsrlw $8, %ymm0, %ymm1
+; AVX512BW-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX512BW-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512BW-NEXT: vpmovwb %ymm1, (%rdx)
+; AVX512BW-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: load_i8_stride2_vf16:
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512BW-FCP-NEXT: vpsrlw $8, %ymm0, %ymm1
+; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX512BW-FCP-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512BW-FCP-NEXT: vpmovwb %ymm1, (%rdx)
+; AVX512BW-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i8_stride2_vf16:
; AVX512DQ-BW: # %bb.0:
; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-BW-NEXT: vpsrlw $8, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX512DQ-BW-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512DQ-BW-NEXT: vpmovwb %ymm1, (%rdx)
+; AVX512DQ-BW-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: load_i8_stride2_vf16:
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-BW-FCP-NEXT: vpsrlw $8, %ymm0, %ymm1
+; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]
; AVX512DQ-BW-FCP-NEXT: vpmovwb %ymm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vpmovwb %ymm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm1, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <32 x i8>, ptr %in.vec, align 64
@@ -616,28 +592,28 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i8_stride2_vf32:
; AVX: # %bb.0:
-; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vmovdqa 32(%rdi), %xmm3
-; AVX-NEXT: vmovdqa 48(%rdi), %xmm4
-; AVX-NEXT: vpand %xmm0, %xmm4, %xmm5
-; AVX-NEXT: vpand %xmm0, %xmm3, %xmm6
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vmovdqa 32(%rdi), %xmm2
+; AVX-NEXT: vmovdqa 48(%rdi), %xmm3
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; AVX-NEXT: vpand %xmm4, %xmm3, %xmm5
+; AVX-NEXT: vpand %xmm4, %xmm2, %xmm6
; AVX-NEXT: vpackuswb %xmm5, %xmm6, %xmm5
-; AVX-NEXT: vpand %xmm0, %xmm2, %xmm6
-; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX-NEXT: vpackuswb %xmm6, %xmm0, %xmm0
+; AVX-NEXT: vpand %xmm4, %xmm1, %xmm6
+; AVX-NEXT: vpand %xmm4, %xmm0, %xmm4
+; AVX-NEXT: vpackuswb %xmm6, %xmm4, %xmm4
; AVX-NEXT: vmovq {{.*#+}} xmm6 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX-NEXT: vpshufb %xmm6, %xmm4, %xmm4
; AVX-NEXT: vpshufb %xmm6, %xmm3, %xmm3
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
; AVX-NEXT: vpshufb %xmm6, %xmm2, %xmm2
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
; AVX-NEXT: vpshufb %xmm6, %xmm1, %xmm1
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
+; AVX-NEXT: vpshufb %xmm6, %xmm0, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX-NEXT: vmovdqa %xmm4, (%rsi)
; AVX-NEXT: vmovdqa %xmm5, 16(%rsi)
-; AVX-NEXT: vmovaps %ymm1, (%rdx)
+; AVX-NEXT: vmovaps %ymm0, (%rdx)
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
;
@@ -700,20 +676,20 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512-LABEL: load_i8_stride2_vf32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512-NEXT: vpshufb %ymm0, %ymm1, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
-; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -735,20 +711,20 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512DQ-LABEL: load_i8_stride2_vf32:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT: vpshufb %ymm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512DQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512DQ-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
@@ -771,36 +747,44 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
; AVX512BW-LABEL: load_i8_stride2_vf32:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm1
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-NEXT: vpshufb {{.*#+}} zmm2 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT: vpermq %zmm2, %zmm1, %zmm1
; AVX512BW-NEXT: vpmovwb %zmm0, (%rsi)
-; AVX512BW-NEXT: vpmovwb %zmm1, (%rdx)
+; AVX512BW-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BW-FCP-LABEL: load_i8_stride2_vf32:
; AVX512BW-FCP: # %bb.0:
; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FCP-NEXT: vpsrlw $8, %zmm0, %zmm1
+; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} zmm2 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512BW-FCP-NEXT: vpermq %zmm2, %zmm1, %zmm1
; AVX512BW-FCP-NEXT: vpmovwb %zmm0, (%rsi)
-; AVX512BW-FCP-NEXT: vpmovwb %zmm1, (%rdx)
+; AVX512BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512BW-FCP-NEXT: vzeroupper
; AVX512BW-FCP-NEXT: retq
;
; AVX512DQ-BW-LABEL: load_i8_stride2_vf32:
; AVX512DQ-BW: # %bb.0:
; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-BW-NEXT: vpsrlw $8, %zmm0, %zmm1
+; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} zmm2 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-NEXT: vpermq %zmm2, %zmm1, %zmm1
; AVX512DQ-BW-NEXT: vpmovwb %zmm0, (%rsi)
-; AVX512DQ-BW-NEXT: vpmovwb %zmm1, (%rdx)
+; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-BW-NEXT: vzeroupper
; AVX512DQ-BW-NEXT: retq
;
; AVX512DQ-BW-FCP-LABEL: load_i8_stride2_vf32:
; AVX512DQ-BW-FCP: # %bb.0:
; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-BW-FCP-NEXT: vpsrlw $8, %zmm0, %zmm1
+; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,4,6]
+; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} zmm2 = zmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u,33,35,37,39,41,43,45,47,u,u,u,u,u,u,u,u,49,51,53,55,57,59,61,63,u,u,u,u,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT: vpermq %zmm2, %zmm1, %zmm1
; AVX512DQ-BW-FCP-NEXT: vpmovwb %zmm0, (%rsi)
-; AVX512DQ-BW-FCP-NEXT: vpmovwb %zmm1, (%rdx)
+; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, (%rdx)
; AVX512DQ-BW-FCP-NEXT: vzeroupper
; AVX512DQ-BW-FCP-NEXT: retq
%wide.vec = load <64 x i8>, ptr %in.vec, align 64
@@ -1012,123 +996,123 @@ define void @load_i8_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512-LABEL: load_i8_stride2_vf64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX512-NEXT: vpshufb %ymm0, %ymm4, %ymm5
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-NEXT: vmovdqa 64(%rdi), %ymm2
+; AVX512-NEXT: vmovdqa 96(%rdi), %ymm3
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-NEXT: vpshufb %ymm4, %ymm3, %ymm5
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512-NEXT: vpshufb %ymm6, %ymm2, %ymm7
; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm0
-; AVX512-NEXT: vpshufb %ymm6, %ymm1, %ymm6
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5],ymm0[6,7]
-; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0
-; AVX512-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512-NEXT: vpshufb %ymm4, %ymm1, %ymm4
+; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm6
+; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]
+; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512-NEXT: vpermq {{.*#+}} zmm4 = zmm4[0,2,1,3,4,6,5,7]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX512-NEXT: vpshufb %ymm5, %ymm3, %ymm3
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-NEXT: vpshufb %ymm6, %ymm3, %ymm3
-; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
-; AVX512-NEXT: vpshufb %ymm5, %ymm2, %ymm2
-; AVX512-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,2,1,3,4,6,5,7]
-; AVX512-NEXT: vmovdqa64 %zmm0, (%rsi)
-; AVX512-NEXT: vmovdqa64 %zmm1, (%rdx)
+; AVX512-NEXT: vpshufb %ymm6, %ymm2, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm0
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i8_stride2_vf64:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm5
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm3
+; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm7
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,2,5,7]
; AVX512-FCP-NEXT: vpermt2q %ymm5, %ymm8, %ymm7
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm0
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm5
-; AVX512-FCP-NEXT: vpermt2q %ymm0, %ymm8, %ymm5
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm0
+; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm4
+; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm5
+; AVX512-FCP-NEXT: vpermt2q %ymm4, %ymm8, %ymm5
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm4
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm3
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
-; AVX512-FCP-NEXT: vpermt2q %ymm4, %ymm8, %ymm3
-; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm2
-; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-FCP-NEXT: vpermt2q %ymm2, %ymm8, %ymm1
-; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rsi)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm1, (%rdx)
+; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
+; AVX512-FCP-NEXT: vpermt2q %ymm3, %ymm8, %ymm2
+; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
+; AVX512-FCP-NEXT: vpermt2q %ymm1, %ymm8, %ymm0
+; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-FCP-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i8_stride2_vf64:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm4, %ymm5
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm2
+; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm3
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-NEXT: vpshufb %ymm4, %ymm3, %ymm5
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512DQ-NEXT: vpshufb %ymm6, %ymm2, %ymm7
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-NEXT: vpshufb %ymm6, %ymm1, %ymm6
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5],ymm0[6,7]
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512DQ-NEXT: vpshufb %ymm4, %ymm1, %ymm4
+; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm6
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512DQ-NEXT: vpermq {{.*#+}} zmm4 = zmm4[0,2,1,3,4,6,5,7]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX512DQ-NEXT: vpshufb %ymm5, %ymm3, %ymm3
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-NEXT: vpshufb %ymm6, %ymm3, %ymm3
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
-; AVX512DQ-NEXT: vpshufb %ymm5, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,2,1,3,4,6,5,7]
-; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa64 %zmm1, (%rdx)
+; AVX512DQ-NEXT: vpshufb %ymm6, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512DQ-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i8_stride2_vf64:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm4
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm3
+; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm7
+; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm7
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm8 = [0,2,5,7]
; AVX512DQ-FCP-NEXT: vpermt2q %ymm5, %ymm8, %ymm7
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm5
-; AVX512DQ-FCP-NEXT: vpermt2q %ymm0, %ymm8, %ymm5
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm0
+; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm4
+; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm5
+; AVX512DQ-FCP-NEXT: vpermt2q %ymm4, %ymm8, %ymm5
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm4
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm3
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT: vpermt2q %ymm4, %ymm8, %ymm3
-; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vpermt2q %ymm2, %ymm8, %ymm1
-; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT: vpermt2q %ymm3, %ymm8, %ymm2
+; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT: vpermt2q %ymm1, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
index 995607a6857bd..fb051be560f0d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -412,19 +412,12 @@ define <16 x i32> @shuffle_v16i32_0_1_2_19_u_u_u_u_u_u_u_u_u_u_u_u(<16 x i32> %a
;FIXME: can do better with vpcompress
define <8 x i32> @test_v16i32_1_3_5_7_9_11_13_15(<16 x i32> %v) {
-; SLOW-LABEL: test_v16i32_1_3_5_7_9_11_13_15:
-; SLOW: # %bb.0:
-; SLOW-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; SLOW-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; SLOW-NEXT: retq
-;
-; FAST-LABEL: test_v16i32_1_3_5_7_9_11_13_15:
-; FAST: # %bb.0:
-; FAST-NEXT: vmovaps {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
-; FAST-NEXT: vpermps %zmm0, %zmm1, %zmm0
-; FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
-; FAST-NEXT: retq
+; ALL-LABEL: test_v16i32_1_3_5_7_9_11_13_15:
+; ALL: # %bb.0:
+; ALL-NEXT: vmovaps {{.*#+}} ymm1 = [1,3,5,7,9,11,13,15]
+; ALL-NEXT: vpermps %zmm0, %zmm1, %zmm0
+; ALL-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
+; ALL-NEXT: retq
%res = shufflevector <16 x i32> %v, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
ret <8 x i32> %res
}
diff --git a/llvm/test/CodeGen/X86/vpdpwssd.ll b/llvm/test/CodeGen/X86/vpdpwssd.ll
index ea97800505bc2..339c04611e318 100644
--- a/llvm/test/CodeGen/X86/vpdpwssd.ll
+++ b/llvm/test/CodeGen/X86/vpdpwssd.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI,AVX512VL-VNNI-SLOW
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512-VNNI
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI,AVX512VL-VNNI-FAST
define <16 x i32> @vpdpwssd_test(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) {
; CHECK-LABEL: vpdpwssd_test:
@@ -31,23 +31,53 @@ define <16 x i32> @vpdpwssd_v16i32_accumulate(<32 x i16> %a0, <32 x i16> %a1, <1
}
define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x i32> %a2) {
-; AVX512VL-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
-; AVX512VL-VNNI: # %bb.0:
-; AVX512VL-VNNI-NEXT: vpdpwssd %ymm1, %ymm0, %ymm2
-; AVX512VL-VNNI-NEXT: vmovdqa %ymm2, %ymm0
-; AVX512VL-VNNI-NEXT: retq
+; AVX512VL-VNNI-SLOW-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI-SLOW: # %bb.0:
+; AVX512VL-VNNI-SLOW-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512VL-VNNI-SLOW-NEXT: vmovdqa {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
+; AVX512VL-VNNI-SLOW-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512VL-VNNI-SLOW-NEXT: vpermd %zmm0, %zmm3, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512VL-VNNI-SLOW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512VL-VNNI-SLOW-NEXT: retq
;
; AVX-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
; AVX-VNNI: # %bb.0:
-; AVX-VNNI-NEXT: {vex} vpdpwssd %ymm1, %ymm0, %ymm2
-; AVX-VNNI-NEXT: vmovdqa %ymm2, %ymm0
+; AVX-VNNI-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX-VNNI-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX-VNNI-NEXT: vmovdqa {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
+; AVX-VNNI-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX-VNNI-NEXT: vpmovqd %zmm0, %ymm1
+; AVX-VNNI-NEXT: vpermd %zmm0, %zmm3, %zmm0
+; AVX-VNNI-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX-VNNI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX-VNNI-NEXT: retq
;
; AVX512-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
; AVX512-VNNI: # %bb.0:
-; AVX512-VNNI-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-VNNI-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-VNNI-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-VNNI-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-VNNI-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512-VNNI-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
+; AVX512-VNNI-NEXT: vpermd %zmm0, %zmm3, %zmm0
+; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512-VNNI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX512-VNNI-NEXT: retq
+;
+; AVX512VL-VNNI-FAST-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI-FAST: # %bb.0:
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512VL-VNNI-FAST-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
+; AVX512VL-VNNI-FAST-NEXT: vpermd %zmm0, %zmm3, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512VL-VNNI-FAST-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512VL-VNNI-FAST-NEXT: retq
%x0 = sext <16 x i16> %a0 to <16 x i32>
%x1 = sext <16 x i16> %a1 to <16 x i32>
%m = mul nsw <16 x i32> %x0, %x1
@@ -61,20 +91,41 @@ define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x
define <4 x i32> @vpdpwssd_v4i32_accumulate(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
; AVX512VL-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX512VL-VNNI: # %bb.0:
-; AVX512VL-VNNI-NEXT: vpdpwssd %xmm1, %xmm0, %xmm2
-; AVX512VL-VNNI-NEXT: vmovdqa %xmm2, %xmm0
+; AVX512VL-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512VL-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512VL-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX512VL-VNNI-NEXT: vpmovqd %ymm0, %xmm1
+; AVX512VL-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512VL-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
+; AVX512VL-VNNI-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512VL-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512VL-VNNI-NEXT: vzeroupper
; AVX512VL-VNNI-NEXT: retq
;
; AVX-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX-VNNI: # %bb.0:
-; AVX-VNNI-NEXT: {vex} vpdpwssd %xmm1, %xmm0, %xmm2
-; AVX-VNNI-NEXT: vmovdqa %xmm2, %xmm0
+; AVX-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX-VNNI-NEXT: vpmovqd %ymm0, %xmm1
+; AVX-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
+; AVX-VNNI-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX-VNNI-NEXT: vzeroupper
; AVX-VNNI-NEXT: retq
;
; AVX512-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX512-VNNI: # %bb.0:
-; AVX512-VNNI-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} xmm3 = xmm0[0,2],xmm1[0,2]
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
+; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm3, %xmm1
+; AVX512-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-VNNI-NEXT: vzeroupper
; AVX512-VNNI-NEXT: retq
%x0 = sext <8 x i16> %a0 to <8 x i32>
%x1 = sext <8 x i16> %a1 to <8 x i32>
- Previous message: [llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
- Next message: [llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the llvm-commits
mailing list