[llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 27 07:19:35 PDT 2026
- Previous message: [llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
- Next message: [llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/180573
>From 2f8b119d42399f16021924aef51f2b6a2cf3efb9 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 9 Feb 2026 17:46:25 +0000
Subject: [PATCH] [DAG] SelectionDAGBuilder::visitShuffleVector - split
shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns
If an unary shuffle sources are twice the width of the result, then split the lower/upper subvectors and perform a binary shuffle instead of scalarizing
Fixes #88030
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 34 ++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 14 +-
llvm/test/CodeGen/AArch64/addp-shuffle.ll | 57 +-
...ffer-fat-pointers-contents-legalization.ll | 44 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 288 +++++----
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 186 +++---
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 239 ++++----
.../AMDGPU/load-local-redundant-copies.ll | 33 +-
llvm/test/CodeGen/Thumb2/mve-shuffle.ll | 8 +-
.../any_extend_vector_inreg_of_broadcast.ll | 164 +-----
...d_vector_inreg_of_broadcast_from_memory.ll | 88 +--
llvm/test/CodeGen/X86/madd.ll | 373 ++++++++++--
llvm/test/CodeGen/X86/pmaddubsw.ll | 361 +++++++++++-
.../test/CodeGen/X86/setcc-non-simple-type.ll | 22 +-
.../X86/shuffle-strided-with-offset-256.ll | 25 +-
llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll | 547 +++---------------
llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll | 82 +--
.../vector-interleaved-load-i16-stride-2.ll | 80 +--
.../vector-interleaved-load-i8-stride-2.ll | 208 +++----
llvm/test/CodeGen/X86/vector-narrow-binop.ll | 4 +-
llvm/test/CodeGen/X86/vpdpwssd.ll | 89 ++-
llvm/test/CodeGen/X86/vselect-avx.ll | 10 +-
22 files changed, 1469 insertions(+), 1487 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index bacea075da00c..5614a013afeea 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4280,6 +4280,40 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
assert(SrcNumElts > MaskNumElts);
+ // See if we can recreate a binary shuffle by splitting or widening an unary
+ // shuffle with sources twice the destination size.
+ if (SrcNumElts == (MaskNumElts * 2) && Src2.isUndef() &&
+ TLI.isTypeLegal(VT)) {
+ SmallVector<int, 16> SplitMask(Mask.begin(), Mask.end());
+ for (int &M : SplitMask)
+ M = M >= (int)SrcNumElts ? -1 : M;
+ bool LoOnly = all_of(SplitMask,
+ [MaskNumElts](int M) { return M < (int)MaskNumElts; });
+ bool HiOnly = all_of(SplitMask, [MaskNumElts](int M) {
+ return M < 0 || M >= (int)MaskNumElts;
+ });
+ if (LoOnly || HiOnly || Src1.getOpcode() == ISD::CONCAT_VECTORS ||
+ isa<LoadSDNode>(Src1) ||
+ (TLI.isExtractSubvectorCheap(VT, SrcVT, 0) &&
+ TLI.isExtractSubvectorCheap(VT, SrcVT, MaskNumElts))) {
+ SDValue LHS = DAG.getExtractSubvector(DL, VT, Src1, 0);
+ SDValue RHS = DAG.getExtractSubvector(DL, VT, Src1, MaskNumElts);
+ SDValue Result = DAG.getVectorShuffle(VT, DL, LHS, RHS, SplitMask);
+ setValue(&I, Result);
+ return;
+ }
+ if (TLI.isTypeLegal(SrcVT)) {
+ SmallVector<int, 16> WideMask(Mask.begin(), Mask.end());
+ WideMask.append(SrcNumElts - MaskNumElts, -1);
+ if (TLI.isShuffleMaskLegal(WideMask, SrcVT.getSimpleVT())) {
+ SDValue Wide = DAG.getVectorShuffle(SrcVT, DL, Src1, Src2, WideMask);
+ SDValue Result = DAG.getExtractSubvector(DL, VT, Wide, 0);
+ setValue(&I, Result);
+ return;
+ }
+ }
+ }
+
// Analyze the access pattern of the vector to see if we can extract
// two subvectors and do the shuffle.
int StartIdx[2] = {-1, -1}; // StartIdx to extract from
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f19b543ffed0d..8fbfe08cc220f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -11036,13 +11036,13 @@ multiclass PairwiseZIPBinOp<SDPatternOperator Op, ValueType VT,
}
// add(uzp1(X, Y), uzp2(X, Y)) -> addp(X, Y)
-defm : PairwiseZIPBinOp<add, v2i64, FPR128, ADDPv2i64>;
-defm : PairwiseZIPBinOp<add, v2i32, FPR64, ADDPv2i32>;
-defm : PairwiseUZPBinOp<add, v4i32, FPR128, ADDPv4i32>;
-defm : PairwiseUZPBinOp<add, v4i16, FPR64, ADDPv4i16>;
-defm : PairwiseUZPBinOp<add, v8i16, FPR128, ADDPv8i16>;
-defm : PairwiseUZPBinOp<add, v8i8, FPR64, ADDPv8i8>;
-defm : PairwiseUZPBinOp<add, v16i8, FPR128, ADDPv16i8>;
+defm : PairwiseZIPBinOp<add_like, v2i64, FPR128, ADDPv2i64>;
+defm : PairwiseZIPBinOp<add_like, v2i32, FPR64, ADDPv2i32>;
+defm : PairwiseUZPBinOp<add_like, v4i32, FPR128, ADDPv4i32>;
+defm : PairwiseUZPBinOp<add_like, v4i16, FPR64, ADDPv4i16>;
+defm : PairwiseUZPBinOp<add_like, v8i16, FPR128, ADDPv8i16>;
+defm : PairwiseUZPBinOp<add_like, v8i8, FPR64, ADDPv8i8>;
+defm : PairwiseUZPBinOp<add_like, v16i8, FPR128, ADDPv16i8>;
def : Pat<(v2i32 (add (AArch64zip1 (extract_subvector (v4i32 FPR128:$Rn), (i64 0)),
(extract_subvector (v4i32 FPR128:$Rn), (i64 2))),
diff --git a/llvm/test/CodeGen/AArch64/addp-shuffle.ll b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
index cf02a090b8e3c..4c8801f55c000 100644
--- a/llvm/test/CodeGen/AArch64/addp-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
@@ -361,9 +361,7 @@ entry:
define <4 x i32> @or_deinterleave_shuffle_v8i32(<8 x i32> %a) {
; CHECK-LABEL: or_deinterleave_shuffle_v8i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: addp v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
%r0 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%r1 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -374,9 +372,7 @@ define <4 x i32> @or_deinterleave_shuffle_v8i32(<8 x i32> %a) {
define <4 x i32> @or_deinterleave_shuffle_v8i32_c(<8 x i32> %a) {
; CHECK-LABEL: or_deinterleave_shuffle_v8i32_c:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: orr v0.16b, v0.16b, v2.16b
+; CHECK-NEXT: addp v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
%r0 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%r1 = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -388,9 +384,7 @@ define <2 x i32> @or_deinterleave_shuffle_v4i32(<4 x i32> %a) {
; CHECK-LABEL: or_deinterleave_shuffle_v4i32:
; CHECK: // %bb.0:
; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-NEXT: addp v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ret
%r0 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
%r1 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
@@ -401,9 +395,7 @@ define <2 x i32> @or_deinterleave_shuffle_v4i32(<4 x i32> %a) {
define <8 x i16> @or_deinterleave_shuffle_v16i16(<16 x i16> %a) {
; CHECK-LABEL: or_deinterleave_shuffle_v16i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v2.8h, v0.8h, v1.8h
-; CHECK-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: addp v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ret
%r0 = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
%r1 = shufflevector <16 x i16> %a, <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -427,9 +419,7 @@ define <4 x i16> @or_deinterleave_shuffle_v8i16(<8 x i16> %a) {
define <16 x i8> @or_deinterleave_shuffle_v32i8(<32 x i8> %a) {
; CHECK-LABEL: or_deinterleave_shuffle_v32i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v2.16b, v0.16b, v1.16b
-; CHECK-NEXT: uzp2 v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%r0 = shufflevector <32 x i8> %a, <32 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%r1 = shufflevector <32 x i8> %a, <32 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
@@ -453,12 +443,9 @@ define <8 x i8> @or_deinterleave_shuffle_v16i8(<16 x i8> %a) {
define <4 x i64> @or_deinterleave_shuffle_v8i64(<8 x i64> %a) {
; CHECK-LABEL: or_deinterleave_shuffle_v8i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v4.2d, v2.2d, v3.2d
-; CHECK-NEXT: zip1 v5.2d, v0.2d, v1.2d
-; CHECK-NEXT: zip2 v2.2d, v2.2d, v3.2d
-; CHECK-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-NEXT: orr v1.16b, v4.16b, v2.16b
-; CHECK-NEXT: orr v0.16b, v5.16b, v0.16b
+; CHECK-NEXT: addp v2.2d, v2.2d, v3.2d
+; CHECK-NEXT: addp v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: mov v1.16b, v2.16b
; CHECK-NEXT: ret
%r0 = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%r1 = shufflevector <8 x i64> %a, <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -469,9 +456,7 @@ define <4 x i64> @or_deinterleave_shuffle_v8i64(<8 x i64> %a) {
define <8 x i8> @or_manual_addp_v8i8(<8 x i8> %a, <8 x i8> %b) {
; CHECK-LABEL: or_manual_addp_v8i8:
; CHECK: // %bb.0: // %start
-; CHECK-NEXT: uzp1 v2.8b, v0.8b, v1.8b
-; CHECK-NEXT: uzp2 v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-NEXT: addp v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ret
start:
%0 = shufflevector <8 x i8> %a, <8 x i8> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -483,9 +468,7 @@ start:
define <4 x i16> @or_manual_addp_v4i16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-LABEL: or_manual_addp_v4i16:
; CHECK: // %bb.0: // %start
-; CHECK-NEXT: uzp1 v2.4h, v0.4h, v1.4h
-; CHECK-NEXT: uzp2 v0.4h, v0.4h, v1.4h
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-NEXT: addp v0.4h, v0.4h, v1.4h
; CHECK-NEXT: ret
start:
%0 = shufflevector <4 x i16> %a, <4 x i16> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -497,9 +480,7 @@ start:
define <2 x i32> @or_manual_addp_v2i32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-LABEL: or_manual_addp_v2i32:
; CHECK: // %bb.0: // %start
-; CHECK-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-NEXT: addp v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ret
start:
%0 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 2>
@@ -511,9 +492,7 @@ start:
define <16 x i8> @or_manual_addp_v16i8(<16 x i8> %a, <16 x i8> %b) {
; CHECK-LABEL: or_manual_addp_v16i8:
; CHECK: // %bb.0: // %start
-; CHECK-NEXT: uzp1 v2.16b, v0.16b, v1.16b
-; CHECK-NEXT: uzp2 v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: addp v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
start:
%0 = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -525,9 +504,7 @@ start:
define <8 x i16> @or_manual_addp_v8i16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-LABEL: or_manual_addp_v8i16:
; CHECK: // %bb.0: // %start
-; CHECK-NEXT: uzp1 v2.8h, v0.8h, v1.8h
-; CHECK-NEXT: uzp2 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: addp v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ret
start:
%0 = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -539,9 +516,7 @@ start:
define <4 x i32> @or_manual_addp_v4i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: or_manual_addp_v4i32:
; CHECK: // %bb.0: // %start
-; CHECK-NEXT: uzp1 v2.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: addp v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
start:
%0 = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -553,9 +528,7 @@ start:
define <2 x i64> @or_manual_addp_v2i64(<2 x i64> %a, <2 x i64> %b) {
; CHECK-LABEL: or_manual_addp_v2i64:
; CHECK: // %bb.0: // %start
-; CHECK-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-NEXT: zip2 v0.2d, v0.2d, v1.2d
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: addp v0.2d, v0.2d, v1.2d
; CHECK-NEXT: ret
start:
%0 = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 2>
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index 021b15ef09bee..8d18c6dbd9339 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2872,34 +2872,34 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: s_mov_b32 s4, 0xc0c0004
-; SDAG-NEXT: v_perm_b32 v8, v8, v9, s4
-; SDAG-NEXT: v_perm_b32 v9, v10, v11, s4
-; SDAG-NEXT: buffer_load_ubyte v10, off, s[0:3], s32
+; SDAG-NEXT: v_perm_b32 v20, v20, v21, s4
+; SDAG-NEXT: v_perm_b32 v21, v22, v23, s4
+; SDAG-NEXT: buffer_load_ubyte v22, off, s[0:3], s32
; SDAG-NEXT: v_perm_b32 v12, v12, v13, s4
; SDAG-NEXT: v_perm_b32 v13, v14, v15, s4
+; SDAG-NEXT: v_perm_b32 v8, v8, v9, s4
+; SDAG-NEXT: v_perm_b32 v9, v10, v11, s4
; SDAG-NEXT: v_perm_b32 v4, v4, v5, s4
; SDAG-NEXT: v_perm_b32 v5, v6, v7, s4
-; SDAG-NEXT: v_perm_b32 v0, v0, v1, s4
-; SDAG-NEXT: v_perm_b32 v6, v2, v3, s4
-; SDAG-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; SDAG-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; SDAG-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; SDAG-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; SDAG-NEXT: v_perm_b32 v7, v28, v29, s4
-; SDAG-NEXT: v_perm_b32 v11, v24, v25, s4
-; SDAG-NEXT: v_perm_b32 v14, v26, v27, s4
-; SDAG-NEXT: v_perm_b32 v15, v20, v21, s4
-; SDAG-NEXT: v_perm_b32 v20, v22, v23, s4
+; SDAG-NEXT: v_perm_b32 v10, v0, v1, s4
+; SDAG-NEXT: v_perm_b32 v3, v2, v3, s4
+; SDAG-NEXT: v_perm_b32 v24, v24, v25, s4
+; SDAG-NEXT: v_perm_b32 v25, v26, v27, s4
+; SDAG-NEXT: v_perm_b32 v23, v28, v29, s4
; SDAG-NEXT: v_perm_b32 v16, v16, v17, s4
; SDAG-NEXT: v_perm_b32 v17, v18, v19, s4
-; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; SDAG-NEXT: v_lshl_or_b32 v5, v14, 16, v11
-; SDAG-NEXT: v_lshl_or_b32 v4, v20, 16, v15
-; SDAG-NEXT: v_lshl_or_b32 v3, v17, 16, v16
-; SDAG-NEXT: s_waitcnt vmcnt(1)
-; SDAG-NEXT: v_perm_b32 v0, v30, v10, s4
-; SDAG-NEXT: v_lshl_or_b32 v6, v0, 16, v7
-; SDAG-NEXT: buffer_store_dwordx4 v[3:6], off, s[16:19], 0 offset:16
+; SDAG-NEXT: v_lshl_or_b32 v7, v13, 16, v12
+; SDAG-NEXT: v_lshl_or_b32 v6, v9, 16, v8
+; SDAG-NEXT: v_lshl_or_b32 v5, v5, 16, v4
+; SDAG-NEXT: v_lshl_or_b32 v4, v3, 16, v10
+; SDAG-NEXT: v_lshl_or_b32 v2, v25, 16, v24
+; SDAG-NEXT: v_lshl_or_b32 v1, v21, 16, v20
+; SDAG-NEXT: v_lshl_or_b32 v0, v17, 16, v16
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_perm_b32 v3, v30, v22, s4
+; SDAG-NEXT: v_lshl_or_b32 v3, v3, 16, v23
+; SDAG-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0
+; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index a1fe463de1c54..4927b8110e5cc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -2859,15 +2859,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v36, s24
-; SDAG-NEXT: v_mov_b32_e32 v37, s25
-; SDAG-NEXT: v_mov_b32_e32 v38, s26
-; SDAG-NEXT: v_mov_b32_e32 v39, s27
+; SDAG-NEXT: v_mov_b32_e32 v40, s24
+; SDAG-NEXT: v_mov_b32_e32 v41, s25
+; SDAG-NEXT: v_mov_b32_e32 v42, s26
+; SDAG-NEXT: v_mov_b32_e32 v43, s27
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b32_e32 v40, s28
-; SDAG-NEXT: v_mov_b32_e32 v41, s29
-; SDAG-NEXT: v_mov_b32_e32 v42, s30
-; SDAG-NEXT: v_mov_b32_e32 v43, s31
+; SDAG-NEXT: v_mov_b32_e32 v36, s28
+; SDAG-NEXT: v_mov_b32_e32 v37, s29
+; SDAG-NEXT: v_mov_b32_e32 v38, s30
+; SDAG-NEXT: v_mov_b32_e32 v39, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -2876,7 +2876,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[40:43], v[36:39], v[16:31]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -2978,15 +2978,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v4, s24
-; HEURRC-NEXT: v_mov_b32_e32 v5, s25
-; HEURRC-NEXT: v_mov_b32_e32 v6, s26
-; HEURRC-NEXT: v_mov_b32_e32 v7, s27
+; HEURRC-NEXT: v_mov_b32_e32 v8, s24
+; HEURRC-NEXT: v_mov_b32_e32 v9, s25
+; HEURRC-NEXT: v_mov_b32_e32 v10, s26
+; HEURRC-NEXT: v_mov_b32_e32 v11, s27
; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
-; HEURRC-NEXT: v_mov_b32_e32 v8, s28
-; HEURRC-NEXT: v_mov_b32_e32 v9, s29
-; HEURRC-NEXT: v_mov_b32_e32 v10, s30
-; HEURRC-NEXT: v_mov_b32_e32 v11, s31
+; HEURRC-NEXT: v_mov_b32_e32 v4, s28
+; HEURRC-NEXT: v_mov_b32_e32 v5, s29
+; HEURRC-NEXT: v_mov_b32_e32 v6, s30
+; HEURRC-NEXT: v_mov_b32_e32 v7, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
@@ -3003,7 +3003,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[8:11], v[4:7], a[16:31]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
@@ -3049,15 +3049,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3066,7 +3066,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[40:43], v[36:39], v[16:31]
; VGPRRC-NEXT: s_nop 11
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
@@ -3260,15 +3260,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[32:33], 48
; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v36, s24
-; SDAG-NEXT: v_mov_b32_e32 v37, s25
-; SDAG-NEXT: v_mov_b32_e32 v38, s26
-; SDAG-NEXT: v_mov_b32_e32 v39, s27
+; SDAG-NEXT: v_mov_b32_e32 v40, s24
+; SDAG-NEXT: v_mov_b32_e32 v41, s25
+; SDAG-NEXT: v_mov_b32_e32 v42, s26
+; SDAG-NEXT: v_mov_b32_e32 v43, s27
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; SDAG-NEXT: v_mov_b32_e32 v40, s28
-; SDAG-NEXT: v_mov_b32_e32 v41, s29
-; SDAG-NEXT: v_mov_b32_e32 v42, s30
-; SDAG-NEXT: v_mov_b32_e32 v43, s31
+; SDAG-NEXT: v_mov_b32_e32 v36, s28
+; SDAG-NEXT: v_mov_b32_e32 v37, s29
+; SDAG-NEXT: v_mov_b32_e32 v38, s30
+; SDAG-NEXT: v_mov_b32_e32 v39, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3277,7 +3277,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[40:43], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -3379,15 +3379,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v4, s24
-; HEURRC-NEXT: v_mov_b32_e32 v5, s25
-; HEURRC-NEXT: v_mov_b32_e32 v6, s26
-; HEURRC-NEXT: v_mov_b32_e32 v7, s27
+; HEURRC-NEXT: v_mov_b32_e32 v8, s24
+; HEURRC-NEXT: v_mov_b32_e32 v9, s25
+; HEURRC-NEXT: v_mov_b32_e32 v10, s26
+; HEURRC-NEXT: v_mov_b32_e32 v11, s27
; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
-; HEURRC-NEXT: v_mov_b32_e32 v8, s28
-; HEURRC-NEXT: v_mov_b32_e32 v9, s29
-; HEURRC-NEXT: v_mov_b32_e32 v10, s30
-; HEURRC-NEXT: v_mov_b32_e32 v11, s31
+; HEURRC-NEXT: v_mov_b32_e32 v4, s28
+; HEURRC-NEXT: v_mov_b32_e32 v5, s29
+; HEURRC-NEXT: v_mov_b32_e32 v6, s30
+; HEURRC-NEXT: v_mov_b32_e32 v7, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
@@ -3404,7 +3404,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[8:11], v[4:7], a[16:31] cbsz:2 abid:3 blgp:1
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
@@ -3450,15 +3450,15 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
+; VGPRRC-NEXT: v_mov_b32_e32 v36, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v37, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v38, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v39, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3467,7 +3467,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[40:43], v[36:39], v[16:31] cbsz:2 abid:3 blgp:1
; VGPRRC-NEXT: s_nop 11
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
@@ -3962,21 +3962,20 @@ define <16 x i32> @test_mfma_i32_32x32x32_i8__mac__flags(<4 x i32> %arg0, <4 x i
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s20
-; SDAG-NEXT: v_mov_b32_e32 v33, s21
-; SDAG-NEXT: v_mov_b32_e32 v34, s22
-; SDAG-NEXT: v_mov_b32_e32 v35, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v36, s24
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b32_e32 v32, s28
+; SDAG-NEXT: v_mov_b32_e32 v33, s29
+; SDAG-NEXT: v_mov_b32_e32 v34, s30
+; SDAG-NEXT: v_mov_b32_e32 v35, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3985,7 +3984,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -4072,21 +4071,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v32, s20
-; HEURRC-NEXT: v_mov_b32_e32 v33, s21
-; HEURRC-NEXT: v_mov_b32_e32 v34, s22
-; HEURRC-NEXT: v_mov_b32_e32 v35, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v36, s24
; HEURRC-NEXT: v_mov_b32_e32 v37, s25
; HEURRC-NEXT: v_mov_b32_e32 v38, s26
; HEURRC-NEXT: v_mov_b32_e32 v39, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; HEURRC-NEXT: v_mov_b32_e32 v32, s28
+; HEURRC-NEXT: v_mov_b32_e32 v33, s29
+; HEURRC-NEXT: v_mov_b32_e32 v34, s30
+; HEURRC-NEXT: v_mov_b32_e32 v35, s31
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4095,7 +4093,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4136,21 +4134,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4159,7 +4156,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4339,21 +4336,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s20
-; SDAG-NEXT: v_mov_b32_e32 v33, s21
-; SDAG-NEXT: v_mov_b32_e32 v34, s22
-; SDAG-NEXT: v_mov_b32_e32 v35, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v36, s24
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b32_e32 v32, s28
+; SDAG-NEXT: v_mov_b32_e32 v33, s29
+; SDAG-NEXT: v_mov_b32_e32 v34, s30
+; SDAG-NEXT: v_mov_b32_e32 v35, s31
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4362,7 +4358,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -4449,21 +4445,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v32, s20
-; HEURRC-NEXT: v_mov_b32_e32 v33, s21
-; HEURRC-NEXT: v_mov_b32_e32 v34, s22
-; HEURRC-NEXT: v_mov_b32_e32 v35, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v36, s24
; HEURRC-NEXT: v_mov_b32_e32 v37, s25
; HEURRC-NEXT: v_mov_b32_e32 v38, s26
; HEURRC-NEXT: v_mov_b32_e32 v39, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; HEURRC-NEXT: v_mov_b32_e32 v32, s28
+; HEURRC-NEXT: v_mov_b32_e32 v33, s29
+; HEURRC-NEXT: v_mov_b32_e32 v34, s30
+; HEURRC-NEXT: v_mov_b32_e32 v35, s31
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4472,7 +4467,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4513,21 +4508,20 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -4536,7 +4530,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4716,20 +4710,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v20, s24
; SDAG-NEXT: v_mov_b32_e32 v21, s25
; SDAG-NEXT: v_mov_b32_e32 v22, s26
; SDAG-NEXT: v_mov_b32_e32 v23, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v16, s28
+; SDAG-NEXT: v_mov_b32_e32 v17, s29
+; SDAG-NEXT: v_mov_b32_e32 v18, s30
+; SDAG-NEXT: v_mov_b32_e32 v19, s31
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4738,7 +4731,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4777,20 +4770,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v20, s24
; HEURRC-NEXT: v_mov_b32_e32 v21, s25
; HEURRC-NEXT: v_mov_b32_e32 v22, s26
; HEURRC-NEXT: v_mov_b32_e32 v23, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b32_e32 v16, s28
+; HEURRC-NEXT: v_mov_b32_e32 v17, s29
+; HEURRC-NEXT: v_mov_b32_e32 v18, s30
+; HEURRC-NEXT: v_mov_b32_e32 v19, s31
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4799,7 +4791,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 10
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4810,20 +4802,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4832,7 +4823,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 10
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4922,20 +4913,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v20, s24
; SDAG-NEXT: v_mov_b32_e32 v21, s25
; SDAG-NEXT: v_mov_b32_e32 v22, s26
; SDAG-NEXT: v_mov_b32_e32 v23, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v16, s28
+; SDAG-NEXT: v_mov_b32_e32 v17, s29
+; SDAG-NEXT: v_mov_b32_e32 v18, s30
+; SDAG-NEXT: v_mov_b32_e32 v19, s31
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -4944,7 +4934,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4983,20 +4973,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v20, s24
; HEURRC-NEXT: v_mov_b32_e32 v21, s25
; HEURRC-NEXT: v_mov_b32_e32 v22, s26
; HEURRC-NEXT: v_mov_b32_e32 v23, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; HEURRC-NEXT: v_mov_b32_e32 v16, s28
+; HEURRC-NEXT: v_mov_b32_e32 v17, s29
+; HEURRC-NEXT: v_mov_b32_e32 v18, s30
+; HEURRC-NEXT: v_mov_b32_e32 v19, s31
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -5005,7 +4994,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 10
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -5016,20 +5005,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -5038,7 +5026,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 10
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index 97a89ec819bae..c0137b1a2ce95 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -1159,34 +1159,34 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
+; SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x40
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v21, s12
-; SDAG-NEXT: v_mov_b32_e32 v22, s13
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v21, s20
+; SDAG-NEXT: v_mov_b32_e32 v22, s21
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[22:23]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -1227,26 +1227,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -1290,26 +1290,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -1353,26 +1353,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -1416,26 +1416,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 0a36d3dd28f06..6629f528f16cb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -3195,22 +3195,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v20, s12
-; SDAG-NEXT: v_mov_b32_e32 v21, s13
-; SDAG-NEXT: v_mov_b32_e32 v22, s14
-; SDAG-NEXT: v_mov_b32_e32 v23, s15
-; SDAG-NEXT: v_mov_b32_e32 v24, s16
-; SDAG-NEXT: v_mov_b32_e32 v25, s17
-; SDAG-NEXT: v_mov_b32_e32 v26, s18
-; SDAG-NEXT: v_mov_b32_e32 v27, s19
-; SDAG-NEXT: v_mov_b32_e32 v28, s20
-; SDAG-NEXT: v_mov_b32_e32 v29, s21
-; SDAG-NEXT: v_mov_b32_e32 v30, s22
-; SDAG-NEXT: v_mov_b32_e32 v31, s23
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v28, s12
+; SDAG-NEXT: v_mov_b32_e32 v29, s13
+; SDAG-NEXT: v_mov_b32_e32 v30, s14
+; SDAG-NEXT: v_mov_b32_e32 v31, s15
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v20, s20
+; SDAG-NEXT: v_mov_b32_e32 v21, s21
+; SDAG-NEXT: v_mov_b32_e32 v22, s22
+; SDAG-NEXT: v_mov_b32_e32 v23, s23
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -3221,7 +3221,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: v_mov_b32_e32 v32, s0
; SDAG-NEXT: v_mov_b32_e32 v33, s1
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -3274,37 +3274,38 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v32, -2
; SDAG-NEXT: v_mov_b32_e32 v33, 0x41
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v20, s12
-; SDAG-NEXT: v_mov_b32_e32 v21, s13
-; SDAG-NEXT: v_mov_b32_e32 v22, s14
-; SDAG-NEXT: v_mov_b32_e32 v23, s15
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b32_e32 v24, s16
-; SDAG-NEXT: v_mov_b32_e32 v25, s17
-; SDAG-NEXT: v_mov_b32_e32 v26, s18
-; SDAG-NEXT: v_mov_b32_e32 v27, s19
-; SDAG-NEXT: v_mov_b32_e32 v28, s20
-; SDAG-NEXT: v_mov_b32_e32 v29, s21
-; SDAG-NEXT: v_mov_b32_e32 v30, s22
-; SDAG-NEXT: v_mov_b32_e32 v31, s23
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v20, s20
+; SDAG-NEXT: v_mov_b32_e32 v21, s21
+; SDAG-NEXT: v_mov_b32_e32 v22, s22
+; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: s_load_dwordx16 s[16:31], s[4:5], 0x40
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v28, s12
+; SDAG-NEXT: v_mov_b32_e32 v29, s13
+; SDAG-NEXT: v_mov_b32_e32 v30, s14
+; SDAG-NEXT: v_mov_b32_e32 v31, s15
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -3356,28 +3357,27 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %scale0, i32 %scale1) #1 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
+; SDAG-NEXT: v_mov_b32_e32 v26, s36
+; SDAG-NEXT: v_mov_b32_e32 v27, s37
+; SDAG-NEXT: v_mov_b32_e32 v28, s38
+; SDAG-NEXT: v_mov_b32_e32 v29, s39
+; SDAG-NEXT: v_mov_b32_e32 v30, s40
+; SDAG-NEXT: v_mov_b32_e32 v31, s41
+; SDAG-NEXT: v_mov_b32_e32 v32, s42
+; SDAG-NEXT: v_mov_b32_e32 v33, s43
+; SDAG-NEXT: v_mov_b32_e32 v18, s44
+; SDAG-NEXT: v_mov_b32_e32 v19, s45
+; SDAG-NEXT: v_mov_b32_e32 v20, s46
+; SDAG-NEXT: v_mov_b32_e32 v21, s47
+; SDAG-NEXT: v_mov_b32_e32 v22, s48
+; SDAG-NEXT: v_mov_b32_e32 v23, s49
+; SDAG-NEXT: v_mov_b32_e32 v24, s50
+; SDAG-NEXT: v_mov_b32_e32 v25, s51
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -3388,7 +3388,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
; SDAG-NEXT: v_mov_b32_e32 v16, s0
; SDAG-NEXT: v_mov_b32_e32 v17, s1
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v16, v17 op_sel_hi:[0,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[26:33], v[18:25], v[0:15], v16, v17 op_sel_hi:[0,0,0]
; SDAG-NEXT: v_mov_b32_e32 v18, s20
; SDAG-NEXT: v_mov_b32_e32 v19, s21
; SDAG-NEXT: v_mov_b32_e32 v20, s22
@@ -3492,29 +3492,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #1 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v16, 42
; SDAG-NEXT: v_mov_b32_e32 v17, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v26, s36
+; SDAG-NEXT: v_mov_b32_e32 v27, s37
+; SDAG-NEXT: v_mov_b32_e32 v28, s38
+; SDAG-NEXT: v_mov_b32_e32 v29, s39
+; SDAG-NEXT: v_mov_b32_e32 v30, s40
+; SDAG-NEXT: v_mov_b32_e32 v31, s41
+; SDAG-NEXT: v_mov_b32_e32 v32, s42
+; SDAG-NEXT: v_mov_b32_e32 v33, s43
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v18, s44
+; SDAG-NEXT: v_mov_b32_e32 v19, s45
+; SDAG-NEXT: v_mov_b32_e32 v20, s46
+; SDAG-NEXT: v_mov_b32_e32 v21, s47
+; SDAG-NEXT: v_mov_b32_e32 v22, s48
+; SDAG-NEXT: v_mov_b32_e32 v23, s49
+; SDAG-NEXT: v_mov_b32_e32 v24, s50
+; SDAG-NEXT: v_mov_b32_e32 v25, s51
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -3523,7 +3522,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[26:33], v[18:25], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v18, s20
; SDAG-NEXT: v_mov_b32_e32 v19, s21
; SDAG-NEXT: v_mov_b32_e32 v20, s22
@@ -3625,27 +3624,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s12
-; SDAG-NEXT: v_mov_b32_e32 v33, s13
-; SDAG-NEXT: v_mov_b32_e32 v34, s14
-; SDAG-NEXT: v_mov_b32_e32 v35, s15
-; SDAG-NEXT: v_mov_b32_e32 v36, s16
-; SDAG-NEXT: v_mov_b32_e32 v37, s17
-; SDAG-NEXT: v_mov_b32_e32 v38, s18
-; SDAG-NEXT: v_mov_b32_e32 v39, s19
-; SDAG-NEXT: v_mov_b32_e32 v40, s20
-; SDAG-NEXT: v_mov_b32_e32 v41, s21
-; SDAG-NEXT: v_mov_b32_e32 v42, s22
-; SDAG-NEXT: v_mov_b32_e32 v43, s23
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v44, s24
-; SDAG-NEXT: v_mov_b32_e32 v45, s25
-; SDAG-NEXT: v_mov_b32_e32 v46, s26
-; SDAG-NEXT: v_mov_b32_e32 v47, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v40, s36
+; SDAG-NEXT: v_mov_b32_e32 v41, s37
+; SDAG-NEXT: v_mov_b32_e32 v42, s38
+; SDAG-NEXT: v_mov_b32_e32 v43, s39
+; SDAG-NEXT: v_mov_b32_e32 v44, s40
+; SDAG-NEXT: v_mov_b32_e32 v45, s41
+; SDAG-NEXT: v_mov_b32_e32 v46, s42
+; SDAG-NEXT: v_mov_b32_e32 v47, s43
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b32_e32 v32, s44
+; SDAG-NEXT: v_mov_b32_e32 v33, s45
+; SDAG-NEXT: v_mov_b32_e32 v34, s46
+; SDAG-NEXT: v_mov_b32_e32 v35, s47
+; SDAG-NEXT: v_mov_b32_e32 v36, s48
+; SDAG-NEXT: v_mov_b32_e32 v37, s49
+; SDAG-NEXT: v_mov_b32_e32 v38, s50
+; SDAG-NEXT: v_mov_b32_e32 v39, s51
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
@@ -3654,7 +3652,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
+; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[40:47], v[32:39], v[16:31] blgp:2
; SDAG-NEXT: s_nop 14
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -3749,29 +3747,28 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v32, 42
; SDAG-NEXT: v_mov_b32_e32 v33, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v24, s20
-; SDAG-NEXT: v_mov_b32_e32 v25, s21
-; SDAG-NEXT: v_mov_b32_e32 v26, s22
-; SDAG-NEXT: v_mov_b32_e32 v27, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v28, s24
-; SDAG-NEXT: v_mov_b32_e32 v29, s25
-; SDAG-NEXT: v_mov_b32_e32 v30, s26
-; SDAG-NEXT: v_mov_b32_e32 v31, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v24, s36
+; SDAG-NEXT: v_mov_b32_e32 v25, s37
+; SDAG-NEXT: v_mov_b32_e32 v26, s38
+; SDAG-NEXT: v_mov_b32_e32 v27, s39
+; SDAG-NEXT: v_mov_b32_e32 v28, s40
+; SDAG-NEXT: v_mov_b32_e32 v29, s41
+; SDAG-NEXT: v_mov_b32_e32 v30, s42
+; SDAG-NEXT: v_mov_b32_e32 v31, s43
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v16, s44
+; SDAG-NEXT: v_mov_b32_e32 v17, s45
+; SDAG-NEXT: v_mov_b32_e32 v18, s46
+; SDAG-NEXT: v_mov_b32_e32 v19, s47
+; SDAG-NEXT: v_mov_b32_e32 v20, s48
+; SDAG-NEXT: v_mov_b32_e32 v21, s49
+; SDAG-NEXT: v_mov_b32_e32 v22, s50
+; SDAG-NEXT: v_mov_b32_e32 v23, s51
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -3780,7 +3777,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
index 157b4fbe6803d..57b907f3123d8 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
@@ -30,26 +30,25 @@ define amdgpu_vs void @test(ptr addrspace(8) inreg %arg1, ptr addrspace(3) %arg2
define amdgpu_vs void @test_2(ptr addrspace(8) inreg %arg1, i32 %arg2, i32 inreg %arg3, ptr addrspace(3) %arg4) {
; CHECK-LABEL: test_2:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, 20, v1
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, 16, v1
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, 28, v1
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, 24, v1
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, 12, v1
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, 8, v1
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, 4, v1
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, 12, v1
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, 8, v1
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, 4, v1
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, 28, v1
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, 24, v1
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, 20, v1
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, 16, v1
; CHECK-NEXT: s_mov_b32 m0, -1
-; CHECK-NEXT: ds_read_b32 v2, v2
-; CHECK-NEXT: ds_read_b32 v5, v4
-; CHECK-NEXT: ds_read_b32 v4, v6
-; CHECK-NEXT: ds_read_b32 v9, v7
-; CHECK-NEXT: ds_read_b32 v8, v8
-; CHECK-NEXT: ds_read_b32 v7, v10
-; CHECK-NEXT: ds_read_b32 v6, v1
+; CHECK-NEXT: ds_read_b32 v4, v2
; CHECK-NEXT: ds_read_b32 v3, v3
-; CHECK-NEXT: s_waitcnt lgkmcnt(1)
-; CHECK-NEXT: tbuffer_store_format_xyzw v[6:9], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen glc slc
+; CHECK-NEXT: ds_read_b32 v2, v5
+; CHECK-NEXT: ds_read_b32 v8, v6
+; CHECK-NEXT: ds_read_b32 v7, v7
+; CHECK-NEXT: ds_read_b32 v6, v9
+; CHECK-NEXT: ds_read_b32 v5, v10
+; CHECK-NEXT: ds_read_b32 v1, v1
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: tbuffer_store_format_xyzw v[2:5], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:16 glc slc
+; CHECK-NEXT: tbuffer_store_format_xyzw v[1:4], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen glc slc
+; CHECK-NEXT: tbuffer_store_format_xyzw v[5:8], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:16 glc slc
; CHECK-NEXT: s_endpgm
%load = load <8 x float>, ptr addrspace(3) %arg4, align 4
%vec1 = shufflevector <8 x float> %load, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index 7f624cffa014c..390b2650cbeb5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -337,11 +337,11 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle2step_i16(<16 x i16> %src) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u32 q2, q1, #16
; CHECK-NEXT: vstrh.32 q2, [r0, #8]
; CHECK-NEXT: vshr.u32 q2, q0, #16
-; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: vstrh.32 q2, [r0]
; CHECK-NEXT: vstrh.32 q1, [r1, #8]
; CHECK-NEXT: vstrh.32 q0, [r1]
@@ -627,11 +627,11 @@ define arm_aapcs_vfpcc <16 x i8> @shuffle2step_i8(<32 x i8> %src) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u16 q2, q1, #8
; CHECK-NEXT: vstrb.16 q2, [r0, #8]
; CHECK-NEXT: vshr.u16 q2, q0, #8
-; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: vstrb.16 q2, [r0]
; CHECK-NEXT: vstrb.16 q1, [r1, #8]
; CHECK-NEXT: vstrb.16 q0, [r1]
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index ac2f283166a93..6eb63eee15b1e 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -5,10 +5,10 @@
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
@@ -1959,58 +1959,6 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in.
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT: vpbroadcastd %xmm1, %ymm1
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
-; AVX512F-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT: vpbroadcastd %xmm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,0,11,0,13,0,15]
-; AVX512DQ-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4:
; AVX512BW-SLOW: # %bb.0:
; AVX512BW-SLOW-NEXT: vmovdqa64 (%rdi), %zmm0
@@ -2111,58 +2059,6 @@ define void @vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [8,1,2,3,8,5,6,7]
-; AVX512F-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [8,1,2,3,8,5,6,7]
-; AVX512DQ-FAST-NEXT: vpermi2d %ymm0, %ymm1, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
; AVX512BW-SLOW: # %bb.0:
; AVX512BW-SLOW-NEXT: vmovdqa64 (%rdi), %zmm0
@@ -2263,58 +2159,6 @@ define void @vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1
-; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
-; AVX512F-FAST-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,5,0,7]
-; AVX512DQ-FAST-NEXT: vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm2, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
; AVX512BW-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
; AVX512BW-SLOW: # %bb.0:
; AVX512BW-SLOW-NEXT: vmovdqa64 (%rdi), %zmm0
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
index 4a542949c7859..f28f03f0cb09a 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -5,10 +5,10 @@
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW
@@ -909,10 +909,10 @@ define void @vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16(ptr %in
;
; AVX2-LABEL: vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX2-NEXT: vpbroadcastb (%rdi), %ymm1
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2-NEXT: vpbroadcastb (%rdi), %ymm0
+; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
; AVX2-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
; AVX2-NEXT: vzeroupper
@@ -1651,42 +1651,6 @@ define void @vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5,6,7]
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
-; AVX512F-FAST-NEXT: vpermd (%rdi), %zmm0, %zmm0
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5,6,7]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,9,10,11,0,13,14,15]
-; AVX512DQ-FAST-NEXT: vpermd (%rdi), %zmm0, %zmm0
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
; AVX512BW-SLOW-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2:
; AVX512BW-SLOW: # %bb.0:
; AVX512BW-SLOW-NEXT: vpermq {{.*#+}} ymm0 = mem[0,1,0,1]
@@ -1761,42 +1725,6 @@ define void @vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
-; AVX512F-FAST-NEXT: vpermq (%rdi), %zmm0, %zmm0
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
-;
-; AVX512DQ-FAST-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,5,0,7]
-; AVX512DQ-FAST-NEXT: vpermq (%rdi), %zmm0, %zmm0
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, (%rdx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
-;
; AVX512BW-SLOW-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2:
; AVX512BW-SLOW: # %bb.0:
; AVX512BW-SLOW-NEXT: vpbroadcastq (%rdi), %ymm0
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 7c748439b1730..c39b1a5adf382 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -1827,10 +1827,27 @@ define <4 x i32> @pmaddwd_8(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_8:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%b = sext <8 x i16> %B to <8 x i32>
%m = mul nsw <8 x i32> %a, %b
@@ -1846,10 +1863,27 @@ define <4 x i32> @pmaddwd_8_swapped(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_8_swapped:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_8_swapped:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_8_swapped:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%b = sext <8 x i16> %B to <8 x i32>
%m = mul nsw <8 x i32> %a, %b
@@ -1921,10 +1955,20 @@ define <8 x i32> @pmaddwd_16(<16 x i16> %A, <16 x i16> %B) {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddwd_16:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddwd_16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: pmaddwd_16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: retq
%a = sext <16 x i16> %A to <16 x i32>
%b = sext <16 x i16> %B to <16 x i32>
%m = mul nsw <16 x i32> %a, %b
@@ -1991,10 +2035,24 @@ define <4 x i32> @pmaddwd_const(<8 x i16> %A) {
; SSE2-NEXT: pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [32767,32768,0,0,1,7,42,32]
; SSE2-NEXT: retq
;
-; AVX-LABEL: pmaddwd_const:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32767,32768,0,0,1,7,42,32]
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_const:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32767,0,32768,0,0,0,0,0]
+; AVX1-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [1,0,7,0,42,0,32,0]
+; AVX1-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_const:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX256-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [32767,0,32768,0,0,0,0,0,1,0,7,0,42,0,32,0]
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%m = mul nsw <8 x i32> %a, <i32 32767, i32 -32768, i32 0, i32 0, i32 1, i32 7, i32 42, i32 32>
%odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -2098,10 +2156,29 @@ define <4 x i32> @jumbled_indices4(<8 x i16> %A, <8 x i16> %B) {
; SSE2-NEXT: pmaddwd %xmm1, %xmm0
; SSE2-NEXT: retq
;
-; AVX-LABEL: jumbled_indices4:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: jumbled_indices4:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,3]
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: jumbled_indices4:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX256-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,3]
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%exta = sext <8 x i16> %A to <8 x i32>
%extb = sext <8 x i16> %B to <8 x i32>
%m = mul <8 x i32> %exta, %extb
@@ -2127,10 +2204,21 @@ define <8 x i32> @jumbled_indices8(<16 x i16> %A, <16 x i16> %B) {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: jumbled_indices8:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: jumbled_indices8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: jumbled_indices8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,1,5,4,3,2,7,6]
+; AVX512-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT: retq
%exta = sext <16 x i16> %A to <16 x i32>
%extb = sext <16 x i16> %B to <16 x i32>
%m = mul <16 x i32> %exta, %extb
@@ -2306,10 +2394,44 @@ define <4 x i32> @pmaddwd_128(ptr %Aptr, ptr %Bptr) {
define <8 x i32> @pmaddwd_256(ptr %Aptr, ptr %Bptr) {
; SSE2-LABEL: pmaddwd_256:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: movdqa 16(%rdi), %xmm1
-; SSE2-NEXT: pmaddwd (%rsi), %xmm0
-; SSE2-NEXT: pmaddwd 16(%rsi), %xmm1
+; SSE2-NEXT: movdqa (%rdi), %xmm1
+; SSE2-NEXT: movdqa 16(%rdi), %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: psrad $16, %xmm2
+; SSE2-NEXT: packssdw %xmm3, %xmm2
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: psrad $16, %xmm1
+; SSE2-NEXT: packssdw %xmm0, %xmm1
+; SSE2-NEXT: movdqa (%rsi), %xmm0
+; SSE2-NEXT: movdqa 16(%rsi), %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: pslld $16, %xmm4
+; SSE2-NEXT: psrad $16, %xmm4
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: pslld $16, %xmm5
+; SSE2-NEXT: psrad $16, %xmm5
+; SSE2-NEXT: packssdw %xmm4, %xmm5
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: packssdw %xmm3, %xmm0
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: pmulhw %xmm5, %xmm3
+; SSE2-NEXT: pmullw %xmm2, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: pmulhw %xmm0, %xmm3
+; SSE2-NEXT: pmullw %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; SSE2-NEXT: paddd %xmm2, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: retq
;
; AVX1-LABEL: pmaddwd_256:
@@ -2321,11 +2443,52 @@ define <8 x i32> @pmaddwd_256(ptr %Aptr, ptr %Bptr) {
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddwd_256:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vmovdqa (%rdi), %ymm0
-; AVX256-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddwd_256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm3, %xmm4, %xmm3
+; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rsi), %xmm1
+; AVX2-NEXT: vmovdqa 16(%rsi), %xmm4
+; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm2[1],xmm4[2],xmm2[3],xmm4[4],xmm2[5],xmm4[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm5, %xmm2, %xmm2
+; AVX2-NEXT: vpsrld $16, %xmm4, %xmm4
+; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX2-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
+; AVX2-NEXT: vpmovsxwd %xmm3, %ymm3
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX2-NEXT: vpmulld %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: pmaddwd_256:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vpmovdw %zmm0, %ymm1
+; AVX512-NEXT: vpsrld $16, %ymm0, %ymm0
+; AVX512-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512-NEXT: vmovdqa (%rsi), %ymm2
+; AVX512-NEXT: vpmovdw %zmm2, %ymm3
+; AVX512-NEXT: vpsrld $16, %ymm2, %ymm2
+; AVX512-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512-NEXT: vpmovsxwd %xmm3, %ymm3
+; AVX512-NEXT: vpmulld %ymm3, %ymm1, %ymm1
+; AVX512-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX512-NEXT: vpmulld %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT: retq
%A = load <16 x i16>, ptr %Aptr
%B = load <16 x i16>, ptr %Bptr
%A_even = shufflevector <16 x i16> %A, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -2373,24 +2536,67 @@ define <16 x i32> @pmaddwd_512(ptr %Aptr, ptr %Bptr) {
; AVX2: # %bb.0:
; AVX2-NEXT: vmovdqa (%rdi), %ymm0
; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX2-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX2-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vshufps {{.*#+}} ymm2 = ymm3[0,2],ymm2[0,2],ymm3[4,6],ymm2[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vmovdqa (%rsi), %ymm1
+; AVX2-NEXT: vmovdqa 32(%rsi), %ymm4
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm4[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm6 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,2],ymm5[0,2],ymm6[4,6],ymm5[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[0,2,1,3]
+; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm4
+; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm4[2,3],ymm1[4,5],ymm4[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX2-NEXT: vpmovsxwd %xmm2, %ymm2
+; AVX2-NEXT: vpmovsxwd %xmm5, %ymm4
+; AVX2-NEXT: vpmulld %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm4
+; AVX2-NEXT: vpmovsxwd %xmm4, %ymm4
+; AVX2-NEXT: vpmulld %ymm4, %ymm2, %ymm2
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm4
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpmovsxwd %xmm0, %ymm5
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm0
+; AVX2-NEXT: vpmulld %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT: vpmulld %ymm1, %ymm5, %ymm1
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
; AVX2-NEXT: retq
;
-; AVX512F-LABEL: pmaddwd_512:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: retq
-;
-; AVX512BW-LABEL: pmaddwd_512:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
-; AVX512BW-NEXT: retq
+; AVX512-LABEL: pmaddwd_512:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT: vpmovdw %zmm0, %ymm1
+; AVX512-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512-NEXT: vpmovdw %zmm2, %ymm3
+; AVX512-NEXT: vpsrld $16, %zmm2, %zmm2
+; AVX512-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512-NEXT: vpmulld %zmm3, %zmm1, %zmm1
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512-NEXT: vpmulld %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512-NEXT: retq
%A = load <32 x i16>, ptr %Aptr
%B = load <32 x i16>, ptr %Bptr
%A_even = shufflevector <32 x i16> %A, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -2475,24 +2681,69 @@ define <32 x i32> @pmaddwd_1024(ptr %Aptr, ptr %Bptr) {
;
; AVX512F-LABEL: pmaddwd_1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm2
-; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm3
-; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpmaddwd 96(%rsi), %ymm3, %ymm1
-; AVX512F-NEXT: vpmaddwd 64(%rsi), %ymm2, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT: vmovdqa64 64(%rdi), %zmm1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm2
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm3
+; AVX512F-NEXT: vpsrld $16, %zmm1, %zmm1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: vmovdqa64 (%rsi), %zmm4
+; AVX512F-NEXT: vmovdqa64 64(%rsi), %zmm5
+; AVX512F-NEXT: vpmovdw %zmm5, %ymm6
+; AVX512F-NEXT: vpmovdw %zmm4, %ymm7
+; AVX512F-NEXT: vpsrld $16, %zmm5, %zmm5
+; AVX512F-NEXT: vpmovdw %zmm5, %ymm5
+; AVX512F-NEXT: vpsrld $16, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovdw %zmm4, %ymm4
+; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512F-NEXT: vpmovsxwd %ymm7, %zmm7
+; AVX512F-NEXT: vpmulld %zmm7, %zmm3, %zmm3
+; AVX512F-NEXT: vpmovsxwd %ymm6, %zmm6
+; AVX512F-NEXT: vpmulld %zmm6, %zmm2, %zmm2
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm4
+; AVX512F-NEXT: vpmulld %zmm4, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm3, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm5, %zmm3
+; AVX512F-NEXT: vpmulld %zmm3, %zmm1, %zmm1
+; AVX512F-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddwd_1024:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
-; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmaddwd 64(%rsi), %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
+; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm3
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm2, %zmm3
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm0
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm1
+; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm5
+; AVX512BW-NEXT: vpermi2w %zmm5, %zmm1, %zmm2
+; AVX512BW-NEXT: vpermt2w %zmm5, %zmm4, %zmm1
+; AVX512BW-NEXT: vpmovsxwd %ymm3, %zmm4
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm3, %ymm3
+; AVX512BW-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512BW-NEXT: vpmovsxwd %ymm2, %zmm5
+; AVX512BW-NEXT: vpmulld %zmm5, %zmm4, %zmm4
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm2, %ymm2
+; AVX512BW-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512BW-NEXT: vpmulld %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT: vpmovsxwd %ymm0, %zmm3
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512BW-NEXT: vpmovsxwd %ymm0, %zmm5
+; AVX512BW-NEXT: vpmovsxwd %ymm1, %zmm0
+; AVX512BW-NEXT: vpmulld %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm1, %ymm1
+; AVX512BW-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512BW-NEXT: vpmulld %zmm1, %zmm5, %zmm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: retq
%A = load <64 x i16>, ptr %Aptr
%B = load <64 x i16>, ptr %Bptr
diff --git a/llvm/test/CodeGen/X86/pmaddubsw.ll b/llvm/test/CodeGen/X86/pmaddubsw.ll
index d6c9877cd99b6..0dd2b20cc53d1 100644
--- a/llvm/test/CodeGen/X86/pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/pmaddubsw.ll
@@ -43,26 +43,196 @@ define <8 x i16> @pmaddubsw_128(ptr %Aptr, ptr %Bptr) {
define <16 x i16> @pmaddubsw_256(ptr %Aptr, ptr %Bptr) {
; SSE-LABEL: pmaddubsw_256:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa (%rsi), %xmm0
-; SSE-NEXT: movdqa 16(%rsi), %xmm1
-; SSE-NEXT: pmaddubsw (%rdi), %xmm0
-; SSE-NEXT: pmaddubsw 16(%rdi), %xmm1
+; SSE-NEXT: movdqa (%rdi), %xmm2
+; SSE-NEXT: movdqa 16(%rdi), %xmm0
+; SSE-NEXT: movdqa (%rsi), %xmm3
+; SSE-NEXT: movdqa 16(%rsi), %xmm7
+; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT: movdqa %xmm7, %xmm1
+; SSE-NEXT: pand %xmm5, %xmm1
+; SSE-NEXT: pand %xmm3, %xmm5
+; SSE-NEXT: packuswb %xmm1, %xmm5
+; SSE-NEXT: pxor %xmm6, %xmm6
+; SSE-NEXT: movdqa %xmm5, %xmm4
+; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; SSE-NEXT: movdqa %xmm3, %xmm1
+; SSE-NEXT: psrlw $8, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE-NEXT: movdqa %xmm7, %xmm1
+; SSE-NEXT: psrlw $8, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; SSE-NEXT: movdqa %xmm2, %xmm8
+; SSE-NEXT: psllw $8, %xmm8
+; SSE-NEXT: psraw $8, %xmm8
+; SSE-NEXT: movdqa %xmm0, %xmm9
+; SSE-NEXT: psllw $8, %xmm9
+; SSE-NEXT: psraw $8, %xmm9
+; SSE-NEXT: psraw $8, %xmm2
+; SSE-NEXT: psraw $8, %xmm0
+; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; SSE-NEXT: movdqa %xmm9, %xmm1
+; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT: pmaddwd %xmm7, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
+; SSE-NEXT: pmaddwd %xmm5, %xmm9
+; SSE-NEXT: packssdw %xmm9, %xmm1
+; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; SSE-NEXT: movdqa %xmm8, %xmm0
+; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE-NEXT: pmaddwd %xmm3, %xmm0
+; SSE-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
+; SSE-NEXT: pmaddwd %xmm4, %xmm8
+; SSE-NEXT: packssdw %xmm8, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: pmaddubsw_256:
; AVX1: # %bb.0:
+; AVX1-NEXT: vmovdqa (%rdi), %xmm0
+; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm4
+; AVX1-NEXT: vpackuswb %xmm3, %xmm4, %xmm4
+; AVX1-NEXT: vmovq {{.*#+}} xmm5 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT: vpshufb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm0[0],xmm1[0]
; AVX1-NEXT: vmovdqa (%rsi), %xmm0
; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1
-; AVX1-NEXT: vpmaddubsw 16(%rdi), %xmm1, %xmm1
-; AVX1-NEXT: vpmaddubsw (%rdi), %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm7
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vpackuswb %xmm7, %xmm2, %xmm2
+; AVX1-NEXT: vpshufb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT: vpshufb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm0[0],xmm1[0]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; AVX1-NEXT: vpackuswb %xmm3, %xmm3, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm2[1,1,1,1]
+; AVX1-NEXT: vpackuswb %xmm7, %xmm7, %xmm7
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[2,3,2,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm12 = xmm6[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm13 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm14 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm15 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
+; AVX1-NEXT: vpmovsxbw %xmm6, %xmm6
+; AVX1-NEXT: vpmovsxbw %xmm4, %xmm4
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; AVX1-NEXT: vpmaddwd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero,xmm14[4],zero,xmm14[5],zero,xmm14[6],zero,xmm14[7],zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX1-NEXT: vpmovsxbw %xmm12, %xmm6
+; AVX1-NEXT: vpmovsxbw %xmm5, %xmm5
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX1-NEXT: vpmaddwd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpackssdw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero,xmm13[4],zero,xmm13[5],zero,xmm13[6],zero,xmm13[7],zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX1-NEXT: vpmovsxbw %xmm11, %xmm5
+; AVX1-NEXT: vpmovsxbw %xmm3, %xmm3
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX1-NEXT: vpmaddwd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX1-NEXT: vpmovsxbw %xmm10, %xmm4
+; AVX1-NEXT: vpmovsxbw %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddubsw_256:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vmovdqa (%rsi), %ymm0
-; AVX256-NEXT: vpmaddubsw (%rdi), %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddubsw_256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vmovq {{.*#+}} xmm2 = [0,2,4,6,8,10,12,14,0,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vmovq {{.*#+}} xmm4 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpshufb %xmm4, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rsi), %xmm5
+; AVX2-NEXT: vmovdqa 16(%rsi), %xmm6
+; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm7
+; AVX2-NEXT: vpshufb %xmm4, %xmm5, %xmm5
+; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm8
+; AVX2-NEXT: vpmovsxbd %xmm8, %ymm8
+; AVX2-NEXT: vpmovsxbd %xmm3, %ymm3
+; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm2, %ymm8, %ymm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm7, %ymm3, %ymm3
+; AVX2-NEXT: vpshufb %xmm4, %xmm1, %xmm1
+; AVX2-NEXT: vpmovsxbd %xmm1, %ymm1
+; AVX2-NEXT: vpmovsxbd %xmm0, %ymm0
+; AVX2-NEXT: vpshufb %xmm4, %xmm6, %xmm4
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: pmaddubsw_256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512F-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm4
+; AVX512F-NEXT: vpackuswb %xmm3, %xmm4, %xmm3
+; AVX512F-NEXT: vmovq {{.*#+}} xmm4 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX512F-NEXT: vpshufb %xmm4, %xmm1, %xmm1
+; AVX512F-NEXT: vpshufb %xmm4, %xmm0, %xmm0
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512F-NEXT: vmovdqa (%rsi), %xmm1
+; AVX512F-NEXT: vmovdqa 16(%rsi), %xmm5
+; AVX512F-NEXT: vpand %xmm2, %xmm5, %xmm6
+; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT: vpackuswb %xmm6, %xmm2, %xmm2
+; AVX512F-NEXT: vpshufb %xmm4, %xmm5, %xmm5
+; AVX512F-NEXT: vpshufb %xmm4, %xmm1, %xmm1
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm2, %zmm3, %zmm2
+; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT: retq
+;
+; AVX512BW-LABEL: pmaddubsw_256:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
+; AVX512BW-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vmovdqa (%rsi), %ymm2
+; AVX512BW-NEXT: vpsrlw $8, %ymm2, %ymm3
+; AVX512BW-NEXT: vpmovsxbd %xmm1, %zmm1
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
+; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512BW-NEXT: retq
%A = load <32 x i8>, ptr %Aptr
%B = load <32 x i8>, ptr %Bptr
%A_even = shufflevector <32 x i8> %A, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -152,24 +322,167 @@ define <64 x i16> @pmaddubsw_512(ptr %Aptr, ptr %Bptr) {
;
; AVX512F-LABEL: pmaddubsw_512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rsi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1
-; AVX512F-NEXT: vmovdqa 64(%rsi), %ymm2
-; AVX512F-NEXT: vmovdqa 96(%rsi), %ymm3
-; AVX512F-NEXT: vpmaddubsw 32(%rdi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddubsw (%rdi), %ymm0, %ymm0
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm3
+; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm4
+; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm5
+; AVX512F-NEXT: vpshufb %ymm1, %ymm5, %ymm2
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vpshufb %ymm6, %ymm4, %ymm7
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3],ymm7[4,5],ymm2[6,7]
+; AVX512F-NEXT: vpshufb %ymm1, %ymm3, %ymm7
+; AVX512F-NEXT: vpshufb %ymm6, %ymm0, %ymm8
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm7, %zmm2
+; AVX512F-NEXT: vpermq {{.*#+}} zmm2 = zmm2[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm7 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm5, %ymm5
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm8 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm8, %ymm4, %ymm4
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5],ymm5[6,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT: vpshufb %ymm8, %ymm0, %ymm0
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm0
+; AVX512F-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vmovdqa (%rsi), %ymm3
+; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm4
+; AVX512F-NEXT: vmovdqa 64(%rsi), %ymm5
+; AVX512F-NEXT: vmovdqa 96(%rsi), %ymm9
+; AVX512F-NEXT: vpshufb %ymm1, %ymm9, %ymm10
+; AVX512F-NEXT: vpshufb %ymm6, %ymm5, %ymm11
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,3],ymm11[4,5],ymm10[6,7]
+; AVX512F-NEXT: vpshufb %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT: vpshufb %ymm6, %ymm3, %ymm6
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm6[0,1],ymm1[2,3],ymm6[4,5],ymm1[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm10, %zmm1, %zmm1
+; AVX512F-NEXT: vpermq {{.*#+}} zmm6 = zmm1[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm9, %ymm1
+; AVX512F-NEXT: vpshufb %ymm8, %ymm5, %ymm5
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3],ymm5[4,5],ymm1[6,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT: vpshufb %ymm8, %ymm3, %ymm3
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm3
+; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX512F-NEXT: vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm5
+; AVX512F-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512F-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512F-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero,xmm8[4],zero,zero,zero,xmm8[5],zero,zero,zero,xmm8[6],zero,zero,zero,xmm8[7],zero,zero,zero,xmm8[8],zero,zero,zero,xmm8[9],zero,zero,zero,xmm8[10],zero,zero,zero,xmm8[11],zero,zero,zero,xmm8[12],zero,zero,zero,xmm8[13],zero,zero,zero,xmm8[14],zero,zero,zero,xmm8[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm8, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm6, %xmm7
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm5, %zmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm6, %zmm2, %zmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm6
+; AVX512F-NEXT: vextracti128 $1, %ymm6, %xmm7
+; AVX512F-NEXT: vpmovsxbd %xmm7, %zmm7
+; AVX512F-NEXT: vpmovsxbd %xmm6, %zmm6
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm8
+; AVX512F-NEXT: vpmovsxbd %xmm8, %zmm8
+; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm9
+; AVX512F-NEXT: vextracti128 $1, %ymm9, %xmm10
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero,xmm10[8],zero,zero,zero,xmm10[9],zero,zero,zero,xmm10[10],zero,zero,zero,xmm10[11],zero,zero,zero,xmm10[12],zero,zero,zero,xmm10[13],zero,zero,zero,xmm10[14],zero,zero,zero,xmm10[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm10, %zmm7, %zmm7
+; AVX512F-NEXT: vpaddd %zmm7, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero,xmm9[8],zero,zero,zero,xmm9[9],zero,zero,zero,xmm9[10],zero,zero,zero,xmm9[11],zero,zero,zero,xmm9[12],zero,zero,zero,xmm9[13],zero,zero,zero,xmm9[14],zero,zero,zero,xmm9[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm6, %zmm6
+; AVX512F-NEXT: vpaddd %zmm6, %zmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm6, %zmm8, %zmm6
+; AVX512F-NEXT: vpaddd %zmm6, %zmm5, %zmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT: vpmovsdw %zmm5, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpmaddubsw 96(%rdi), %ymm3, %ymm1
-; AVX512F-NEXT: vpmaddubsw 64(%rdi), %ymm2, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT: vpmovsdw %zmm3, %ymm1
+; AVX512F-NEXT: vpmovsdw %zmm4, %ymm2
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddubsw_512:
; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [128,128,128,128,128,128,128,128,0,2,4,6,8,10,12,14,128,128,128,128,128,128,128,128,16,18,20,22,24,26,28,30,128,128,128,128,128,128,128,128,32,34,36,38,40,42,44,46,128,128,128,128,128,128,128,128,48,50,52,54,56,58,60,62]
+; AVX512BW-NEXT: vpshufb %zmm2, %zmm1, %zmm3
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,128,128,128,128,128,128,128,128,16,18,20,22,24,26,28,30,128,128,128,128,128,128,128,128,32,34,36,38,40,42,44,46,128,128,128,128,128,128,128,128,48,50,52,54,56,58,60,62,128,128,128,128,128,128,128,128]
+; AVX512BW-NEXT: vpshufb %zmm4, %zmm0, %zmm5
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm6 = [0,2,4,6,9,11,13,15]
+; AVX512BW-NEXT: vpermt2q %zmm3, %zmm6, %zmm5
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,1,3,5,7,9,11,13,15,128,128,128,128,128,128,128,128,17,19,21,23,25,27,29,31,128,128,128,128,128,128,128,128,33,35,37,39,41,43,45,47,128,128,128,128,128,128,128,128,49,51,53,55,57,59,61,63]
+; AVX512BW-NEXT: vpshufb %zmm3, %zmm1, %zmm7
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm8 = [1,3,5,7,9,11,13,15,128,128,128,128,128,128,128,128,17,19,21,23,25,27,29,31,128,128,128,128,128,128,128,128,33,35,37,39,41,43,45,47,128,128,128,128,128,128,128,128,49,51,53,55,57,59,61,63,128,128,128,128,128,128,128,128]
+; AVX512BW-NEXT: vpshufb %zmm8, %zmm0, %zmm1
+; AVX512BW-NEXT: vpermt2q %zmm7, %zmm6, %zmm1
; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0
-; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm1
-; AVX512BW-NEXT: vpmaddubsw (%rdi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmaddubsw 64(%rdi), %zmm1, %zmm1
+; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm7
+; AVX512BW-NEXT: vpshufb %zmm2, %zmm7, %zmm2
+; AVX512BW-NEXT: vpshufb %zmm4, %zmm0, %zmm4
+; AVX512BW-NEXT: vpermt2q %zmm2, %zmm6, %zmm4
+; AVX512BW-NEXT: vpshufb %zmm3, %zmm7, %zmm2
+; AVX512BW-NEXT: vpshufb %zmm8, %zmm0, %zmm0
+; AVX512BW-NEXT: vpermt2q %zmm2, %zmm6, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm5, %ymm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX512BW-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512BW-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm5, %xmm6
+; AVX512BW-NEXT: vpmovsxbd %xmm6, %zmm6
+; AVX512BW-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm4, %ymm7
+; AVX512BW-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero,xmm8[4],zero,zero,zero,xmm8[5],zero,zero,zero,xmm8[6],zero,zero,zero,xmm8[7],zero,zero,zero,xmm8[8],zero,zero,zero,xmm8[9],zero,zero,zero,xmm8[10],zero,zero,zero,xmm8[11],zero,zero,zero,xmm8[12],zero,zero,zero,xmm8[13],zero,zero,zero,xmm8[14],zero,zero,zero,xmm8[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm8, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm4, %xmm7
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm6, %zmm6
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm4, %zmm5, %zmm4
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm1, %ymm5
+; AVX512BW-NEXT: vextracti128 $1, %ymm5, %xmm7
+; AVX512BW-NEXT: vpmovsxbd %xmm7, %zmm7
+; AVX512BW-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512BW-NEXT: vextracti128 $1, %ymm1, %xmm8
+; AVX512BW-NEXT: vpmovsxbd %xmm8, %zmm8
+; AVX512BW-NEXT: vpmovsxbd %xmm1, %zmm1
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm9
+; AVX512BW-NEXT: vextracti128 $1, %ymm9, %xmm10
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero,xmm10[8],zero,zero,zero,xmm10[9],zero,zero,zero,xmm10[10],zero,zero,zero,xmm10[11],zero,zero,zero,xmm10[12],zero,zero,zero,xmm10[13],zero,zero,zero,xmm10[14],zero,zero,zero,xmm10[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm10, %zmm7, %zmm7
+; AVX512BW-NEXT: vpaddd %zmm7, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero,xmm9[8],zero,zero,zero,xmm9[9],zero,zero,zero,xmm9[10],zero,zero,zero,xmm9[11],zero,zero,zero,xmm9[12],zero,zero,zero,xmm9[13],zero,zero,zero,xmm9[14],zero,zero,zero,xmm9[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm5, %zmm5
+; AVX512BW-NEXT: vpaddd %zmm5, %zmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm5, %zmm8, %zmm5
+; AVX512BW-NEXT: vpaddd %zmm5, %zmm6, %zmm5
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,6,7],zmm5[2,3,6,7]
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm5[0,1,4,5]
+; AVX512BW-NEXT: vpackssdw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[2,3,6,7],zmm3[2,3,6,7]
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,4,5],zmm3[0,1,4,5]
+; AVX512BW-NEXT: vpackssdw %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: retq
%A = load <128 x i8>, ptr %Aptr
%B = load <128 x i8>, ptr %Bptr
diff --git a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
index 2ac2be5545dfd..e270da5e13ea4 100644
--- a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
+++ b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
@@ -61,22 +61,22 @@ define void @failing(ptr %0, ptr %1) nounwind {
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
; CHECK-NEXT: movdqu 1024(%rdx,%rsi), %xmm5
; CHECK-NEXT: movdqu 1040(%rdx,%rsi), %xmm6
-; CHECK-NEXT: movq %xmm5, %rdi
-; CHECK-NEXT: movq %xmm6, %r8
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm5[2,3,2,3]
+; CHECK-NEXT: movq %xmm7, %rdi
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; CHECK-NEXT: movq %xmm7, %r8
; CHECK-NEXT: movq %xmm5, %r9
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm6[2,3,2,3]
-; CHECK-NEXT: movq %xmm5, %r10
-; CHECK-NEXT: negq %r8
-; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: sbbq %r10, %r8
+; CHECK-NEXT: movq %xmm6, %r10
+; CHECK-NEXT: negq %r10
+; CHECK-NEXT: movq %rcx, %r10
+; CHECK-NEXT: sbbq %r8, %r10
; CHECK-NEXT: setge %r8b
; CHECK-NEXT: movzbl %r8b, %r8d
; CHECK-NEXT: negq %r8
; CHECK-NEXT: movq %r8, %xmm5
-; CHECK-NEXT: negq %rdi
-; CHECK-NEXT: movq %rcx, %rdi
-; CHECK-NEXT: sbbq %r9, %rdi
+; CHECK-NEXT: negq %r9
+; CHECK-NEXT: movq %rcx, %r8
+; CHECK-NEXT: sbbq %rdi, %r8
; CHECK-NEXT: setge %dil
; CHECK-NEXT: movzbl %dil, %edi
; CHECK-NEXT: negq %rdi
diff --git a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
index f632654f89e04..e4bd324a402c7 100644
--- a/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-strided-with-offset-256.ll
@@ -1,15 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST,AVX2-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST,AVX2-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512VL-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512VL-FAST-PERLANE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BW-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BW-FAST-PERLANE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512BWVL-FAST-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512BWVL-FAST-PERLANE
define void @shuffle_v32i8_to_v16i8_1(ptr %L, ptr %S) nounwind {
; AVX-LABEL: shuffle_v32i8_to_v16i8_1:
@@ -908,3 +908,12 @@ define void @shuffle_v32i8_to_v4i8_7(ptr %L, ptr %S) nounwind {
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX2-FAST-ALL: {{.*}}
+; AVX2-FAST-PERLANE: {{.*}}
+; AVX512BW-FAST-ALL: {{.*}}
+; AVX512BW-FAST-PERLANE: {{.*}}
+; AVX512BWVL-FAST-ALL: {{.*}}
+; AVX512BWVL-FAST-PERLANE: {{.*}}
+; AVX512VL-FAST-ALL: {{.*}}
+; AVX512VL-FAST-PERLANE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
index 26af46263c0e2..ee29a1c472e25 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
@@ -889,253 +889,54 @@ define <16 x i8> @evenelts_v32i16_trunc_v16i16_to_v16i8(<32 x i16> %n2) nounwind
;
; AVX1-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %rbx
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpextrw $6, %xmm2, %eax
-; AVX1-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX1-NEXT: vpextrw $2, %xmm2, %edx
-; AVX1-NEXT: vmovd %xmm2, %esi
-; AVX1-NEXT: vpextrw $6, %xmm1, %edi
-; AVX1-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX1-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX1-NEXT: vmovd %xmm1, %r10d
+; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3],xmm1[4],xmm3[5],xmm1[6],xmm3[7]
+; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX1-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX1-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX1-NEXT: vmovd %xmm1, %r14d
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrw $6, %xmm2, %eax
-; AVX2-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm2, %edx
-; AVX2-NEXT: vmovd %xmm2, %esi
-; AVX2-NEXT: vpextrw $6, %xmm1, %edi
-; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX2-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX2-NEXT: vmovd %xmm1, %r10d
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX2-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX2-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX2-NEXT: vmovd %xmm1, %r14d
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
-;
-; AVX512F-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vpmovdb %zmm0, %xmm1
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrw $6, %xmm2, %eax
-; AVX512F-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX512F-NEXT: vpextrw $2, %xmm2, %edx
-; AVX512F-NEXT: vmovd %xmm2, %esi
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrw $6, %xmm2, %edi
-; AVX512F-NEXT: vpextrw $4, %xmm2, %r8d
-; AVX512F-NEXT: vpextrw $2, %xmm2, %r9d
-; AVX512F-NEXT: vmovd %xmm2, %r10d
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512F-NEXT: vpextrw $4, %xmm0, %ebx
-; AVX512F-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512F-NEXT: vpinsrb $5, %ebp, %xmm1, %xmm0
-; AVX512F-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %rbp
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbp
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512VL-NEXT: vmovd %xmm1, %esi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %edi
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX512VL-NEXT: vmovd %xmm1, %r10d
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX512VL-NEXT: vmovd %xmm1, %r14d
-; AVX512VL-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %rbp
-; AVX512VL-NEXT: vzeroupper
-; AVX512VL-NEXT: retq
+; AVX2-SLOW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-SLOW-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vzeroupper
+; AVX2-SLOW-NEXT: retq
;
-; AVX512BW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: pushq %rbp
-; AVX512BW-NEXT: pushq %rbx
-; AVX512BW-NEXT: vpmovdb %zmm0, %xmm1
-; AVX512BW-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512BW-NEXT: vpextrw $6, %xmm2, %eax
-; AVX512BW-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX512BW-NEXT: vpextrw $2, %xmm2, %edx
-; AVX512BW-NEXT: vmovd %xmm2, %esi
-; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX512BW-NEXT: vpextrw $6, %xmm2, %edi
-; AVX512BW-NEXT: vpextrw $4, %xmm2, %r8d
-; AVX512BW-NEXT: vpextrw $2, %xmm2, %r9d
-; AVX512BW-NEXT: vmovd %xmm2, %r10d
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512BW-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512BW-NEXT: vpextrw $4, %xmm0, %ebx
-; AVX512BW-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512BW-NEXT: vpinsrb $5, %ebp, %xmm1, %xmm0
-; AVX512BW-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BW-NEXT: popq %rbx
-; AVX512BW-NEXT: popq %rbp
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
+; AVX2-FAST-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpmovsxdq {{.*#+}} ymm2 = [84148480,218892552,353636624,488380696]
+; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX2-FAST-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-FAST-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
;
-; AVX512BWVL-ONLY-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BWVL-ONLY: # %bb.0:
-; AVX512BWVL-ONLY-NEXT: pushq %rbp
-; AVX512BWVL-ONLY-NEXT: pushq %r14
-; AVX512BWVL-ONLY-NEXT: pushq %rbx
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %esi
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %edi
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %r10d
-; AVX512BWVL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %r14d
-; AVX512BWVL-ONLY-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: popq %rbx
-; AVX512BWVL-ONLY-NEXT: popq %r14
-; AVX512BWVL-ONLY-NEXT: popq %rbp
-; AVX512BWVL-ONLY-NEXT: vzeroupper
-; AVX512BWVL-ONLY-NEXT: retq
-;
-; AVX512VBMI-FAST-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-FAST: # %bb.0:
-; AVX512VBMI-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,79]
-; AVX512VBMI-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI-FAST-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
-; AVX512VBMI-FAST-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-FAST-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512VBMI-FAST-NEXT: vpinsrb $15, %eax, %xmm1, %xmm0
-; AVX512VBMI-FAST-NEXT: vzeroupper
-; AVX512VBMI-FAST-NEXT: retq
-;
-; AVX512VBMI-SLOW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-SLOW: # %bb.0:
-; AVX512VBMI-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [0,4,8,12,16,20,24,28,32,36,40,44,48,77,78,79]
-; AVX512VBMI-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI-SLOW-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
-; AVX512VBMI-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512VBMI-SLOW-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512VBMI-SLOW-NEXT: vpextrw $2, %xmm0, %edx
-; AVX512VBMI-SLOW-NEXT: vpinsrb $13, %edx, %xmm1, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vzeroupper
-; AVX512VBMI-SLOW-NEXT: retq
+; AVX512-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%n0 = shufflevector <32 x i16> %n2, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%n1 = trunc <16 x i16> %n0 to <16 x i8>
ret <16 x i8> %n1
@@ -1230,260 +1031,46 @@ define <16 x i8> @oddelts_v32i16_trunc_v16i16_to_v16i8(<32 x i16> %n2) nounwind
;
; AVX1-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %rbx
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpextrw $7, %xmm2, %eax
-; AVX1-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX1-NEXT: vpextrw $3, %xmm2, %edx
-; AVX1-NEXT: vpextrw $1, %xmm2, %esi
-; AVX1-NEXT: vpextrw $7, %xmm1, %edi
-; AVX1-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX1-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX1-NEXT: vpextrw $1, %xmm1, %r10d
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX1-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX1-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX1-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrw $7, %xmm2, %eax
-; AVX2-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX2-NEXT: vpextrw $3, %xmm2, %edx
-; AVX2-NEXT: vpextrw $1, %xmm2, %esi
-; AVX2-NEXT: vpextrw $7, %xmm1, %edi
-; AVX2-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX2-NEXT: vpextrw $1, %xmm1, %r10d
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX2-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX2-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX2-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: pushq %r14
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512F-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512F-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512F-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512F-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512F-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512F-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512F-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512F-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512F-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %r14
-; AVX512F-NEXT: popq %rbp
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbp
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512VL-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %rbp
-; AVX512VL-NEXT: vzeroupper
-; AVX512VL-NEXT: retq
-;
-; AVX512BW-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: pushq %rbp
-; AVX512BW-NEXT: pushq %r14
-; AVX512BW-NEXT: pushq %rbx
-; AVX512BW-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BW-NEXT: popq %rbx
-; AVX512BW-NEXT: popq %r14
-; AVX512BW-NEXT: popq %rbp
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-ONLY-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BWVL-ONLY: # %bb.0:
-; AVX512BWVL-ONLY-NEXT: pushq %rbp
-; AVX512BWVL-ONLY-NEXT: pushq %r14
-; AVX512BWVL-ONLY-NEXT: pushq %rbx
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512BWVL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512BWVL-ONLY-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BWVL-ONLY-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: popq %rbx
-; AVX512BWVL-ONLY-NEXT: popq %r14
-; AVX512BWVL-ONLY-NEXT: popq %rbp
-; AVX512BWVL-ONLY-NEXT: vzeroupper
-; AVX512BWVL-ONLY-NEXT: retq
-;
-; AVX512VBMI-FAST-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-FAST: # %bb.0:
-; AVX512VBMI-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]
-; AVX512VBMI-FAST-NEXT: vpermb %zmm0, %zmm1, %zmm0
-; AVX512VBMI-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512VBMI-FAST-NEXT: vzeroupper
-; AVX512VBMI-FAST-NEXT: retq
-;
-; AVX512VBMI-SLOW-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-SLOW: # %bb.0:
-; AVX512VBMI-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [2,6,10,14,18,22,26,30,34,38,42,46,50,u,u,u]
-; AVX512VBMI-SLOW-NEXT: vpermb %zmm0, %zmm1, %zmm1
-; AVX512VBMI-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512VBMI-SLOW-NEXT: vpextrw $5, %xmm0, %ecx
-; AVX512VBMI-SLOW-NEXT: vpextrw $3, %xmm0, %edx
-; AVX512VBMI-SLOW-NEXT: vpinsrb $13, %edx, %xmm1, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vzeroupper
-; AVX512VBMI-SLOW-NEXT: retq
+; AVX512-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%n0 = shufflevector <32 x i16> %n2, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
%n1 = trunc <16 x i16> %n0 to <16 x i8>
ret <16 x i8> %n1
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512BWVL-ONLY: {{.*}}
; AVX512VBMI: {{.*}}
+; AVX512VBMI-FAST: {{.*}}
+; AVX512VBMI-SLOW: {{.*}}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 05c855ed90b3f..597253118e3c9 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -6,8 +6,8 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-ALL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST-PERLANE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BWVL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BWVL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512VBMIVL
@@ -1301,73 +1301,74 @@ define void @trunc_v4i64_to_v4i8(ptr %L, ptr %S) nounwind {
define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
; AVX1-LABEL: negative:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[u,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u],zero,zero,zero,zero,zero,zero,zero,xmm0[0,2,4,6,8,10,12,14]
-; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615]
-; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vmovd %xmm1, %eax
+; AVX1-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: negative:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX2-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm1, %eax
+; AVX2-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: negative:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512F-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX512F-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT: vpbroadcastw {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX512F-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: negative:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm0[2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm1 & ~ymm2) | ymm0
-; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm2[0,3,2,3]
-; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512VL-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX512VL-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX512VL-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: negative:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BW-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX512BW-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vmovd %xmm1, %eax
+; AVX512BW-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: negative:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BWVL-NEXT: movl $65537, %eax # imm = 0x10001
-; AVX512BWVL-NEXT: kmovd %eax, %k1
-; AVX512BWVL-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
-; AVX512BWVL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512BWVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vmovd %xmm1, %eax
+; AVX512BWVL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
;
; AVX512VBMIVL-LABEL: negative:
; AVX512VBMIVL: # %bb.0:
-; AVX512VBMIVL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [32,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
-; AVX512VBMIVL-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512VBMIVL-NEXT: vpermt2b %ymm1, %ymm2, %ymm0
-; AVX512VBMIVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMIVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512VBMIVL-NEXT: vmovd %xmm1, %eax
+; AVX512VBMIVL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512VBMIVL-NEXT: vzeroupper
; AVX512VBMIVL-NEXT: retq
%strided.vec = shufflevector <32 x i8> %v, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -1375,3 +1376,6 @@ define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
%merged = insertelement <16 x i8> %strided.vec, i8 %w0, i32 0
ret <16 x i8> %merged
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512BW-ALL: {{.*}}
+; AVX512BW-FAST-PERLANE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
index e0410ae0cc5cb..7a146f78dafb7 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
@@ -164,62 +164,62 @@ define void @load_i16_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i16_stride2_vf8:
; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i16_stride2_vf8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i16_stride2_vf8:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-FP-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FP-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-FP-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FP-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-FP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i16_stride2_vf8:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FCP-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-FCP-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FCP-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i16_stride2_vf8:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
index b609299e5f757..4f11e30bfe0d0 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
@@ -416,130 +416,130 @@ define void @load_i8_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i8_stride2_vf16:
; AVX: # %bb.0:
-; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i8_stride2_vf16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX2-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i8_stride2_vf16:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-FP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-FP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX2-FP-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX2-FP-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX2-FP-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX2-FP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-FP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-FP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i8_stride2_vf16:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX2-FCP-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX2-FCP-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX2-FCP-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i8_stride2_vf16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX512-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX512-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i8_stride2_vf16:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512-FCP-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512-FCP-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX512-FCP-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i8_stride2_vf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512DQ-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512DQ-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX512DQ-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX512DQ-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512DQ-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512DQ-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512DQ-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512DQ-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX512DQ-NEXT: vmovdqa %xmm0, (%rdx)
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i8_stride2_vf16:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512DQ-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512DQ-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512DQ-FCP-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512DQ-FCP-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX512DQ-FCP-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i8_stride2_vf16:
@@ -700,20 +700,20 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512-LABEL: load_i8_stride2_vf32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512-NEXT: vpshufb %ymm0, %ymm1, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
-; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
@@ -735,20 +735,20 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512DQ-LABEL: load_i8_stride2_vf32:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT: vpshufb %ymm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512DQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512DQ-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-narrow-binop.ll b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
index ad345213c1472..801f5929e35d8 100644
--- a/llvm/test/CodeGen/X86/vector-narrow-binop.ll
+++ b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
@@ -60,15 +60,15 @@ define <8 x i32> @PR32790(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d
define <4 x i32> @do_not_use_256bit_op(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) {
; SSE-LABEL: do_not_use_256bit_op:
; SSE: # %bb.0:
-; SSE-NEXT: pand %xmm2, %xmm0
; SSE-NEXT: pand %xmm3, %xmm1
+; SSE-NEXT: pand %xmm2, %xmm0
; SSE-NEXT: psubd %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX-LABEL: do_not_use_256bit_op:
; AVX: # %bb.0:
-; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX-NEXT: vpand %xmm3, %xmm1, %xmm1
+; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%concat1 = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/CodeGen/X86/vpdpwssd.ll b/llvm/test/CodeGen/X86/vpdpwssd.ll
index ea97800505bc2..5cc2e78fe7d11 100644
--- a/llvm/test/CodeGen/X86/vpdpwssd.ll
+++ b/llvm/test/CodeGen/X86/vpdpwssd.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI,AVX512VL-VNNI-SLOW
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512-VNNI
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI,AVX512VL-VNNI-FAST
define <16 x i32> @vpdpwssd_test(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) {
; CHECK-LABEL: vpdpwssd_test:
@@ -31,23 +31,57 @@ define <16 x i32> @vpdpwssd_v16i32_accumulate(<32 x i16> %a0, <32 x i16> %a1, <1
}
define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x i32> %a2) {
-; AVX512VL-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
-; AVX512VL-VNNI: # %bb.0:
-; AVX512VL-VNNI-NEXT: vpdpwssd %ymm1, %ymm0, %ymm2
-; AVX512VL-VNNI-NEXT: vmovdqa %ymm2, %ymm0
-; AVX512VL-VNNI-NEXT: retq
+; AVX512VL-VNNI-SLOW-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI-SLOW: # %bb.0:
+; AVX512VL-VNNI-SLOW-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512VL-VNNI-SLOW-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512VL-VNNI-SLOW-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-VNNI-SLOW-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
+; AVX512VL-VNNI-SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512VL-VNNI-SLOW-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX512VL-VNNI-SLOW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512VL-VNNI-SLOW-NEXT: retq
;
; AVX-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
; AVX-VNNI: # %bb.0:
-; AVX-VNNI-NEXT: {vex} vpdpwssd %ymm1, %ymm0, %ymm2
-; AVX-VNNI-NEXT: vmovdqa %ymm2, %ymm0
+; AVX-VNNI-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX-VNNI-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX-VNNI-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX-VNNI-NEXT: vpmovqd %zmm0, %ymm1
+; AVX-VNNI-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX-VNNI-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
+; AVX-VNNI-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX-VNNI-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX-VNNI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX-VNNI-NEXT: retq
;
; AVX512-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
; AVX512-VNNI: # %bb.0:
-; AVX512-VNNI-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-VNNI-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-VNNI-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-VNNI-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-VNNI-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512-VNNI-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
+; AVX512-VNNI-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512-VNNI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX512-VNNI-NEXT: retq
+;
+; AVX512VL-VNNI-FAST-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI-FAST: # %bb.0:
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512VL-VNNI-FAST-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512VL-VNNI-FAST-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-VNNI-FAST-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
+; AVX512VL-VNNI-FAST-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512VL-VNNI-FAST-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512VL-VNNI-FAST-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512VL-VNNI-FAST-NEXT: retq
%x0 = sext <16 x i16> %a0 to <16 x i32>
%x1 = sext <16 x i16> %a1 to <16 x i32>
%m = mul nsw <16 x i32> %x0, %x1
@@ -61,20 +95,41 @@ define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x
define <4 x i32> @vpdpwssd_v4i32_accumulate(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
; AVX512VL-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX512VL-VNNI: # %bb.0:
-; AVX512VL-VNNI-NEXT: vpdpwssd %xmm1, %xmm0, %xmm2
-; AVX512VL-VNNI-NEXT: vmovdqa %xmm2, %xmm0
+; AVX512VL-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512VL-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512VL-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX512VL-VNNI-NEXT: vpmovqd %ymm0, %xmm1
+; AVX512VL-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512VL-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
+; AVX512VL-VNNI-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512VL-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512VL-VNNI-NEXT: vzeroupper
; AVX512VL-VNNI-NEXT: retq
;
; AVX-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX-VNNI: # %bb.0:
-; AVX-VNNI-NEXT: {vex} vpdpwssd %xmm1, %xmm0, %xmm2
-; AVX-VNNI-NEXT: vmovdqa %xmm2, %xmm0
+; AVX-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX-VNNI-NEXT: vpmovqd %ymm0, %xmm1
+; AVX-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
+; AVX-VNNI-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX-VNNI-NEXT: vzeroupper
; AVX-VNNI-NEXT: retq
;
; AVX512-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX512-VNNI: # %bb.0:
-; AVX512-VNNI-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-VNNI-NEXT: vpmovsxwd %xmm0, %ymm0
+; AVX512-VNNI-NEXT: vpmovsxwd %xmm1, %ymm1
+; AVX512-VNNI-NEXT: vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} xmm3 = xmm0[0,2],xmm1[0,2]
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
+; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm3, %xmm1
+; AVX512-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-VNNI-NEXT: vzeroupper
; AVX512-VNNI-NEXT: retq
%x0 = sext <8 x i16> %a0 to <8 x i32>
%x1 = sext <8 x i16> %a1 to <8 x i32>
diff --git a/llvm/test/CodeGen/X86/vselect-avx.ll b/llvm/test/CodeGen/X86/vselect-avx.ll
index b684bb015ecd0..469a05a90f970 100644
--- a/llvm/test/CodeGen/X86/vselect-avx.ll
+++ b/llvm/test/CodeGen/X86/vselect-avx.ll
@@ -263,11 +263,11 @@ define <4 x i64> @vselect_concat_split_v16i8(<4 x i64> %a, <4 x i64> %b, <4 x i6
; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5
; AVX1-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
; AVX1-NEXT: vpcmpgtb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT: vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: vselect_concat_split_v16i8:
- Previous message: [llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
- Next message: [llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the llvm-commits
mailing list