[llvm] [DAG] SelectionDAGBuilder::visitShuffleVector - split shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns (PR #180573)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 03:04:39 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/180573
>From 3b5d9390b868bca36e2991a18f060746014093c1 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 9 Feb 2026 17:46:25 +0000
Subject: [PATCH] [DAG] SelectionDAGBuilder::visitShuffleVector - split
shuffle(concat(x,y),undef,mask) -> shuffle(x,y,mask) patterns
If an unary shuffle's sources are twice the width of the result, then split the lower/upper subvectors and perform a binary shuffle instead of scalarizing
Fixes #88030
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 25 +
...ffer-fat-pointers-contents-legalization.ll | 88 +--
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 288 +++++----
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 372 ++++++------
...m.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll | 478 ++++++++-------
.../AMDGPU/load-local-redundant-copies.ll | 33 +-
llvm/test/CodeGen/AMDGPU/packed-fp64.ll | 20 +-
llvm/test/CodeGen/AMDGPU/packed-u64.ll | 20 +-
llvm/test/CodeGen/AMDGPU/shl.v2i64.ll | 49 +-
llvm/test/CodeGen/LoongArch/lasx/vhaddw.ll | 170 +++---
llvm/test/CodeGen/LoongArch/lasx/vhsubw.ll | 170 +++---
llvm/test/CodeGen/LoongArch/lasx/vsllwil.ll | 30 +-
llvm/test/CodeGen/LoongArch/lasx/xvexth.ll | 48 +-
llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll | 4 +-
llvm/test/CodeGen/Thumb2/mve-shuffle.ll | 8 +-
llvm/test/CodeGen/Thumb2/mve-vld2.ll | 4 +-
...d_vector_inreg_of_broadcast_from_memory.ll | 8 +-
llvm/test/CodeGen/X86/combine-pmaddwd.ll | 17 +-
llvm/test/CodeGen/X86/madd.ll | 354 +++++++++--
llvm/test/CodeGen/X86/pmaddubsw.ll | 361 +++++++++++-
.../test/CodeGen/X86/setcc-non-simple-type.ll | 22 +-
llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll | 554 +++---------------
llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll | 75 +--
.../vector-interleaved-load-i16-stride-2.ll | 80 +--
.../vector-interleaved-load-i8-stride-2.ll | 248 ++++----
llvm/test/CodeGen/X86/vector-narrow-binop.ll | 4 +-
.../X86/vector-reduce-add-subvector.ll | 120 +++-
llvm/test/CodeGen/X86/vpdpwssd.ll | 89 ++-
llvm/test/CodeGen/X86/vselect-avx.ll | 10 +-
29 files changed, 2079 insertions(+), 1670 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 0f6bd53cafdd8..bb660f0f6dda0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4331,6 +4331,31 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
assert(SrcNumElts > MaskNumElts);
+ // See if we can recreate a binary shuffle by splitting an unary shuffle with
+ // sources twice the destination size.
+ if (SrcNumElts == (MaskNumElts * 2) && Src2.isUndef() &&
+ TLI.isTypeLegal(VT)) {
+ // Canonicalize refs to undef Src2 to undef mask index.
+ SmallVector<int, 16> SplitMask(Mask.begin(), Mask.end());
+ for (int &M : SplitMask)
+ M = M >= (int)SrcNumElts ? -1 : M;
+ bool LoOnly = all_of(SplitMask,
+ [MaskNumElts](int M) { return M < (int)MaskNumElts; });
+ bool HiOnly = all_of(SplitMask, [MaskNumElts](int M) {
+ return M < 0 || M >= (int)MaskNumElts;
+ });
+ if (LoOnly || HiOnly || Src1.getOpcode() == ISD::CONCAT_VECTORS ||
+ isa<LoadSDNode>(Src1) ||
+ (TLI.isExtractSubvectorCheap(VT, SrcVT, 0) &&
+ TLI.isExtractSubvectorCheap(VT, SrcVT, MaskNumElts))) {
+ SDValue LHS = DAG.getExtractSubvector(DL, VT, Src1, 0);
+ SDValue RHS = DAG.getExtractSubvector(DL, VT, Src1, MaskNumElts);
+ SDValue Result = DAG.getVectorShuffle(VT, DL, LHS, RHS, SplitMask);
+ setValue(&I, Result);
+ return;
+ }
+ }
+
// Analyze the access pattern of the vector to see if we can extract
// two subvectors and do the shuffle.
int StartIdx[2] = {-1, -1}; // StartIdx to extract from
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index 1d2372af519da..8fdefef44c425 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2735,34 +2735,34 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: s_mov_b32 s4, 0xc0c0004
-; SDAG-NEXT: v_perm_b32 v8, v8, v9, s4
-; SDAG-NEXT: v_perm_b32 v9, v10, v11, s4
-; SDAG-NEXT: buffer_load_ubyte v10, off, s[0:3], s32
+; SDAG-NEXT: v_perm_b32 v20, v20, v21, s4
+; SDAG-NEXT: v_perm_b32 v21, v22, v23, s4
+; SDAG-NEXT: buffer_load_ubyte v22, off, s[0:3], s32
; SDAG-NEXT: v_perm_b32 v12, v12, v13, s4
; SDAG-NEXT: v_perm_b32 v13, v14, v15, s4
+; SDAG-NEXT: v_perm_b32 v8, v8, v9, s4
+; SDAG-NEXT: v_perm_b32 v9, v10, v11, s4
; SDAG-NEXT: v_perm_b32 v4, v4, v5, s4
; SDAG-NEXT: v_perm_b32 v5, v6, v7, s4
-; SDAG-NEXT: v_perm_b32 v0, v0, v1, s4
-; SDAG-NEXT: v_perm_b32 v6, v2, v3, s4
-; SDAG-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; SDAG-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; SDAG-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; SDAG-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; SDAG-NEXT: v_perm_b32 v7, v28, v29, s4
-; SDAG-NEXT: v_perm_b32 v11, v24, v25, s4
-; SDAG-NEXT: v_perm_b32 v14, v26, v27, s4
-; SDAG-NEXT: v_perm_b32 v15, v20, v21, s4
-; SDAG-NEXT: v_perm_b32 v20, v22, v23, s4
+; SDAG-NEXT: v_perm_b32 v10, v0, v1, s4
+; SDAG-NEXT: v_perm_b32 v3, v2, v3, s4
+; SDAG-NEXT: v_perm_b32 v24, v24, v25, s4
+; SDAG-NEXT: v_perm_b32 v25, v26, v27, s4
+; SDAG-NEXT: v_perm_b32 v23, v28, v29, s4
; SDAG-NEXT: v_perm_b32 v16, v16, v17, s4
; SDAG-NEXT: v_perm_b32 v17, v18, v19, s4
-; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; SDAG-NEXT: v_lshl_or_b32 v5, v14, 16, v11
-; SDAG-NEXT: v_lshl_or_b32 v4, v20, 16, v15
-; SDAG-NEXT: v_lshl_or_b32 v3, v17, 16, v16
-; SDAG-NEXT: s_waitcnt vmcnt(1)
-; SDAG-NEXT: v_perm_b32 v0, v30, v10, s4
-; SDAG-NEXT: v_lshl_or_b32 v6, v0, 16, v7
-; SDAG-NEXT: buffer_store_dwordx4 v[3:6], off, s[16:19], 0 offset:16
+; SDAG-NEXT: v_lshl_or_b32 v7, v13, 16, v12
+; SDAG-NEXT: v_lshl_or_b32 v6, v9, 16, v8
+; SDAG-NEXT: v_lshl_or_b32 v5, v5, 16, v4
+; SDAG-NEXT: v_lshl_or_b32 v4, v3, 16, v10
+; SDAG-NEXT: v_lshl_or_b32 v2, v25, 16, v24
+; SDAG-NEXT: v_lshl_or_b32 v1, v21, 16, v20
+; SDAG-NEXT: v_lshl_or_b32 v0, v17, 16, v16
+; SDAG-NEXT: s_waitcnt vmcnt(0)
+; SDAG-NEXT: v_perm_b32 v3, v30, v22, s4
+; SDAG-NEXT: v_lshl_or_b32 v3, v3, 16, v23
+; SDAG-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0
+; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:16
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2770,34 +2770,34 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v8, v8, v9, s4
-; GISEL-NEXT: v_perm_b32 v9, v10, v11, s4
-; GISEL-NEXT: buffer_load_ubyte v10, off, s[0:3], s32
+; GISEL-NEXT: v_perm_b32 v20, v20, v21, s4
+; GISEL-NEXT: v_perm_b32 v21, v22, v23, s4
+; GISEL-NEXT: buffer_load_ubyte v22, off, s[0:3], s32
; GISEL-NEXT: v_perm_b32 v12, v12, v13, s4
; GISEL-NEXT: v_perm_b32 v13, v14, v15, s4
+; GISEL-NEXT: v_perm_b32 v8, v8, v9, s4
+; GISEL-NEXT: v_perm_b32 v9, v10, v11, s4
; GISEL-NEXT: v_perm_b32 v4, v4, v5, s4
; GISEL-NEXT: v_perm_b32 v5, v6, v7, s4
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v6, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; GISEL-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; GISEL-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GISEL-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; GISEL-NEXT: v_perm_b32 v7, v28, v29, s4
-; GISEL-NEXT: v_perm_b32 v11, v24, v25, s4
-; GISEL-NEXT: v_perm_b32 v14, v26, v27, s4
-; GISEL-NEXT: v_perm_b32 v15, v20, v21, s4
-; GISEL-NEXT: v_perm_b32 v20, v22, v23, s4
+; GISEL-NEXT: v_perm_b32 v10, v0, v1, s4
+; GISEL-NEXT: v_perm_b32 v3, v2, v3, s4
+; GISEL-NEXT: v_perm_b32 v24, v24, v25, s4
+; GISEL-NEXT: v_perm_b32 v25, v26, v27, s4
+; GISEL-NEXT: v_perm_b32 v23, v28, v29, s4
; GISEL-NEXT: v_perm_b32 v16, v16, v17, s4
; GISEL-NEXT: v_perm_b32 v17, v18, v19, s4
-; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; GISEL-NEXT: v_lshl_or_b32 v5, v14, 16, v11
-; GISEL-NEXT: v_lshl_or_b32 v4, v20, 16, v15
-; GISEL-NEXT: v_lshl_or_b32 v3, v17, 16, v16
-; GISEL-NEXT: s_waitcnt vmcnt(1)
-; GISEL-NEXT: v_perm_b32 v0, v30, v10, s4
-; GISEL-NEXT: v_lshl_or_b32 v6, v0, 16, v7
-; GISEL-NEXT: buffer_store_dwordx4 v[3:6], off, s[16:19], 0 offset:16
+; GISEL-NEXT: v_lshl_or_b32 v7, v13, 16, v12
+; GISEL-NEXT: v_lshl_or_b32 v6, v9, 16, v8
+; GISEL-NEXT: v_lshl_or_b32 v5, v5, 16, v4
+; GISEL-NEXT: v_lshl_or_b32 v4, v3, 16, v10
+; GISEL-NEXT: v_lshl_or_b32 v2, v25, 16, v24
+; GISEL-NEXT: v_lshl_or_b32 v1, v21, 16, v20
+; GISEL-NEXT: v_lshl_or_b32 v0, v17, 16, v16
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_perm_b32 v3, v30, v22, s4
+; GISEL-NEXT: v_lshl_or_b32 v3, v3, 16, v23
+; GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0
+; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0 offset:16
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index 36b1c9f1ffbfc..ad0b12093042d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -2860,14 +2860,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v40, s24
-; SDAG-NEXT: v_mov_b32_e32 v41, s25
-; SDAG-NEXT: v_mov_b32_e32 v42, s26
-; SDAG-NEXT: v_mov_b32_e32 v43, s27
-; SDAG-NEXT: v_mov_b32_e32 v44, s28
-; SDAG-NEXT: v_mov_b32_e32 v45, s29
-; SDAG-NEXT: v_mov_b32_e32 v46, s30
-; SDAG-NEXT: v_mov_b32_e32 v47, s31
+; SDAG-NEXT: v_mov_b32_e32 v44, s24
+; SDAG-NEXT: v_mov_b32_e32 v45, s25
+; SDAG-NEXT: v_mov_b32_e32 v46, s26
+; SDAG-NEXT: v_mov_b32_e32 v47, s27
+; SDAG-NEXT: v_mov_b32_e32 v40, s28
+; SDAG-NEXT: v_mov_b32_e32 v41, s29
+; SDAG-NEXT: v_mov_b32_e32 v42, s30
+; SDAG-NEXT: v_mov_b32_e32 v43, s31
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -2878,7 +2878,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[44:47], v[40:43], v[0:15]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -2976,14 +2976,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v8, s24
-; HEURRC-NEXT: v_mov_b32_e32 v9, s25
-; HEURRC-NEXT: v_mov_b32_e32 v10, s26
-; HEURRC-NEXT: v_mov_b32_e32 v11, s27
-; HEURRC-NEXT: v_mov_b32_e32 v12, s28
-; HEURRC-NEXT: v_mov_b32_e32 v13, s29
-; HEURRC-NEXT: v_mov_b32_e32 v14, s30
-; HEURRC-NEXT: v_mov_b32_e32 v15, s31
+; HEURRC-NEXT: v_mov_b32_e32 v12, s24
+; HEURRC-NEXT: v_mov_b32_e32 v13, s25
+; HEURRC-NEXT: v_mov_b32_e32 v14, s26
+; HEURRC-NEXT: v_mov_b32_e32 v15, s27
+; HEURRC-NEXT: v_mov_b32_e32 v8, s28
+; HEURRC-NEXT: v_mov_b32_e32 v9, s29
+; HEURRC-NEXT: v_mov_b32_e32 v10, s30
+; HEURRC-NEXT: v_mov_b32_e32 v11, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
@@ -3002,7 +3002,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[12:15], v[8:11], a[0:15]
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
; HEURRC-NEXT: v_mov_b32_e32 v9, s17
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
@@ -3047,14 +3047,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v46, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v47, s31
+; VGPRRC-NEXT: v_mov_b32_e32 v44, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v45, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v46, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v47, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -3065,7 +3065,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[44:47], v[40:43], v[0:15]
; VGPRRC-NEXT: s_nop 11
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
@@ -3255,14 +3255,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[34:35], 32
; SDAG-NEXT: v_mov_b64_e32 v[36:37], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v40, s24
-; SDAG-NEXT: v_mov_b32_e32 v41, s25
-; SDAG-NEXT: v_mov_b32_e32 v42, s26
-; SDAG-NEXT: v_mov_b32_e32 v43, s27
-; SDAG-NEXT: v_mov_b32_e32 v44, s28
-; SDAG-NEXT: v_mov_b32_e32 v45, s29
-; SDAG-NEXT: v_mov_b32_e32 v46, s30
-; SDAG-NEXT: v_mov_b32_e32 v47, s31
+; SDAG-NEXT: v_mov_b32_e32 v44, s24
+; SDAG-NEXT: v_mov_b32_e32 v45, s25
+; SDAG-NEXT: v_mov_b32_e32 v46, s26
+; SDAG-NEXT: v_mov_b32_e32 v47, s27
+; SDAG-NEXT: v_mov_b32_e32 v40, s28
+; SDAG-NEXT: v_mov_b32_e32 v41, s29
+; SDAG-NEXT: v_mov_b32_e32 v42, s30
+; SDAG-NEXT: v_mov_b32_e32 v43, s31
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -3273,7 +3273,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], 0
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[44:47], v[40:43], v[0:15] cbsz:2 abid:3 blgp:1
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -3371,14 +3371,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v8, s24
-; HEURRC-NEXT: v_mov_b32_e32 v9, s25
-; HEURRC-NEXT: v_mov_b32_e32 v10, s26
-; HEURRC-NEXT: v_mov_b32_e32 v11, s27
-; HEURRC-NEXT: v_mov_b32_e32 v12, s28
-; HEURRC-NEXT: v_mov_b32_e32 v13, s29
-; HEURRC-NEXT: v_mov_b32_e32 v14, s30
-; HEURRC-NEXT: v_mov_b32_e32 v15, s31
+; HEURRC-NEXT: v_mov_b32_e32 v12, s24
+; HEURRC-NEXT: v_mov_b32_e32 v13, s25
+; HEURRC-NEXT: v_mov_b32_e32 v14, s26
+; HEURRC-NEXT: v_mov_b32_e32 v15, s27
+; HEURRC-NEXT: v_mov_b32_e32 v8, s28
+; HEURRC-NEXT: v_mov_b32_e32 v9, s29
+; HEURRC-NEXT: v_mov_b32_e32 v10, s30
+; HEURRC-NEXT: v_mov_b32_e32 v11, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
@@ -3397,7 +3397,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: s_nop 0
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[12:15], v[8:11], a[0:15] cbsz:2 abid:3 blgp:1
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
; HEURRC-NEXT: v_mov_b32_e32 v9, s17
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
@@ -3442,14 +3442,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v40, s24
-; VGPRRC-NEXT: v_mov_b32_e32 v41, s25
-; VGPRRC-NEXT: v_mov_b32_e32 v42, s26
-; VGPRRC-NEXT: v_mov_b32_e32 v43, s27
-; VGPRRC-NEXT: v_mov_b32_e32 v44, s28
-; VGPRRC-NEXT: v_mov_b32_e32 v45, s29
-; VGPRRC-NEXT: v_mov_b32_e32 v46, s30
-; VGPRRC-NEXT: v_mov_b32_e32 v47, s31
+; VGPRRC-NEXT: v_mov_b32_e32 v44, s24
+; VGPRRC-NEXT: v_mov_b32_e32 v45, s25
+; VGPRRC-NEXT: v_mov_b32_e32 v46, s26
+; VGPRRC-NEXT: v_mov_b32_e32 v47, s27
+; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -3460,7 +3460,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], 0
; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[44:47], v[40:43], v[0:15] cbsz:2 abid:3 blgp:1
; VGPRRC-NEXT: s_nop 11
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
@@ -3950,20 +3950,19 @@ define <16 x i32> @test_mfma_i32_32x32x32_i8__mac__flags(<4 x i32> %arg0, <4 x i
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s20
-; SDAG-NEXT: v_mov_b32_e32 v33, s21
-; SDAG-NEXT: v_mov_b32_e32 v34, s22
-; SDAG-NEXT: v_mov_b32_e32 v35, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v36, s24
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v32, s28
+; SDAG-NEXT: v_mov_b32_e32 v33, s29
+; SDAG-NEXT: v_mov_b32_e32 v34, s30
+; SDAG-NEXT: v_mov_b32_e32 v35, s31
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -3973,7 +3972,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -4060,20 +4059,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v32, s20
-; HEURRC-NEXT: v_mov_b32_e32 v33, s21
-; HEURRC-NEXT: v_mov_b32_e32 v34, s22
-; HEURRC-NEXT: v_mov_b32_e32 v35, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v36, s24
; HEURRC-NEXT: v_mov_b32_e32 v37, s25
; HEURRC-NEXT: v_mov_b32_e32 v38, s26
; HEURRC-NEXT: v_mov_b32_e32 v39, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v32, s28
+; HEURRC-NEXT: v_mov_b32_e32 v33, s29
+; HEURRC-NEXT: v_mov_b32_e32 v34, s30
+; HEURRC-NEXT: v_mov_b32_e32 v35, s31
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -4083,7 +4081,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4124,20 +4122,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -4147,7 +4144,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31]
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4327,20 +4324,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s20
-; SDAG-NEXT: v_mov_b32_e32 v33, s21
-; SDAG-NEXT: v_mov_b32_e32 v34, s22
-; SDAG-NEXT: v_mov_b32_e32 v35, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v36, s24
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v32, s28
+; SDAG-NEXT: v_mov_b32_e32 v33, s29
+; SDAG-NEXT: v_mov_b32_e32 v34, s30
+; SDAG-NEXT: v_mov_b32_e32 v35, s31
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -4350,7 +4346,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -4437,20 +4433,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v32, s20
-; HEURRC-NEXT: v_mov_b32_e32 v33, s21
-; HEURRC-NEXT: v_mov_b32_e32 v34, s22
-; HEURRC-NEXT: v_mov_b32_e32 v35, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v36, s24
; HEURRC-NEXT: v_mov_b32_e32 v37, s25
; HEURRC-NEXT: v_mov_b32_e32 v38, s26
; HEURRC-NEXT: v_mov_b32_e32 v39, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v32, s28
+; HEURRC-NEXT: v_mov_b32_e32 v33, s29
+; HEURRC-NEXT: v_mov_b32_e32 v34, s30
+; HEURRC-NEXT: v_mov_b32_e32 v35, s31
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -4460,7 +4455,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4501,20 +4496,19 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v32, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v33, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v34, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v35, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -4524,7 +4518,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[32:35], v[16:31] cbsz:1 abid:2 blgp:3
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
@@ -4704,19 +4698,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v20, s24
; SDAG-NEXT: v_mov_b32_e32 v21, s25
; SDAG-NEXT: v_mov_b32_e32 v22, s26
; SDAG-NEXT: v_mov_b32_e32 v23, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v16, s28
+; SDAG-NEXT: v_mov_b32_e32 v17, s29
+; SDAG-NEXT: v_mov_b32_e32 v18, s30
+; SDAG-NEXT: v_mov_b32_e32 v19, s31
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4726,7 +4719,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4765,19 +4758,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v20, s24
; HEURRC-NEXT: v_mov_b32_e32 v21, s25
; HEURRC-NEXT: v_mov_b32_e32 v22, s26
; HEURRC-NEXT: v_mov_b32_e32 v23, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v16, s28
+; HEURRC-NEXT: v_mov_b32_e32 v17, s29
+; HEURRC-NEXT: v_mov_b32_e32 v18, s30
+; HEURRC-NEXT: v_mov_b32_e32 v19, s31
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4787,7 +4779,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 10
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4798,19 +4790,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4820,7 +4811,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15]
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 10
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4910,19 +4901,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s20
-; SDAG-NEXT: v_mov_b32_e32 v17, s21
-; SDAG-NEXT: v_mov_b32_e32 v18, s22
-; SDAG-NEXT: v_mov_b32_e32 v19, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v20, s24
; SDAG-NEXT: v_mov_b32_e32 v21, s25
; SDAG-NEXT: v_mov_b32_e32 v22, s26
; SDAG-NEXT: v_mov_b32_e32 v23, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v16, s28
+; SDAG-NEXT: v_mov_b32_e32 v17, s29
+; SDAG-NEXT: v_mov_b32_e32 v18, s30
+; SDAG-NEXT: v_mov_b32_e32 v19, s31
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4932,7 +4922,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -4971,19 +4961,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; HEURRC: ; %bb.0:
-; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT: v_mov_b32_e32 v16, s20
-; HEURRC-NEXT: v_mov_b32_e32 v17, s21
-; HEURRC-NEXT: v_mov_b32_e32 v18, s22
-; HEURRC-NEXT: v_mov_b32_e32 v19, s23
-; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v20, s24
; HEURRC-NEXT: v_mov_b32_e32 v21, s25
; HEURRC-NEXT: v_mov_b32_e32 v22, s26
; HEURRC-NEXT: v_mov_b32_e32 v23, s27
-; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
+; HEURRC-NEXT: v_mov_b32_e32 v16, s28
+; HEURRC-NEXT: v_mov_b32_e32 v17, s29
+; HEURRC-NEXT: v_mov_b32_e32 v18, s30
+; HEURRC-NEXT: v_mov_b32_e32 v19, s31
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -4993,7 +4982,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 10
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
@@ -5004,19 +4993,18 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; VGPRRC: ; %bb.0:
-; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
+; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
-; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
-; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
+; VGPRRC-NEXT: v_mov_b32_e32 v16, s28
+; VGPRRC-NEXT: v_mov_b32_e32 v17, s29
+; VGPRRC-NEXT: v_mov_b32_e32 v18, s30
+; VGPRRC-NEXT: v_mov_b32_e32 v19, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -5026,7 +5014,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32>
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[20:23], v[16:19], v[0:15] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 10
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index 0977caa3d677a..e1e9611705546 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -2515,34 +2515,34 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
+; SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x40
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v21, s12
-; SDAG-NEXT: v_mov_b32_e32 v22, s13
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v21, s20
+; SDAG-NEXT: v_mov_b32_e32 v22, s21
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
+; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[22:23]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -2574,34 +2574,34 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd(<8 x i32
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
-; AGPR-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s23
+; AGPR-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x40
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s21
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v21, v22 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: s_nop 11
-; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[14:15]
+; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[22:23]
; AGPR-SDAG-NEXT: s_endpgm
;
; AGPR-GISEL-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd:
@@ -2642,26 +2642,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -2700,26 +2700,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
@@ -2763,26 +2763,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -2821,26 +2821,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
@@ -2884,26 +2884,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -2942,26 +2942,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
@@ -3005,26 +3005,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: v_mov_b32_e32 v4, s12
-; SDAG-NEXT: v_mov_b32_e32 v5, s13
-; SDAG-NEXT: v_mov_b32_e32 v6, s14
-; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: v_mov_b32_e32 v8, s16
-; SDAG-NEXT: v_mov_b32_e32 v9, s17
-; SDAG-NEXT: v_mov_b32_e32 v10, s18
-; SDAG-NEXT: v_mov_b32_e32 v11, s19
-; SDAG-NEXT: v_mov_b32_e32 v12, s20
-; SDAG-NEXT: v_mov_b32_e32 v13, s21
-; SDAG-NEXT: v_mov_b32_e32 v14, s22
-; SDAG-NEXT: v_mov_b32_e32 v15, s23
+; SDAG-NEXT: v_mov_b32_e32 v8, s8
+; SDAG-NEXT: v_mov_b32_e32 v9, s9
+; SDAG-NEXT: v_mov_b32_e32 v10, s10
+; SDAG-NEXT: v_mov_b32_e32 v11, s11
+; SDAG-NEXT: v_mov_b32_e32 v12, s12
+; SDAG-NEXT: v_mov_b32_e32 v13, s13
+; SDAG-NEXT: v_mov_b32_e32 v14, s14
+; SDAG-NEXT: v_mov_b32_e32 v15, s15
+; SDAG-NEXT: v_mov_b32_e32 v0, s16
+; SDAG-NEXT: v_mov_b32_e32 v1, s17
+; SDAG-NEXT: v_mov_b32_e32 v2, s18
+; SDAG-NEXT: v_mov_b32_e32 v3, s19
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
; SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
@@ -3063,26 +3063,26 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, 0
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[8:15], v[0:7], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; AGPR-SDAG-NEXT: s_nop 11
; AGPR-SDAG-NEXT: global_store_dwordx4 v20, v[0:3], s[6:7]
; AGPR-SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 202f9fc81f290..cadaf73410320 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -7307,22 +7307,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v20, s12
-; SDAG-NEXT: v_mov_b32_e32 v21, s13
-; SDAG-NEXT: v_mov_b32_e32 v22, s14
-; SDAG-NEXT: v_mov_b32_e32 v23, s15
-; SDAG-NEXT: v_mov_b32_e32 v24, s16
-; SDAG-NEXT: v_mov_b32_e32 v25, s17
-; SDAG-NEXT: v_mov_b32_e32 v26, s18
-; SDAG-NEXT: v_mov_b32_e32 v27, s19
-; SDAG-NEXT: v_mov_b32_e32 v28, s20
-; SDAG-NEXT: v_mov_b32_e32 v29, s21
-; SDAG-NEXT: v_mov_b32_e32 v30, s22
-; SDAG-NEXT: v_mov_b32_e32 v31, s23
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v28, s12
+; SDAG-NEXT: v_mov_b32_e32 v29, s13
+; SDAG-NEXT: v_mov_b32_e32 v30, s14
+; SDAG-NEXT: v_mov_b32_e32 v31, s15
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v20, s20
+; SDAG-NEXT: v_mov_b32_e32 v21, s21
+; SDAG-NEXT: v_mov_b32_e32 v22, s22
+; SDAG-NEXT: v_mov_b32_e32 v23, s23
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -7333,7 +7333,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; SDAG-NEXT: v_mov_b32_e32 v32, s0
; SDAG-NEXT: v_mov_b32_e32 v33, s1
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -7385,22 +7385,22 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; AGPR-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x80
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s23
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s15
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
@@ -7411,7 +7411,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd(<8 x i32>
; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s0
; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s1
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v32, v33 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
; AGPR-SDAG-NEXT: s_nop 15
; AGPR-SDAG-NEXT: s_nop 2
@@ -7464,37 +7464,38 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v32, -2
; SDAG-NEXT: v_mov_b32_e32 v33, 0x41
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s8
-; SDAG-NEXT: v_mov_b32_e32 v17, s9
-; SDAG-NEXT: v_mov_b32_e32 v18, s10
-; SDAG-NEXT: v_mov_b32_e32 v19, s11
-; SDAG-NEXT: v_mov_b32_e32 v20, s12
-; SDAG-NEXT: v_mov_b32_e32 v21, s13
-; SDAG-NEXT: v_mov_b32_e32 v22, s14
-; SDAG-NEXT: v_mov_b32_e32 v23, s15
-; SDAG-NEXT: v_mov_b32_e32 v24, s16
-; SDAG-NEXT: v_mov_b32_e32 v25, s17
-; SDAG-NEXT: v_mov_b32_e32 v26, s18
-; SDAG-NEXT: v_mov_b32_e32 v27, s19
-; SDAG-NEXT: v_mov_b32_e32 v28, s20
-; SDAG-NEXT: v_mov_b32_e32 v29, s21
-; SDAG-NEXT: v_mov_b32_e32 v30, s22
-; SDAG-NEXT: v_mov_b32_e32 v31, s23
-; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
-; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
-; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
-; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
-; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; SDAG-NEXT: v_mov_b32_e32 v16, s16
+; SDAG-NEXT: v_mov_b32_e32 v17, s17
+; SDAG-NEXT: v_mov_b32_e32 v18, s18
+; SDAG-NEXT: v_mov_b32_e32 v19, s19
+; SDAG-NEXT: v_mov_b32_e32 v20, s20
+; SDAG-NEXT: v_mov_b32_e32 v21, s21
+; SDAG-NEXT: v_mov_b32_e32 v22, s22
+; SDAG-NEXT: v_mov_b32_e32 v23, s23
+; SDAG-NEXT: s_load_dwordx16 s[16:31], s[4:5], 0x40
+; SDAG-NEXT: v_mov_b32_e32 v24, s8
+; SDAG-NEXT: v_mov_b32_e32 v25, s9
+; SDAG-NEXT: v_mov_b32_e32 v26, s10
+; SDAG-NEXT: v_mov_b32_e32 v27, s11
+; SDAG-NEXT: v_mov_b32_e32 v28, s12
+; SDAG-NEXT: v_mov_b32_e32 v29, s13
+; SDAG-NEXT: v_mov_b32_e32 v30, s14
+; SDAG-NEXT: v_mov_b32_e32 v31, s15
+; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 15
; SDAG-NEXT: s_nop 2
@@ -7542,37 +7543,38 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
; AGPR-SDAG: ; %bb.0:
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x40
; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, -2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, 0x41
; AGPR-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s8
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s9
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s10
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s11
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s23
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
-; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s16
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s17
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s18
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s19
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s21
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s22
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s23
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[16:31], s[4:5], 0x40
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s9
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s10
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s11
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s12
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s13
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s14
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s15
+; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[20:21]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[22:23]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[24:25]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[26:27]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[28:29]
+; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[30:31]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v33, v32 op_sel:[1,1,0] op_sel_hi:[1,0,0] blgp:2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, 0
; AGPR-SDAG-NEXT: s_nop 15
; AGPR-SDAG-NEXT: s_nop 2
@@ -7624,28 +7626,27 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4__vgprcd___scaleA_
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2, i32 %scale0, i32 %scale1) #1 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v26, s36
+; SDAG-NEXT: v_mov_b32_e32 v27, s37
+; SDAG-NEXT: v_mov_b32_e32 v28, s38
+; SDAG-NEXT: v_mov_b32_e32 v29, s39
+; SDAG-NEXT: v_mov_b32_e32 v30, s40
+; SDAG-NEXT: v_mov_b32_e32 v31, s41
+; SDAG-NEXT: v_mov_b32_e32 v32, s42
+; SDAG-NEXT: v_mov_b32_e32 v33, s43
+; SDAG-NEXT: v_mov_b32_e32 v18, s44
+; SDAG-NEXT: v_mov_b32_e32 v19, s45
+; SDAG-NEXT: v_mov_b32_e32 v20, s46
+; SDAG-NEXT: v_mov_b32_e32 v21, s47
+; SDAG-NEXT: v_mov_b32_e32 v22, s48
+; SDAG-NEXT: v_mov_b32_e32 v23, s49
+; SDAG-NEXT: v_mov_b32_e32 v24, s50
+; SDAG-NEXT: v_mov_b32_e32 v25, s51
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
@@ -7656,7 +7657,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
; SDAG-NEXT: v_mov_b32_e32 v16, s0
; SDAG-NEXT: v_mov_b32_e32 v17, s1
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v16, v17 op_sel_hi:[0,0,0]
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[26:33], v[18:25], v[0:15], v16, v17 op_sel_hi:[0,0,0]
; SDAG-NEXT: v_mov_b32_e32 v18, s20
; SDAG-NEXT: v_mov_b32_e32 v19, s21
; SDAG-NEXT: v_mov_b32_e32 v20, s22
@@ -7754,28 +7755,27 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
;
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac:
; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s23
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; AGPR-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x80
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s27
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, s8
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s36
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s37
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s38
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s39
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s40
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s42
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s43
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s44
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s45
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s46
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s47
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s48
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s49
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s50
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s51
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a3, s11
@@ -7794,7 +7794,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, s0
; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, s1
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[2:9], v[10:17], a[0:15], v0, v1 op_sel_hi:[0,0,0]
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[10:17], v[2:9], a[0:15], v0, v1 op_sel_hi:[0,0,0]
; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s20
; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s21
; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s22
@@ -7922,28 +7922,27 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #1 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v16, 42
; SDAG-NEXT: v_mov_b32_e32 v17, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v18, s12
-; SDAG-NEXT: v_mov_b32_e32 v19, s13
-; SDAG-NEXT: v_mov_b32_e32 v20, s14
-; SDAG-NEXT: v_mov_b32_e32 v21, s15
-; SDAG-NEXT: v_mov_b32_e32 v22, s16
-; SDAG-NEXT: v_mov_b32_e32 v23, s17
-; SDAG-NEXT: v_mov_b32_e32 v24, s18
-; SDAG-NEXT: v_mov_b32_e32 v25, s19
-; SDAG-NEXT: v_mov_b32_e32 v26, s20
-; SDAG-NEXT: v_mov_b32_e32 v27, s21
-; SDAG-NEXT: v_mov_b32_e32 v28, s22
-; SDAG-NEXT: v_mov_b32_e32 v29, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v30, s24
-; SDAG-NEXT: v_mov_b32_e32 v31, s25
-; SDAG-NEXT: v_mov_b32_e32 v32, s26
-; SDAG-NEXT: v_mov_b32_e32 v33, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v26, s36
+; SDAG-NEXT: v_mov_b32_e32 v27, s37
+; SDAG-NEXT: v_mov_b32_e32 v28, s38
+; SDAG-NEXT: v_mov_b32_e32 v29, s39
+; SDAG-NEXT: v_mov_b32_e32 v30, s40
+; SDAG-NEXT: v_mov_b32_e32 v31, s41
+; SDAG-NEXT: v_mov_b32_e32 v32, s42
+; SDAG-NEXT: v_mov_b32_e32 v33, s43
+; SDAG-NEXT: v_mov_b32_e32 v18, s44
+; SDAG-NEXT: v_mov_b32_e32 v19, s45
+; SDAG-NEXT: v_mov_b32_e32 v20, s46
+; SDAG-NEXT: v_mov_b32_e32 v21, s47
+; SDAG-NEXT: v_mov_b32_e32 v22, s48
+; SDAG-NEXT: v_mov_b32_e32 v23, s49
+; SDAG-NEXT: v_mov_b32_e32 v24, s50
+; SDAG-NEXT: v_mov_b32_e32 v25, s51
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -7953,7 +7952,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[18:25], v[26:33], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[26:33], v[18:25], v[0:15], v17, v16 op_sel_hi:[0,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v18, s20
; SDAG-NEXT: v_mov_b32_e32 v19, s21
; SDAG-NEXT: v_mov_b32_e32 v20, s22
@@ -8049,28 +8048,27 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
;
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac:
; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; AGPR-SDAG-NEXT: v_mov_b32_e32 v0, 42
; AGPR-SDAG-NEXT: v_mov_b32_e32 v1, 25
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s23
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s27
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v10, s36
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v11, s37
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v12, s38
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v13, s39
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v14, s40
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v15, s41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s42
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s43
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s44
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s45
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s46
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v5, s47
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v6, s48
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v7, s49
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v8, s50
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v9, s51
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a2, s10
@@ -8088,7 +8086,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-SDAG-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[2:9], v[10:17], a[0:15], v1, v0 op_sel_hi:[0,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[10:17], v[2:9], a[0:15], v1, v0 op_sel_hi:[0,0,0] blgp:2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v2, s20
; AGPR-SDAG-NEXT: v_mov_b32_e32 v3, s21
; AGPR-SDAG-NEXT: v_mov_b32_e32 v4, s22
@@ -8214,26 +8212,25 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__nonmac(<8
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v32, s12
-; SDAG-NEXT: v_mov_b32_e32 v33, s13
-; SDAG-NEXT: v_mov_b32_e32 v34, s14
-; SDAG-NEXT: v_mov_b32_e32 v35, s15
-; SDAG-NEXT: v_mov_b32_e32 v36, s16
-; SDAG-NEXT: v_mov_b32_e32 v37, s17
-; SDAG-NEXT: v_mov_b32_e32 v38, s18
-; SDAG-NEXT: v_mov_b32_e32 v39, s19
-; SDAG-NEXT: v_mov_b32_e32 v40, s20
-; SDAG-NEXT: v_mov_b32_e32 v41, s21
-; SDAG-NEXT: v_mov_b32_e32 v42, s22
-; SDAG-NEXT: v_mov_b32_e32 v43, s23
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v44, s24
-; SDAG-NEXT: v_mov_b32_e32 v45, s25
-; SDAG-NEXT: v_mov_b32_e32 v46, s26
-; SDAG-NEXT: v_mov_b32_e32 v47, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v40, s36
+; SDAG-NEXT: v_mov_b32_e32 v41, s37
+; SDAG-NEXT: v_mov_b32_e32 v42, s38
+; SDAG-NEXT: v_mov_b32_e32 v43, s39
+; SDAG-NEXT: v_mov_b32_e32 v44, s40
+; SDAG-NEXT: v_mov_b32_e32 v45, s41
+; SDAG-NEXT: v_mov_b32_e32 v46, s42
+; SDAG-NEXT: v_mov_b32_e32 v47, s43
+; SDAG-NEXT: v_mov_b32_e32 v32, s44
+; SDAG-NEXT: v_mov_b32_e32 v33, s45
+; SDAG-NEXT: v_mov_b32_e32 v34, s46
+; SDAG-NEXT: v_mov_b32_e32 v35, s47
+; SDAG-NEXT: v_mov_b32_e32 v36, s48
+; SDAG-NEXT: v_mov_b32_e32 v37, s49
+; SDAG-NEXT: v_mov_b32_e32 v38, s50
+; SDAG-NEXT: v_mov_b32_e32 v39, s51
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -8243,7 +8240,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
+; SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[40:47], v[32:39], v[16:31] blgp:2
; SDAG-NEXT: s_nop 14
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -8332,26 +8329,25 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
;
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonmac:
; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v34, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v35, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v36, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v37, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v38, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v39, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v40, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v41, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v42, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v43, s23
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v44, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v45, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v46, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v47, s27
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v40, s36
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v41, s37
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v42, s38
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v43, s39
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v44, s40
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v45, s41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v46, s42
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v47, s43
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, s44
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, s45
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v34, s46
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v35, s47
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v36, s48
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v37, s49
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v38, s50
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v39, s51
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
@@ -8361,7 +8357,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[32:39], v[40:47], v[16:31] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_f32_32x32x64_f8f6f4 v[0:15], v[40:47], v[32:39], v[16:31] blgp:2
; AGPR-SDAG-NEXT: s_nop 14
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
@@ -8456,28 +8452,27 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__vgprcd_nonma
define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac(<8 x i32> %arg0, <8 x i32> %arg1, <16 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v32, 42
; SDAG-NEXT: v_mov_b32_e32 v33, 25
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v16, s12
-; SDAG-NEXT: v_mov_b32_e32 v17, s13
-; SDAG-NEXT: v_mov_b32_e32 v18, s14
-; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: v_mov_b32_e32 v20, s16
-; SDAG-NEXT: v_mov_b32_e32 v21, s17
-; SDAG-NEXT: v_mov_b32_e32 v22, s18
-; SDAG-NEXT: v_mov_b32_e32 v23, s19
-; SDAG-NEXT: v_mov_b32_e32 v24, s20
-; SDAG-NEXT: v_mov_b32_e32 v25, s21
-; SDAG-NEXT: v_mov_b32_e32 v26, s22
-; SDAG-NEXT: v_mov_b32_e32 v27, s23
-; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; SDAG-NEXT: v_mov_b32_e32 v28, s24
-; SDAG-NEXT: v_mov_b32_e32 v29, s25
-; SDAG-NEXT: v_mov_b32_e32 v30, s26
-; SDAG-NEXT: v_mov_b32_e32 v31, s27
-; SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v24, s36
+; SDAG-NEXT: v_mov_b32_e32 v25, s37
+; SDAG-NEXT: v_mov_b32_e32 v26, s38
+; SDAG-NEXT: v_mov_b32_e32 v27, s39
+; SDAG-NEXT: v_mov_b32_e32 v28, s40
+; SDAG-NEXT: v_mov_b32_e32 v29, s41
+; SDAG-NEXT: v_mov_b32_e32 v30, s42
+; SDAG-NEXT: v_mov_b32_e32 v31, s43
+; SDAG-NEXT: v_mov_b32_e32 v16, s44
+; SDAG-NEXT: v_mov_b32_e32 v17, s45
+; SDAG-NEXT: v_mov_b32_e32 v18, s46
+; SDAG-NEXT: v_mov_b32_e32 v19, s47
+; SDAG-NEXT: v_mov_b32_e32 v20, s48
+; SDAG-NEXT: v_mov_b32_e32 v21, s49
+; SDAG-NEXT: v_mov_b32_e32 v22, s50
+; SDAG-NEXT: v_mov_b32_e32 v23, s51
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -8487,7 +8482,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
+; SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
@@ -8584,28 +8579,27 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
;
; AGPR-SDAG-LABEL: test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_nonmac:
; AGPR-SDAG: ; %bb.0:
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[12:27], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[36:51], s[4:5], 0x0
+; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
; AGPR-SDAG-NEXT: v_mov_b32_e32 v32, 42
; AGPR-SDAG-NEXT: v_mov_b32_e32 v33, 25
; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s12
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s13
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s14
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s15
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s16
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s17
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s18
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s19
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s20
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s21
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s22
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s23
-; AGPR-SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x40
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s24
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s25
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s26
-; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s27
-; AGPR-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v24, s36
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v25, s37
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v26, s38
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v27, s39
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v28, s40
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v29, s41
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v30, s42
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v31, s43
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s44
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s45
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s46
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v19, s47
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v20, s48
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v21, s49
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v22, s50
+; AGPR-SDAG-NEXT: v_mov_b32_e32 v23, s51
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
@@ -8615,7 +8609,7 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_25_42__vgprcd_non
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; AGPR-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; AGPR-SDAG-NEXT: s_nop 1
-; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[16:23], v[24:31], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
+; AGPR-SDAG-NEXT: v_mfma_scale_f32_32x32x64_f8f6f4 v[0:15], v[24:31], v[16:23], v[0:15], v33, v32 op_sel_hi:[0,0,0] blgp:2
; AGPR-SDAG-NEXT: v_mov_b32_e32 v16, s20
; AGPR-SDAG-NEXT: v_mov_b32_e32 v17, s21
; AGPR-SDAG-NEXT: v_mov_b32_e32 v18, s22
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
index eb3386b64e72b..75e5b4a001b8f 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
@@ -30,26 +30,25 @@ define amdgpu_vs void @test(ptr addrspace(8) inreg %arg1, ptr addrspace(3) %arg2
define amdgpu_vs void @test_2(ptr addrspace(8) inreg %arg1, i32 %arg2, i32 inreg %arg3, ptr addrspace(3) %arg4) {
; CHECK-LABEL: test_2:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, 20, v1
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, 16, v1
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, 28, v1
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, 24, v1
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, 12, v1
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, 8, v1
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, 12, v1
+; CHECK-NEXT: v_add_i32_e32 v3, vcc, 8, v1
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, 4, v1
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, 28, v1
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, 24, v1
; CHECK-NEXT: s_mov_b32 m0, -1
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, 4, v1
-; CHECK-NEXT: ds_read_b32 v2, v2
-; CHECK-NEXT: ds_read_b32 v5, v4
-; CHECK-NEXT: ds_read_b32 v4, v6
-; CHECK-NEXT: ds_read_b32 v9, v7
-; CHECK-NEXT: ds_read_b32 v8, v8
-; CHECK-NEXT: ds_read_b32 v7, v10
-; CHECK-NEXT: ds_read_b32 v6, v1
+; CHECK-NEXT: v_add_i32_e32 v9, vcc, 20, v1
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, 16, v1
+; CHECK-NEXT: ds_read_b32 v4, v2
; CHECK-NEXT: ds_read_b32 v3, v3
-; CHECK-NEXT: s_waitcnt lgkmcnt(1)
-; CHECK-NEXT: tbuffer_store_format_xyzw v[6:9], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen glc slc
+; CHECK-NEXT: ds_read_b32 v2, v5
+; CHECK-NEXT: ds_read_b32 v8, v6
+; CHECK-NEXT: ds_read_b32 v7, v7
+; CHECK-NEXT: ds_read_b32 v6, v9
+; CHECK-NEXT: ds_read_b32 v5, v10
+; CHECK-NEXT: ds_read_b32 v1, v1
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: tbuffer_store_format_xyzw v[2:5], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:16 glc slc
+; CHECK-NEXT: tbuffer_store_format_xyzw v[1:4], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen glc slc
+; CHECK-NEXT: tbuffer_store_format_xyzw v[5:8], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:16 glc slc
; CHECK-NEXT: s_endpgm
%load = load <8 x float>, ptr addrspace(3) %arg4, align 4
%vec1 = shufflevector <8 x float> %load, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index 95fce2c9fa14a..daa631c68ab93 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -61,12 +61,12 @@ define amdgpu_kernel void @fadd_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], v[4:7]
+; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[4:7], v[0:3]
; GFX1251-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1251-SDAG-NEXT: s_endpgm
@@ -925,12 +925,12 @@ define amdgpu_kernel void @fmul_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-SDAG-NEXT: v_pk_mul_f64 v[0:3], v[0:3], v[4:7]
+; GFX1251-SDAG-NEXT: v_pk_mul_f64 v[0:3], v[4:7], v[0:3]
; GFX1251-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX1251-SDAG-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1251-SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/packed-u64.ll b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
index 7ee1a55a6be03..361749e144d6d 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
@@ -62,12 +62,12 @@ define amdgpu_kernel void @add_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_mov_b32_e32 v8, 0
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-SDAG-NEXT: v_pk_add_nc_u64 v[0:3], v[0:3], v[4:7]
+; GFX1251-SDAG-NEXT: v_pk_add_nc_u64 v[0:3], v[4:7], v[0:3]
; GFX1251-SDAG-NEXT: global_store_b128 v8, v[0:3], s[0:1]
; GFX1251-SDAG-NEXT: s_endpgm
;
@@ -672,12 +672,12 @@ define amdgpu_kernel void @sub_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_mov_b32_e32 v8, 0
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
-; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX1251-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
; GFX1251-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1251-SDAG-NEXT: v_pk_sub_nc_u64 v[0:3], v[0:3], v[4:7]
+; GFX1251-SDAG-NEXT: v_pk_sub_nc_u64 v[0:3], v[4:7], v[0:3]
; GFX1251-SDAG-NEXT: global_store_b128 v8, v[0:3], s[0:1]
; GFX1251-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
index f92f52d3020f0..01bf42d9f027b 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
@@ -120,22 +120,39 @@ define amdgpu_kernel void @s_shl_v2i64_imm_s(ptr addrspace(1) %out, <2 x i64> %r
}
define amdgpu_kernel void @v_shl_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; GCN-LABEL: v_shl_v2i64:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_wb
-; GCN-NEXT: v_nop
-; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GCN-NEXT: v_and_b32_e32 v8, 0x3ff, v0
-; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_clause 0x1
-; GCN-NEXT: global_load_b128 v[0:3], v8, s[2:3] offset:16 scale_offset
-; GCN-NEXT: global_load_b128 v[4:7], v8, s[2:3] scale_offset
-; GCN-NEXT: s_wait_loadcnt 0x0
-; GCN-NEXT: v_lshlrev_b64_e32 v[2:3], v2, v[6:7]
-; GCN-NEXT: v_lshlrev_b64_e32 v[0:1], v0, v[4:5]
-; GCN-NEXT: global_store_b128 v8, v[0:3], s[0:1] scale_offset
-; GCN-NEXT: s_endpgm
+; GFX1250-LABEL: v_shl_v2i64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_and_b32_e32 v8, 0x3ff, v0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_load_b128 v[0:3], v8, s[2:3] offset:16 scale_offset
+; GFX1250-NEXT: global_load_b128 v[4:7], v8, s[2:3] scale_offset
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v2, v[6:7]
+; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v0, v[4:5]
+; GFX1250-NEXT: global_store_b128 v8, v[0:3], s[0:1] scale_offset
+; GFX1250-NEXT: s_endpgm
+;
+; GFX1251-LABEL: v_shl_v2i64:
+; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
+; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1251-NEXT: v_and_b32_e32 v8, 0x3ff, v0
+; GFX1251-NEXT: s_wait_kmcnt 0x0
+; GFX1251-NEXT: s_clause 0x1
+; GFX1251-NEXT: global_load_b128 v[0:3], v8, s[2:3] scale_offset
+; GFX1251-NEXT: global_load_b128 v[4:7], v8, s[2:3] offset:16 scale_offset
+; GFX1251-NEXT: s_wait_loadcnt 0x0
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[2:3], v6, v[2:3]
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[0:1], v4, v[0:1]
+; GFX1251-NEXT: global_store_b128 v8, v[0:3], s[0:1] scale_offset
+; GFX1251-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <2 x i64>, ptr addrspace(1) %in, i64 %tid.ext
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vhaddw.ll b/llvm/test/CodeGen/LoongArch/lasx/vhaddw.ll
index 2304a5d6f83fd..66984087298f5 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/vhaddw.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/vhaddw.ll
@@ -5,10 +5,43 @@
define void @xvhaddw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-LABEL: xvhaddw_h_b_illegal:
; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: xvld $xr2, $a0, 32
; CHECK-NEXT: xvld $xr4, $a0, 0
-; CHECK-NEXT: xvld $xr3, $a0, 32
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvld $xr0, $a1, 32
+; CHECK-NEXT: xvld $xr3, $a1, 32
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 1
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 0
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 3
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 5
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 2
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 7
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 3
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 9
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 4
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 11
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 5
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 13
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 6
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 15
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 7
+; CHECK-NEXT: xvpermi.d $xr2, $xr2, 14
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 1
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 8
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 3
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 9
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 5
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 10
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 7
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 11
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 9
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 12
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 11
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 13
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 13
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 14
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 15
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 15
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 1
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 0
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 3
@@ -24,8 +57,8 @@ define void @xvhaddw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 13
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 6
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 15
-; CHECK-NEXT: xvpermi.d $xr4, $xr4, 14
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 7
+; CHECK-NEXT: xvpermi.d $xr4, $xr4, 14
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 1
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 8
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 3
@@ -42,38 +75,38 @@ define void @xvhaddw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 14
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 15
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 15
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 1
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 0
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 0
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 3
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 2
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 1
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 5
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 4
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 2
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 7
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 6
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 3
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 9
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 8
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 4
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 11
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 10
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 5
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 13
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 12
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 6
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 15
-; CHECK-NEXT: xvpermi.d $xr3, $xr3, 14
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 14
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 7
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 1
+; CHECK-NEXT: xvpermi.d $xr3, $xr3, 14
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 0
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 8
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 3
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 2
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 9
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 5
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 4
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 10
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 7
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 6
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 11
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 9
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 8
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 12
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 11
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 10
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 13
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 13
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 12
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 14
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 15
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 14
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 15
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 0
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 0
@@ -90,8 +123,8 @@ define void @xvhaddw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 12
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 6
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 14
-; CHECK-NEXT: xvpermi.d $xr1, $xr1, 14
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 7
+; CHECK-NEXT: xvpermi.d $xr1, $xr1, 14
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 0
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 8
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 2
@@ -108,47 +141,14 @@ define void @xvhaddw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 14
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 14
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 15
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 0
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 0
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 2
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 1
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 4
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 2
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 6
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 3
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 8
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 4
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 10
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 5
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 12
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 6
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 14
-; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 7
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 0
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 8
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 2
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 9
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 4
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 10
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 6
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 11
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 8
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 12
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 10
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 13
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 12
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 14
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 14
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 15
-; CHECK-NEXT: vext2xv.h.b $xr0, $xr2
-; CHECK-NEXT: vext2xv.h.b $xr2, $xr4
-; CHECK-NEXT: vext2xv.h.b $xr3, $xr3
-; CHECK-NEXT: vext2xv.h.b $xr1, $xr1
-; CHECK-NEXT: xvadd.h $xr1, $xr2, $xr1
+; CHECK-NEXT: vext2xv.h.b $xr1, $xr2
+; CHECK-NEXT: vext2xv.h.b $xr0, $xr0
+; CHECK-NEXT: vext2xv.h.b $xr2, $xr3
+; CHECK-NEXT: vext2xv.h.b $xr3, $xr4
; CHECK-NEXT: xvadd.h $xr0, $xr0, $xr3
-; CHECK-NEXT: xvst $xr0, $a2, 0
-; CHECK-NEXT: xvst $xr1, $a2, 32
+; CHECK-NEXT: xvadd.h $xr1, $xr1, $xr2
+; CHECK-NEXT: xvst $xr1, $a2, 0
+; CHECK-NEXT: xvst $xr0, $a2, 32
; CHECK-NEXT: ret
entry:
%0 = load <64 x i8>, ptr %a
@@ -175,7 +175,13 @@ define void @xvhaddw_h_b(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhaddw.h.b $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.b $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.b $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.h.b $xr0, $xr0
+; CHECK-NEXT: vext2xv.h.b $xr1, $xr1
+; CHECK-NEXT: xvadd.h $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -199,7 +205,13 @@ define void @xvhaddw_w_h(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhaddw.w.h $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.h $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.h $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.w.h $xr0, $xr0
+; CHECK-NEXT: vext2xv.w.h $xr1, $xr1
+; CHECK-NEXT: xvadd.w $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -221,7 +233,13 @@ define void @xvhaddw_d_w(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhaddw.d.w $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.w $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.w $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.d.w $xr0, $xr0
+; CHECK-NEXT: vext2xv.d.w $xr1, $xr1
+; CHECK-NEXT: xvadd.d $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -241,7 +259,13 @@ define void @xvhaddw_hu_bu(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhaddw.hu.bu $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.b $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.b $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.hu.bu $xr0, $xr0
+; CHECK-NEXT: vext2xv.hu.bu $xr1, $xr1
+; CHECK-NEXT: xvadd.h $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -265,7 +289,13 @@ define void @xvhaddw_wu_hu(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhaddw.wu.hu $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.h $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.h $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.wu.hu $xr0, $xr0
+; CHECK-NEXT: vext2xv.wu.hu $xr1, $xr1
+; CHECK-NEXT: xvadd.w $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -287,7 +317,13 @@ define void @xvhaddw_du_wu(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhaddw.du.wu $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.w $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.w $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.du.wu $xr0, $xr0
+; CHECK-NEXT: vext2xv.du.wu $xr1, $xr1
+; CHECK-NEXT: xvadd.d $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vhsubw.ll b/llvm/test/CodeGen/LoongArch/lasx/vhsubw.ll
index 20b959aa74d7a..228363937b4af 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/vhsubw.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/vhsubw.ll
@@ -5,10 +5,43 @@
define void @xvhsubw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-LABEL: xvhsubw_h_b_illegal:
; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: xvld $xr2, $a0, 32
; CHECK-NEXT: xvld $xr4, $a0, 0
-; CHECK-NEXT: xvld $xr3, $a0, 32
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvld $xr0, $a1, 32
+; CHECK-NEXT: xvld $xr3, $a1, 32
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 1
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 0
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 3
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 5
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 2
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 7
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 3
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 9
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 4
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 11
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 5
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 13
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 6
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 15
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 7
+; CHECK-NEXT: xvpermi.d $xr2, $xr2, 14
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 1
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 8
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 3
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 9
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 5
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 10
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 7
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 11
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 9
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 12
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 11
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 13
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 13
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 14
+; CHECK-NEXT: vpickve2gr.b $a0, $vr2, 15
+; CHECK-NEXT: vinsgr2vr.b $vr0, $a0, 15
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 1
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 0
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 3
@@ -24,8 +57,8 @@ define void @xvhsubw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 13
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 6
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 15
-; CHECK-NEXT: xvpermi.d $xr4, $xr4, 14
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 7
+; CHECK-NEXT: xvpermi.d $xr4, $xr4, 14
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 1
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 8
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 3
@@ -42,38 +75,38 @@ define void @xvhsubw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 14
; CHECK-NEXT: vpickve2gr.b $a0, $vr4, 15
; CHECK-NEXT: vinsgr2vr.b $vr2, $a0, 15
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 1
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 0
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 0
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 3
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 2
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 1
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 5
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 4
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 2
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 7
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 6
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 3
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 9
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 8
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 4
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 11
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 10
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 5
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 13
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 12
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 6
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 15
-; CHECK-NEXT: xvpermi.d $xr3, $xr3, 14
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 14
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 7
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 1
+; CHECK-NEXT: xvpermi.d $xr3, $xr3, 14
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 0
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 8
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 3
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 2
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 9
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 5
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 4
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 10
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 7
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 6
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 11
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 9
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 8
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 12
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 11
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 10
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 13
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 13
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 12
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 14
-; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 15
+; CHECK-NEXT: vpickve2gr.b $a0, $vr3, 14
; CHECK-NEXT: vinsgr2vr.b $vr4, $a0, 15
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 0
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 0
@@ -90,8 +123,8 @@ define void @xvhsubw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 12
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 6
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 14
-; CHECK-NEXT: xvpermi.d $xr1, $xr1, 14
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 7
+; CHECK-NEXT: xvpermi.d $xr1, $xr1, 14
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 0
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 8
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 2
@@ -108,47 +141,14 @@ define void @xvhsubw_h_b_illegal(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 14
; CHECK-NEXT: vpickve2gr.b $a0, $vr1, 14
; CHECK-NEXT: vinsgr2vr.b $vr3, $a0, 15
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 0
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 0
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 2
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 1
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 4
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 2
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 6
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 3
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 8
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 4
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 10
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 5
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 12
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 6
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 14
-; CHECK-NEXT: xvpermi.d $xr0, $xr0, 14
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 7
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 0
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 8
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 2
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 9
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 4
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 10
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 6
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 11
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 8
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 12
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 10
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 13
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 12
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 14
-; CHECK-NEXT: vpickve2gr.b $a0, $vr0, 14
-; CHECK-NEXT: vinsgr2vr.b $vr1, $a0, 15
-; CHECK-NEXT: vext2xv.h.b $xr0, $xr2
-; CHECK-NEXT: vext2xv.h.b $xr2, $xr4
-; CHECK-NEXT: vext2xv.h.b $xr3, $xr3
-; CHECK-NEXT: vext2xv.h.b $xr1, $xr1
-; CHECK-NEXT: xvsub.h $xr1, $xr2, $xr1
+; CHECK-NEXT: vext2xv.h.b $xr1, $xr2
+; CHECK-NEXT: vext2xv.h.b $xr0, $xr0
+; CHECK-NEXT: vext2xv.h.b $xr2, $xr3
+; CHECK-NEXT: vext2xv.h.b $xr3, $xr4
; CHECK-NEXT: xvsub.h $xr0, $xr0, $xr3
-; CHECK-NEXT: xvst $xr0, $a2, 0
-; CHECK-NEXT: xvst $xr1, $a2, 32
+; CHECK-NEXT: xvsub.h $xr1, $xr1, $xr2
+; CHECK-NEXT: xvst $xr1, $a2, 0
+; CHECK-NEXT: xvst $xr0, $a2, 32
; CHECK-NEXT: ret
entry:
%0 = load <64 x i8>, ptr %a
@@ -175,7 +175,13 @@ define void @xvhsubw_h_b(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhsubw.h.b $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.b $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.b $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.h.b $xr0, $xr0
+; CHECK-NEXT: vext2xv.h.b $xr1, $xr1
+; CHECK-NEXT: xvsub.h $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -199,7 +205,13 @@ define void @xvhsubw_w_h(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhsubw.w.h $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.h $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.h $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.w.h $xr0, $xr0
+; CHECK-NEXT: vext2xv.w.h $xr1, $xr1
+; CHECK-NEXT: xvsub.w $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -221,7 +233,13 @@ define void @xvhsubw_d_w(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhsubw.d.w $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.w $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.w $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.d.w $xr0, $xr0
+; CHECK-NEXT: vext2xv.d.w $xr1, $xr1
+; CHECK-NEXT: xvsub.d $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -241,7 +259,13 @@ define void @xvhsubw_hu_bu(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhsubw.hu.bu $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.b $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.b $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.hu.bu $xr0, $xr0
+; CHECK-NEXT: vext2xv.hu.bu $xr1, $xr1
+; CHECK-NEXT: xvsub.h $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -265,7 +289,13 @@ define void @xvhsubw_wu_hu(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhsubw.wu.hu $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.h $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.h $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.wu.hu $xr0, $xr0
+; CHECK-NEXT: vext2xv.wu.hu $xr1, $xr1
+; CHECK-NEXT: xvsub.w $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
@@ -287,7 +317,13 @@ define void @xvhsubw_du_wu(ptr %a, ptr %b, ptr %r) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
; CHECK-NEXT: xvld $xr1, $a1, 0
-; CHECK-NEXT: xvhsubw.du.wu $xr0, $xr0, $xr1
+; CHECK-NEXT: xvpermi.q $xr2, $xr0, 1
+; CHECK-NEXT: vpickod.w $vr0, $vr2, $vr0
+; CHECK-NEXT: xvpermi.q $xr2, $xr1, 1
+; CHECK-NEXT: vpickev.w $vr1, $vr2, $vr1
+; CHECK-NEXT: vext2xv.du.wu $xr0, $xr0
+; CHECK-NEXT: vext2xv.du.wu $xr1, $xr1
+; CHECK-NEXT: xvsub.d $xr0, $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a2, 0
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vsllwil.ll b/llvm/test/CodeGen/LoongArch/lasx/vsllwil.ll
index 9706403908522..2bd64157f8394 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/vsllwil.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/vsllwil.ll
@@ -6,7 +6,10 @@ define void @vsllwil_h_b(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: vsllwil_h_b:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvsllwil.h.b $xr0, $xr0, 1
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 68
+; CHECK-NEXT: vext2xv.h.b $xr0, $xr1
+; CHECK-NEXT: xvslli.h $xr0, $xr0, 1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -24,7 +27,10 @@ define void @vsllwil_w_h(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: vsllwil_w_h:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvsllwil.w.h $xr0, $xr0, 15
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 68
+; CHECK-NEXT: vext2xv.w.h $xr0, $xr1
+; CHECK-NEXT: xvslli.w $xr0, $xr0, 15
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -41,7 +47,10 @@ define void @vsllwil_d_w(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: vsllwil_d_w:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvsllwil.d.w $xr0, $xr0, 31
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 68
+; CHECK-NEXT: vext2xv.d.w $xr0, $xr1
+; CHECK-NEXT: xvslli.d $xr0, $xr0, 31
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -58,7 +67,10 @@ define void @vsllwil_hu_bu(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: vsllwil_hu_bu:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvsllwil.hu.bu $xr0, $xr0, 1
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 68
+; CHECK-NEXT: vext2xv.hu.bu $xr0, $xr1
+; CHECK-NEXT: xvslli.h $xr0, $xr0, 1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -76,7 +88,10 @@ define void @vsllwil_wu_hu(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: vsllwil_wu_hu:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvsllwil.wu.hu $xr0, $xr0, 15
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 68
+; CHECK-NEXT: vext2xv.wu.hu $xr0, $xr1
+; CHECK-NEXT: xvslli.w $xr0, $xr0, 15
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -93,7 +108,10 @@ define void @vsllwil_du_wu(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: vsllwil_du_wu:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvsllwil.du.wu $xr0, $xr0, 31
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 68
+; CHECK-NEXT: vext2xv.du.wu $xr0, $xr1
+; CHECK-NEXT: xvslli.d $xr0, $xr0, 31
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/LoongArch/lasx/xvexth.ll b/llvm/test/CodeGen/LoongArch/lasx/xvexth.ll
index db3a4c9c66368..3911915336989 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/xvexth.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/xvexth.ll
@@ -6,7 +6,9 @@ define void @xvexth_h_b(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_h_b:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.h.b $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.h.b $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -23,7 +25,9 @@ define void @xvexth_h_b_poison(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_h_b_poison:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.h.b $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.h.b $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -40,7 +44,9 @@ define void @xvexth_w_h(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_w_h:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.w.h $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.w.h $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -56,7 +62,9 @@ define void @xvexth_w_h_poison(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_w_h_poison:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.w.h $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.w.h $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -72,7 +80,9 @@ define void @xvexth_d_w(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_d_w:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.d.w $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.d.w $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -88,7 +98,9 @@ define void @xvexth_d_w_poison(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_d_w_poison:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.d.w $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 46
+; CHECK-NEXT: vext2xv.d.w $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -104,7 +116,9 @@ define void @xvexth_hu_bu(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_hu_bu:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.hu.bu $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.hu.bu $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -121,7 +135,9 @@ define void @xvexth_hu_bu_poison(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_hu_bu_poison:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.hu.bu $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.hu.bu $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -138,7 +154,9 @@ define void @xvexth_wu_hu(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_wu_hu:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.wu.hu $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.wu.hu $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -154,7 +172,9 @@ define void @xvexth_wu_hu_poison(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_wu_hu_poison:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.wu.hu $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.wu.hu $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -170,7 +190,9 @@ define void @xvexth_du_wu(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_du_wu:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.du.wu $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 238
+; CHECK-NEXT: vext2xv.du.wu $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
@@ -186,7 +208,9 @@ define void @xvexth_du_wu_poison(ptr %a, ptr %r) nounwind {
; CHECK-LABEL: xvexth_du_wu_poison:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvexth.du.wu $xr0, $xr0
+; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT: vpermi.w $vr1, $vr0, 46
+; CHECK-NEXT: vext2xv.du.wu $xr0, $xr1
; CHECK-NEXT: xvst $xr0, $a1, 0
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll
index 16a51b2027d5e..c7318b26dda86 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle-fp16.ll
@@ -181,11 +181,11 @@ define arm_aapcs_vfpcc <8 x half> @shuffle2step_f16(<16 x half> %src) {
; CHECKFP16-NEXT: sub sp, #32
; CHECKFP16-NEXT: vmov q4, q1
; CHECKFP16-NEXT: vshr.u32 q1, q1, #16
-; CHECKFP16-NEXT: add r6, sp, #16
+; CHECKFP16-NEXT: mov r6, sp
; CHECKFP16-NEXT: vmov.u16 r0, q0[0]
; CHECKFP16-NEXT: vstrh.32 q1, [r6, #8]
; CHECKFP16-NEXT: vshr.u32 q1, q0, #16
-; CHECKFP16-NEXT: mov r5, sp
+; CHECKFP16-NEXT: add r5, sp, #16
; CHECKFP16-NEXT: vstrh.32 q1, [r6]
; CHECKFP16-NEXT: vstrh.32 q4, [r5, #8]
; CHECKFP16-NEXT: vstrh.32 q0, [r5]
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index d01734e3c1358..352900521d959 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -337,11 +337,11 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle2step_i16(<16 x i16> %src) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u32 q2, q1, #16
; CHECK-NEXT: vstrh.32 q2, [r0, #8]
; CHECK-NEXT: vshr.u32 q2, q0, #16
-; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: vstrh.32 q2, [r0]
; CHECK-NEXT: vstrh.32 q1, [r1, #8]
; CHECK-NEXT: vstrh.32 q0, [r1]
@@ -627,11 +627,11 @@ define arm_aapcs_vfpcc <16 x i8> @shuffle2step_i8(<32 x i8> %src) {
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u16 q2, q1, #8
; CHECK-NEXT: vstrb.16 q2, [r0, #8]
; CHECK-NEXT: vshr.u16 q2, q0, #8
-; CHECK-NEXT: add r1, sp, #16
+; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: vstrb.16 q2, [r0]
; CHECK-NEXT: vstrb.16 q1, [r1, #8]
; CHECK-NEXT: vstrb.16 q0, [r1]
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld2.ll b/llvm/test/CodeGen/Thumb2/mve-vld2.ll
index dca4fb3d6cfa3..f6256937e5169 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld2.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld2.ll
@@ -207,11 +207,11 @@ define void @vld2_v8i16_align1(ptr %src, ptr %dst) {
; CHECK-NEXT: .pad #32
; CHECK-NEXT: sub sp, #32
; CHECK-NEXT: vldrb.u8 q0, [r0, #16]
-; CHECK-NEXT: add r2, sp, #16
+; CHECK-NEXT: mov r2, sp
; CHECK-NEXT: vshr.u32 q1, q0, #16
; CHECK-NEXT: vstrh.32 q1, [r2, #8]
; CHECK-NEXT: vldrb.u8 q1, [r0]
-; CHECK-NEXT: mov r0, sp
+; CHECK-NEXT: add r0, sp, #16
; CHECK-NEXT: vshr.u32 q2, q1, #16
; CHECK-NEXT: vstrh.32 q2, [r2]
; CHECK-NEXT: vstrh.32 q0, [r0, #8]
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
index 4a542949c7859..58f2ca38b1815 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -909,10 +909,10 @@ define void @vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16(ptr %in
;
; AVX2-LABEL: vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX2-NEXT: vpbroadcastb (%rdi), %ymm1
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX2-NEXT: vpbroadcastb (%rdi), %ymm0
+; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u,17,19,21,23,25,27,29,31,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
; AVX2-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
; AVX2-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/combine-pmaddwd.ll b/llvm/test/CodeGen/X86/combine-pmaddwd.ll
index 4fe3569df88e9..fbe7f93b4ea11 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddwd.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddwd.ll
@@ -369,7 +369,10 @@ define <8 x i32> @sext_pairwise_add(<16 x i16> %x) {
;
; AVX512-LABEL: sext_pairwise_add:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX512-NEXT: retq
%1 = sext <16 x i16> %x to <16 x i32>
%2 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -402,7 +405,11 @@ define <8 x i32> @combine_with_mul(<16 x i16> %v) {
;
; AVX512-LABEL: combine_with_mul:
; AVX512: # %bb.0: # %bb1
-; AVX512-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [4096,1,4096,1,4096,1,4096,1,4096,1,4096,1,4096,1,4096,1]
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX512-NEXT: retq
bb1:
%0 = sext <16 x i16> %v to <16 x i32>
@@ -437,7 +444,11 @@ define <8 x i32> @combine_with_shl(<16 x i16> %v) {
;
; AVX512-LABEL: combine_with_shl:
; AVX512: # %bb.0: # %bb1
-; AVX512-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [4096,1,4096,1,4096,1,4096,1,4096,1,4096,1,4096,1,4096,1]
+; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-NEXT: vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX512-NEXT: retq
bb1:
%0 = sext <16 x i16> %v to <16 x i32>
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 4ee77c0bdf70c..5c1f2b2c1afe4 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -2051,10 +2051,27 @@ define <4 x i32> @pmaddwd_8(<8 x i16> %A, <8 x i16> %B) {
; SSE-NEXT: pmaddwd %xmm1, %xmm0
; SSE-NEXT: retq
;
-; AVX-LABEL: pmaddwd_8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT: vpmaddwd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_8:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%b = sext <8 x i16> %B to <8 x i32>
%m = mul nsw <8 x i32> %a, %b
@@ -2070,10 +2087,27 @@ define <4 x i32> @pmaddwd_8_swapped(<8 x i16> %A, <8 x i16> %B) {
; SSE-NEXT: pmaddwd %xmm1, %xmm0
; SSE-NEXT: retq
;
-; AVX-LABEL: pmaddwd_8_swapped:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_8_swapped:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT: vpmaddwd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_8_swapped:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%b = sext <8 x i16> %B to <8 x i32>
%m = mul nsw <8 x i32> %a, %b
@@ -2156,10 +2190,30 @@ define <8 x i32> @pmaddwd_16(<16 x i16> %A, <16 x i16> %B) {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddwd_16:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddwd_16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: pmaddwd_16:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512F-NEXT: retq
+;
+; AVX512BW-LABEL: pmaddwd_16:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512BW-NEXT: retq
%a = sext <16 x i16> %A to <16 x i32>
%b = sext <16 x i16> %B to <16 x i32>
%m = mul nsw <16 x i32> %a, %b
@@ -2226,10 +2280,23 @@ define <4 x i32> @pmaddwd_const(<8 x i16> %A) {
; SSE-NEXT: pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [32767,32768,0,0,1,7,42,32]
; SSE-NEXT: retq
;
-; AVX-LABEL: pmaddwd_const:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32767,32768,0,0,1,7,42,32]
-; AVX-NEXT: retq
+; AVX1-LABEL: pmaddwd_const:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [1,0,7,0,42,0,32,0]
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32767,0,32768,0,0,0,0,0]
+; AVX1-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: pmaddwd_const:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX256-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [32767,0,32768,0,0,0,0,0,1,0,7,0,42,0,32,0]
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%a = sext <8 x i16> %A to <8 x i32>
%m = mul nsw <8 x i32> %a, <i32 32767, i32 -32768, i32 0, i32 0, i32 1, i32 7, i32 42, i32 32>
%odd = shufflevector <8 x i32> %m, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -2356,10 +2423,29 @@ define <4 x i32> @jumbled_indices4(<8 x i16> %A, <8 x i16> %B) {
; SSE-NEXT: pmaddwd %xmm1, %xmm0
; SSE-NEXT: retq
;
-; AVX-LABEL: jumbled_indices4:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: jumbled_indices4:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT: vpmaddwd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vphaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,3]
+; AVX1-NEXT: retq
+;
+; AVX256-LABEL: jumbled_indices4:
+; AVX256: # %bb.0:
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX256-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,3]
+; AVX256-NEXT: vzeroupper
+; AVX256-NEXT: retq
%exta = sext <8 x i16> %A to <8 x i32>
%extb = sext <8 x i16> %B to <8 x i32>
%m = mul <8 x i32> %exta, %extb
@@ -2385,10 +2471,32 @@ define <8 x i32> @jumbled_indices8(<16 x i16> %A, <16 x i16> %B) {
; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: jumbled_indices8:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: jumbled_indices8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: jumbled_indices8:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,1,5,4,3,2,7,6]
+; AVX512F-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT: retq
+;
+; AVX512BW-LABEL: jumbled_indices8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vphaddd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,1,5,4,3,2,7,6]
+; AVX512BW-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT: retq
%exta = sext <16 x i16> %A to <16 x i32>
%extb = sext <16 x i16> %B to <16 x i32>
%m = mul <16 x i32> %exta, %extb
@@ -2562,13 +2670,21 @@ define <4 x i32> @pmaddwd_128(ptr %Aptr, ptr %Bptr) {
}
define <8 x i32> @pmaddwd_256(ptr %Aptr, ptr %Bptr) {
-; SSE-LABEL: pmaddwd_256:
-; SSE: # %bb.0:
-; SSE-NEXT: movdqa (%rdi), %xmm0
-; SSE-NEXT: movdqa 16(%rdi), %xmm1
-; SSE-NEXT: pmaddwd (%rsi), %xmm0
-; SSE-NEXT: pmaddwd 16(%rsi), %xmm1
-; SSE-NEXT: retq
+; SSE2-LABEL: pmaddwd_256:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movdqa 16(%rdi), %xmm1
+; SSE2-NEXT: movdqa (%rsi), %xmm0
+; SSE2-NEXT: pmaddwd (%rdi), %xmm0
+; SSE2-NEXT: pmaddwd 16(%rsi), %xmm1
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: pmaddwd_256:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movdqa (%rdi), %xmm0
+; SSE42-NEXT: movdqa 16(%rdi), %xmm1
+; SSE42-NEXT: pmaddwd (%rsi), %xmm0
+; SSE42-NEXT: pmaddwd 16(%rsi), %xmm1
+; SSE42-NEXT: retq
;
; AVX1-LABEL: pmaddwd_256:
; AVX1: # %bb.0:
@@ -2579,11 +2695,31 @@ define <8 x i32> @pmaddwd_256(ptr %Aptr, ptr %Bptr) {
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddwd_256:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vmovdqa (%rdi), %ymm0
-; AVX256-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddwd_256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1],mem[2],ymm0[3],mem[4],ymm0[5],mem[6],ymm0[7],mem[8],ymm0[9],mem[10],ymm0[11],mem[12],ymm0[13],mem[14],ymm0[15]
+; AVX2-NEXT: vpmaddwd (%rdi), %ymm0, %ymm0
+; AVX2-NEXT: vmovdqa (%rsi), %ymm1
+; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1
+; AVX2-NEXT: vmovdqa (%rdi), %ymm2
+; AVX2-NEXT: vpsrld $16, %ymm2, %ymm2
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: pmaddwd_256:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vpsrld $16, %ymm0, %ymm1
+; AVX512-NEXT: vmovdqa (%rsi), %ymm2
+; AVX512-NEXT: vpsrld $16, %ymm2, %ymm3
+; AVX512-NEXT: vpmaddwd %ymm3, %ymm1, %ymm1
+; AVX512-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4],ymm3[5],ymm2[6],ymm3[7],ymm2[8],ymm3[9],ymm2[10],ymm3[11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
+; AVX512-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: retq
%A = load <16 x i16>, ptr %Aptr
%B = load <16 x i16>, ptr %Bptr
%A_even = shufflevector <16 x i16> %A, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -2619,35 +2755,90 @@ define <16 x i32> @pmaddwd_512(ptr %Aptr, ptr %Bptr) {
; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2
; AVX1-NEXT: vmovdqa 48(%rdi), %xmm3
-; AVX1-NEXT: vpmaddwd 48(%rsi), %xmm3, %xmm3
-; AVX1-NEXT: vpmaddwd 32(%rsi), %xmm2, %xmm2
; AVX1-NEXT: vpmaddwd 16(%rsi), %xmm1, %xmm1
; AVX1-NEXT: vpmaddwd (%rsi), %xmm0, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm1
+; AVX1-NEXT: vpmaddwd 48(%rsi), %xmm3, %xmm1
+; AVX1-NEXT: vpmaddwd 32(%rsi), %xmm2, %xmm2
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX1-NEXT: retq
;
; AVX2-LABEL: pmaddwd_512:
; AVX2: # %bb.0:
; AVX2-NEXT: vmovdqa (%rdi), %ymm0
; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX2-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX2-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vshufps {{.*#+}} ymm2 = ymm3[0,2],ymm2[0,2],ymm3[4,6],ymm2[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vmovdqa (%rsi), %ymm1
+; AVX2-NEXT: vmovdqa 32(%rsi), %ymm4
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm4[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vpshuflw {{.*#+}} ymm6 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,2],ymm5[0,2],ymm6[4,6],ymm5[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm5 = ymm5[0,2,1,3]
+; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm4
+; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm4[2,3],ymm1[4,5],ymm4[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,1,3]
+; AVX2-NEXT: vextractf128 $1, %ymm5, %xmm1
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm4
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX2-NEXT: vpmaddwd %ymm1, %ymm4, %ymm1
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX2-NEXT: vpmaddwd %ymm4, %ymm2, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX2-NEXT: vpmaddwd %ymm4, %ymm5, %ymm4
+; AVX2-NEXT: vpaddd %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vpmaddwd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX2-NEXT: retq
;
; AVX512F-LABEL: pmaddwd_512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm1
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512F-NEXT: vpmovdw %zmm2, %ymm3
+; AVX512F-NEXT: vpsrld $16, %zmm2, %zmm2
+; AVX512F-NEXT: vpmovdw %zmm2, %ymm2
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512F-NEXT: vpmulld %zmm3, %zmm1, %zmm1
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512F-NEXT: vpmulld %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddwd_512:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT: vpsrld $16, %zmm0, %zmm1
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm2
+; AVX512BW-NEXT: vpsrld $16, %zmm2, %zmm3
+; AVX512BW-NEXT: vpmaddwd %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
+; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: retq
%A = load <32 x i16>, ptr %Aptr
%B = load <32 x i16>, ptr %Bptr
@@ -2733,24 +2924,69 @@ define <32 x i32> @pmaddwd_1024(ptr %Aptr, ptr %Bptr) {
;
; AVX512F-LABEL: pmaddwd_1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm2
-; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm3
-; AVX512F-NEXT: vpmaddwd 96(%rsi), %ymm3, %ymm3
-; AVX512F-NEXT: vpmaddwd 64(%rsi), %ymm2, %ymm2
-; AVX512F-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm1
+; AVX512F-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT: vmovdqa64 64(%rdi), %zmm1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm2
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm3
+; AVX512F-NEXT: vpsrld $16, %zmm1, %zmm1
+; AVX512F-NEXT: vpmovdw %zmm1, %ymm1
+; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512F-NEXT: vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT: vmovdqa64 (%rsi), %zmm4
+; AVX512F-NEXT: vmovdqa64 64(%rsi), %zmm5
+; AVX512F-NEXT: vpmovdw %zmm5, %ymm6
+; AVX512F-NEXT: vpmovdw %zmm4, %ymm7
+; AVX512F-NEXT: vpsrld $16, %zmm5, %zmm5
+; AVX512F-NEXT: vpmovdw %zmm5, %ymm5
+; AVX512F-NEXT: vpsrld $16, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovdw %zmm4, %ymm4
+; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm3
+; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2
+; AVX512F-NEXT: vpmovsxwd %ymm7, %zmm7
+; AVX512F-NEXT: vpmulld %zmm7, %zmm3, %zmm3
+; AVX512F-NEXT: vpmovsxwd %ymm6, %zmm6
+; AVX512F-NEXT: vpmulld %zmm6, %zmm2, %zmm2
+; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm4
+; AVX512F-NEXT: vpmulld %zmm4, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm3, %zmm0
+; AVX512F-NEXT: vpmovsxwd %ymm5, %zmm3
+; AVX512F-NEXT: vpmulld %zmm3, %zmm1, %zmm1
+; AVX512F-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddwd_1024:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
-; AVX512BW-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmaddwd 64(%rsi), %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
+; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm3
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm2, %zmm3
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm0
+; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm5
+; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm1
+; AVX512BW-NEXT: vpermi2w %zmm1, %zmm5, %zmm2
+; AVX512BW-NEXT: vpermt2w %zmm1, %zmm4, %zmm5
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm2, %ymm1
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm3, %ymm4
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
+; AVX512BW-NEXT: vpmaddwd %zmm1, %zmm4, %zmm1
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm5, %ymm3
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm4
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
+; AVX512BW-NEXT: vpmaddwd %zmm3, %zmm4, %zmm3
+; AVX512BW-NEXT: vpaddd %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BW-NEXT: vpmaddwd %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: retq
%A = load <64 x i16>, ptr %Aptr
%B = load <64 x i16>, ptr %Bptr
diff --git a/llvm/test/CodeGen/X86/pmaddubsw.ll b/llvm/test/CodeGen/X86/pmaddubsw.ll
index c4f71e3aaa44f..f2771409279cf 100644
--- a/llvm/test/CodeGen/X86/pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/pmaddubsw.ll
@@ -43,26 +43,196 @@ define <8 x i16> @pmaddubsw_128(ptr %Aptr, ptr %Bptr) {
define <16 x i16> @pmaddubsw_256(ptr %Aptr, ptr %Bptr) {
; SSE-LABEL: pmaddubsw_256:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa (%rsi), %xmm0
-; SSE-NEXT: movdqa 16(%rsi), %xmm1
-; SSE-NEXT: pmaddubsw (%rdi), %xmm0
-; SSE-NEXT: pmaddubsw 16(%rdi), %xmm1
+; SSE-NEXT: movdqa (%rdi), %xmm2
+; SSE-NEXT: movdqa 16(%rdi), %xmm0
+; SSE-NEXT: movdqa (%rsi), %xmm3
+; SSE-NEXT: movdqa 16(%rsi), %xmm7
+; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT: movdqa %xmm7, %xmm1
+; SSE-NEXT: pand %xmm5, %xmm1
+; SSE-NEXT: pand %xmm3, %xmm5
+; SSE-NEXT: packuswb %xmm1, %xmm5
+; SSE-NEXT: pxor %xmm6, %xmm6
+; SSE-NEXT: movdqa %xmm5, %xmm4
+; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
+; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; SSE-NEXT: movdqa %xmm3, %xmm1
+; SSE-NEXT: psrlw $8, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE-NEXT: movdqa %xmm7, %xmm1
+; SSE-NEXT: psrlw $8, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; SSE-NEXT: movdqa %xmm2, %xmm8
+; SSE-NEXT: psllw $8, %xmm8
+; SSE-NEXT: psraw $8, %xmm8
+; SSE-NEXT: movdqa %xmm0, %xmm9
+; SSE-NEXT: psllw $8, %xmm9
+; SSE-NEXT: psraw $8, %xmm9
+; SSE-NEXT: psraw $8, %xmm2
+; SSE-NEXT: psraw $8, %xmm0
+; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
+; SSE-NEXT: movdqa %xmm9, %xmm1
+; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE-NEXT: pmaddwd %xmm7, %xmm1
+; SSE-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
+; SSE-NEXT: pmaddwd %xmm5, %xmm9
+; SSE-NEXT: packssdw %xmm9, %xmm1
+; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
+; SSE-NEXT: movdqa %xmm8, %xmm0
+; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE-NEXT: pmaddwd %xmm3, %xmm0
+; SSE-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
+; SSE-NEXT: pmaddwd %xmm4, %xmm8
+; SSE-NEXT: packssdw %xmm8, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: pmaddubsw_256:
; AVX1: # %bb.0:
+; AVX1-NEXT: vmovdqa (%rdi), %xmm0
+; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm4
+; AVX1-NEXT: vpackuswb %xmm3, %xmm4, %xmm4
+; AVX1-NEXT: vmovq {{.*#+}} xmm5 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX1-NEXT: vpshufb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT: vpshufb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm0[0],xmm1[0]
; AVX1-NEXT: vmovdqa (%rsi), %xmm0
; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1
-; AVX1-NEXT: vpmaddubsw 16(%rdi), %xmm1, %xmm1
-; AVX1-NEXT: vpmaddubsw (%rdi), %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm7
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vpackuswb %xmm7, %xmm2, %xmm2
+; AVX1-NEXT: vpshufb %xmm5, %xmm1, %xmm1
+; AVX1-NEXT: vpshufb %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm0[0],xmm1[0]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; AVX1-NEXT: vpackuswb %xmm3, %xmm3, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm2[1,1,1,1]
+; AVX1-NEXT: vpackuswb %xmm7, %xmm7, %xmm7
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm7[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[2,3,2,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm12 = xmm6[1,1,1,1]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm13 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm14 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm15 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
+; AVX1-NEXT: vpmovsxbw %xmm6, %xmm6
+; AVX1-NEXT: vpmovsxbw %xmm4, %xmm4
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
+; AVX1-NEXT: vpmaddwd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero,xmm14[4],zero,xmm14[5],zero,xmm14[6],zero,xmm14[7],zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm6 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero,xmm8[4],zero,xmm8[5],zero,xmm8[6],zero,xmm8[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX1-NEXT: vpmovsxbw %xmm12, %xmm6
+; AVX1-NEXT: vpmovsxbw %xmm5, %xmm5
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX1-NEXT: vpmaddwd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpackssdw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero,xmm13[4],zero,xmm13[5],zero,xmm13[6],zero,xmm13[7],zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX1-NEXT: vpmovsxbw %xmm11, %xmm5
+; AVX1-NEXT: vpmovsxbw %xmm3, %xmm3
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX1-NEXT: vpmaddwd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX1-NEXT: vpmovsxbw %xmm10, %xmm4
+; AVX1-NEXT: vpmovsxbw %xmm0, %xmm0
+; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
; AVX1-NEXT: retq
;
-; AVX256-LABEL: pmaddubsw_256:
-; AVX256: # %bb.0:
-; AVX256-NEXT: vmovdqa (%rsi), %ymm0
-; AVX256-NEXT: vpmaddubsw (%rdi), %ymm0, %ymm0
-; AVX256-NEXT: retq
+; AVX2-LABEL: pmaddubsw_256:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vmovq {{.*#+}} xmm2 = [0,2,4,6,8,10,12,14,0,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpshufb %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vmovq {{.*#+}} xmm4 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpshufb %xmm4, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rsi), %xmm5
+; AVX2-NEXT: vmovdqa 16(%rsi), %xmm6
+; AVX2-NEXT: vpshufb %xmm2, %xmm5, %xmm7
+; AVX2-NEXT: vpshufb %xmm4, %xmm5, %xmm5
+; AVX2-NEXT: vpshufb %xmm2, %xmm1, %xmm8
+; AVX2-NEXT: vpmovsxbd %xmm8, %ymm8
+; AVX2-NEXT: vpmovsxbd %xmm3, %ymm3
+; AVX2-NEXT: vpshufb %xmm2, %xmm6, %xmm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm2, %ymm8, %ymm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm7, %ymm3, %ymm3
+; AVX2-NEXT: vpshufb %xmm4, %xmm1, %xmm1
+; AVX2-NEXT: vpmovsxbd %xmm1, %ymm1
+; AVX2-NEXT: vpmovsxbd %xmm0, %ymm0
+; AVX2-NEXT: vpshufb %xmm4, %xmm6, %xmm4
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX2-NEXT: vpmaddwd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: pmaddubsw_256:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512F-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm4
+; AVX512F-NEXT: vpackuswb %xmm3, %xmm4, %xmm3
+; AVX512F-NEXT: vmovq {{.*#+}} xmm4 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
+; AVX512F-NEXT: vpshufb %xmm4, %xmm1, %xmm1
+; AVX512F-NEXT: vpshufb %xmm4, %xmm0, %xmm0
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512F-NEXT: vmovdqa (%rsi), %xmm1
+; AVX512F-NEXT: vmovdqa 16(%rsi), %xmm5
+; AVX512F-NEXT: vpand %xmm2, %xmm5, %xmm6
+; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT: vpackuswb %xmm6, %xmm2, %xmm2
+; AVX512F-NEXT: vpshufb %xmm4, %xmm5, %xmm5
+; AVX512F-NEXT: vpshufb %xmm4, %xmm1, %xmm1
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm2, %zmm3, %zmm2
+; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT: retq
+;
+; AVX512BW-LABEL: pmaddubsw_256:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm1
+; AVX512BW-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vmovdqa (%rsi), %ymm2
+; AVX512BW-NEXT: vpsrlw $8, %ymm2, %ymm3
+; AVX512BW-NEXT: vpmovsxbd %xmm1, %zmm1
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
+; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512BW-NEXT: vpmaddwd %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512BW-NEXT: retq
%A = load <32 x i8>, ptr %Aptr
%B = load <32 x i8>, ptr %Bptr
%A_even = shufflevector <32 x i8> %A, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -152,24 +322,167 @@ define <64 x i16> @pmaddubsw_512(ptr %Aptr, ptr %Bptr) {
;
; AVX512F-LABEL: pmaddubsw_512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rsi), %ymm0
-; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1
-; AVX512F-NEXT: vmovdqa 64(%rsi), %ymm2
-; AVX512F-NEXT: vmovdqa 96(%rsi), %ymm3
-; AVX512F-NEXT: vpmaddubsw 96(%rdi), %ymm3, %ymm3
-; AVX512F-NEXT: vpmaddubsw 64(%rdi), %ymm2, %ymm2
-; AVX512F-NEXT: vpmaddubsw 32(%rdi), %ymm1, %ymm1
-; AVX512F-NEXT: vpmaddubsw (%rdi), %ymm0, %ymm0
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm3
+; AVX512F-NEXT: vmovdqa 64(%rdi), %ymm4
+; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm5
+; AVX512F-NEXT: vpshufb %ymm1, %ymm5, %ymm2
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512F-NEXT: vpshufb %ymm6, %ymm4, %ymm7
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3],ymm7[4,5],ymm2[6,7]
+; AVX512F-NEXT: vpshufb %ymm1, %ymm3, %ymm7
+; AVX512F-NEXT: vpshufb %ymm6, %ymm0, %ymm8
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm7, %zmm2
+; AVX512F-NEXT: vpermq {{.*#+}} zmm2 = zmm2[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm7 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm5, %ymm5
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm8 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512F-NEXT: vpshufb %ymm8, %ymm4, %ymm4
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5],ymm5[6,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm3, %ymm3
+; AVX512F-NEXT: vpshufb %ymm8, %ymm0, %ymm0
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm0
+; AVX512F-NEXT: vpermq {{.*#+}} zmm0 = zmm0[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vmovdqa (%rsi), %ymm3
+; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm4
+; AVX512F-NEXT: vmovdqa 64(%rsi), %ymm5
+; AVX512F-NEXT: vmovdqa 96(%rsi), %ymm9
+; AVX512F-NEXT: vpshufb %ymm1, %ymm9, %ymm10
+; AVX512F-NEXT: vpshufb %ymm6, %ymm5, %ymm11
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,3],ymm11[4,5],ymm10[6,7]
+; AVX512F-NEXT: vpshufb %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT: vpshufb %ymm6, %ymm3, %ymm6
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm6[0,1],ymm1[2,3],ymm6[4,5],ymm1[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm10, %zmm1, %zmm1
+; AVX512F-NEXT: vpermq {{.*#+}} zmm6 = zmm1[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm9, %ymm1
+; AVX512F-NEXT: vpshufb %ymm8, %ymm5, %ymm5
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3],ymm5[4,5],ymm1[6,7]
+; AVX512F-NEXT: vpshufb %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT: vpshufb %ymm8, %ymm3, %ymm3
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]
+; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,2,1,3,4,6,5,7]
+; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm3
+; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
+; AVX512F-NEXT: vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm5
+; AVX512F-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512F-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm6, %ymm7
+; AVX512F-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero,xmm8[4],zero,zero,zero,xmm8[5],zero,zero,zero,xmm8[6],zero,zero,zero,xmm8[7],zero,zero,zero,xmm8[8],zero,zero,zero,xmm8[9],zero,zero,zero,xmm8[10],zero,zero,zero,xmm8[11],zero,zero,zero,xmm8[12],zero,zero,zero,xmm8[13],zero,zero,zero,xmm8[14],zero,zero,zero,xmm8[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm8, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm6, %xmm7
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm5, %zmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm6, %zmm2, %zmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm6
+; AVX512F-NEXT: vextracti128 $1, %ymm6, %xmm7
+; AVX512F-NEXT: vpmovsxbd %xmm7, %zmm7
+; AVX512F-NEXT: vpmovsxbd %xmm6, %zmm6
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm8
+; AVX512F-NEXT: vpmovsxbd %xmm8, %zmm8
+; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm9
+; AVX512F-NEXT: vextracti128 $1, %ymm9, %xmm10
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero,xmm10[8],zero,zero,zero,xmm10[9],zero,zero,zero,xmm10[10],zero,zero,zero,xmm10[11],zero,zero,zero,xmm10[12],zero,zero,zero,xmm10[13],zero,zero,zero,xmm10[14],zero,zero,zero,xmm10[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm10, %zmm7, %zmm7
+; AVX512F-NEXT: vpaddd %zmm7, %zmm4, %zmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero,xmm9[8],zero,zero,zero,xmm9[9],zero,zero,zero,xmm9[10],zero,zero,zero,xmm9[11],zero,zero,zero,xmm9[12],zero,zero,zero,xmm9[13],zero,zero,zero,xmm9[14],zero,zero,zero,xmm9[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm7, %zmm6, %zmm6
+; AVX512F-NEXT: vpaddd %zmm6, %zmm3, %zmm3
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm6, %zmm8, %zmm6
+; AVX512F-NEXT: vpaddd %zmm6, %zmm5, %zmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT: vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT: vpmovsdw %zmm5, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm1
+; AVX512F-NEXT: vpmovsdw %zmm3, %ymm1
+; AVX512F-NEXT: vpmovsdw %zmm4, %ymm2
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: pmaddubsw_512:
; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm1
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm2 = [128,128,128,128,128,128,128,128,0,2,4,6,8,10,12,14,128,128,128,128,128,128,128,128,16,18,20,22,24,26,28,30,128,128,128,128,128,128,128,128,32,34,36,38,40,42,44,46,128,128,128,128,128,128,128,128,48,50,52,54,56,58,60,62]
+; AVX512BW-NEXT: vpshufb %zmm2, %zmm1, %zmm3
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,2,4,6,8,10,12,14,128,128,128,128,128,128,128,128,16,18,20,22,24,26,28,30,128,128,128,128,128,128,128,128,32,34,36,38,40,42,44,46,128,128,128,128,128,128,128,128,48,50,52,54,56,58,60,62,128,128,128,128,128,128,128,128]
+; AVX512BW-NEXT: vpshufb %zmm4, %zmm0, %zmm5
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm6 = [0,2,4,6,9,11,13,15]
+; AVX512BW-NEXT: vpermt2q %zmm3, %zmm6, %zmm5
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,1,3,5,7,9,11,13,15,128,128,128,128,128,128,128,128,17,19,21,23,25,27,29,31,128,128,128,128,128,128,128,128,33,35,37,39,41,43,45,47,128,128,128,128,128,128,128,128,49,51,53,55,57,59,61,63]
+; AVX512BW-NEXT: vpshufb %zmm3, %zmm1, %zmm7
+; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm8 = [1,3,5,7,9,11,13,15,128,128,128,128,128,128,128,128,17,19,21,23,25,27,29,31,128,128,128,128,128,128,128,128,33,35,37,39,41,43,45,47,128,128,128,128,128,128,128,128,49,51,53,55,57,59,61,63,128,128,128,128,128,128,128,128]
+; AVX512BW-NEXT: vpshufb %zmm8, %zmm0, %zmm1
+; AVX512BW-NEXT: vpermt2q %zmm7, %zmm6, %zmm1
; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0
-; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm1
-; AVX512BW-NEXT: vpmaddubsw (%rdi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmaddubsw 64(%rdi), %zmm1, %zmm1
+; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm7
+; AVX512BW-NEXT: vpshufb %zmm2, %zmm7, %zmm2
+; AVX512BW-NEXT: vpshufb %zmm4, %zmm0, %zmm4
+; AVX512BW-NEXT: vpermt2q %zmm2, %zmm6, %zmm4
+; AVX512BW-NEXT: vpshufb %zmm3, %zmm7, %zmm2
+; AVX512BW-NEXT: vpshufb %zmm8, %zmm0, %zmm0
+; AVX512BW-NEXT: vpermt2q %zmm2, %zmm6, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm5, %ymm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX512BW-NEXT: vpmovsxbd %xmm3, %zmm3
+; AVX512BW-NEXT: vpmovsxbd %xmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm5, %xmm6
+; AVX512BW-NEXT: vpmovsxbd %xmm6, %zmm6
+; AVX512BW-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm4, %ymm7
+; AVX512BW-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero,xmm8[4],zero,zero,zero,xmm8[5],zero,zero,zero,xmm8[6],zero,zero,zero,xmm8[7],zero,zero,zero,xmm8[8],zero,zero,zero,xmm8[9],zero,zero,zero,xmm8[10],zero,zero,zero,xmm8[11],zero,zero,zero,xmm8[12],zero,zero,zero,xmm8[13],zero,zero,zero,xmm8[14],zero,zero,zero,xmm8[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm8, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm4, %xmm7
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm6, %zmm6
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm4, %zmm5, %zmm4
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm1, %ymm5
+; AVX512BW-NEXT: vextracti128 $1, %ymm5, %xmm7
+; AVX512BW-NEXT: vpmovsxbd %xmm7, %zmm7
+; AVX512BW-NEXT: vpmovsxbd %xmm5, %zmm5
+; AVX512BW-NEXT: vextracti128 $1, %ymm1, %xmm8
+; AVX512BW-NEXT: vpmovsxbd %xmm8, %zmm8
+; AVX512BW-NEXT: vpmovsxbd %xmm1, %zmm1
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm9
+; AVX512BW-NEXT: vextracti128 $1, %ymm9, %xmm10
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero,xmm10[8],zero,zero,zero,xmm10[9],zero,zero,zero,xmm10[10],zero,zero,zero,xmm10[11],zero,zero,zero,xmm10[12],zero,zero,zero,xmm10[13],zero,zero,zero,xmm10[14],zero,zero,zero,xmm10[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm10, %zmm7, %zmm7
+; AVX512BW-NEXT: vpaddd %zmm7, %zmm3, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero,xmm9[8],zero,zero,zero,xmm9[9],zero,zero,zero,xmm9[10],zero,zero,zero,xmm9[11],zero,zero,zero,xmm9[12],zero,zero,zero,xmm9[13],zero,zero,zero,xmm9[14],zero,zero,zero,xmm9[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm7, %zmm5, %zmm5
+; AVX512BW-NEXT: vpaddd %zmm5, %zmm2, %zmm2
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm5
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm5, %zmm8, %zmm5
+; AVX512BW-NEXT: vpaddd %zmm5, %zmm6, %zmm5
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512BW-NEXT: vpmaddwd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,6,7],zmm5[2,3,6,7]
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm5[0,1,4,5]
+; AVX512BW-NEXT: vpackssdw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm2[2,3,6,7],zmm3[2,3,6,7]
+; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,4,5],zmm3[0,1,4,5]
+; AVX512BW-NEXT: vpackssdw %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: retq
%A = load <128 x i8>, ptr %Aptr
%B = load <128 x i8>, ptr %Bptr
diff --git a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
index 2ac2be5545dfd..e270da5e13ea4 100644
--- a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
+++ b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
@@ -61,22 +61,22 @@ define void @failing(ptr %0, ptr %1) nounwind {
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
; CHECK-NEXT: movdqu 1024(%rdx,%rsi), %xmm5
; CHECK-NEXT: movdqu 1040(%rdx,%rsi), %xmm6
-; CHECK-NEXT: movq %xmm5, %rdi
-; CHECK-NEXT: movq %xmm6, %r8
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm5[2,3,2,3]
+; CHECK-NEXT: movq %xmm7, %rdi
+; CHECK-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; CHECK-NEXT: movq %xmm7, %r8
; CHECK-NEXT: movq %xmm5, %r9
-; CHECK-NEXT: pshufd {{.*#+}} xmm5 = xmm6[2,3,2,3]
-; CHECK-NEXT: movq %xmm5, %r10
-; CHECK-NEXT: negq %r8
-; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: sbbq %r10, %r8
+; CHECK-NEXT: movq %xmm6, %r10
+; CHECK-NEXT: negq %r10
+; CHECK-NEXT: movq %rcx, %r10
+; CHECK-NEXT: sbbq %r8, %r10
; CHECK-NEXT: setge %r8b
; CHECK-NEXT: movzbl %r8b, %r8d
; CHECK-NEXT: negq %r8
; CHECK-NEXT: movq %r8, %xmm5
-; CHECK-NEXT: negq %rdi
-; CHECK-NEXT: movq %rcx, %rdi
-; CHECK-NEXT: sbbq %r9, %rdi
+; CHECK-NEXT: negq %r9
+; CHECK-NEXT: movq %rcx, %r8
+; CHECK-NEXT: sbbq %rdi, %r8
; CHECK-NEXT: setge %dil
; CHECK-NEXT: movzbl %dil, %edi
; CHECK-NEXT: negq %rdi
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
index 26af46263c0e2..922ed2bd63fee 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
@@ -10,10 +10,10 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512BWVL-ONLY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512BWVL-ONLY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512VBMI,AVX512VBMI-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL,AVX512VBMI,AVX512VBMI-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
; PR31551
; Pairs of shufflevector:trunc functions with functional equivalence.
@@ -889,253 +889,54 @@ define <16 x i8> @evenelts_v32i16_trunc_v16i16_to_v16i8(<32 x i16> %n2) nounwind
;
; AVX1-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %rbx
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpextrw $6, %xmm2, %eax
-; AVX1-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX1-NEXT: vpextrw $2, %xmm2, %edx
-; AVX1-NEXT: vmovd %xmm2, %esi
-; AVX1-NEXT: vpextrw $6, %xmm1, %edi
-; AVX1-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX1-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX1-NEXT: vmovd %xmm1, %r10d
+; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3],xmm1[4],xmm3[5],xmm1[6],xmm3[7]
+; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX1-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX1-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX1-NEXT: vmovd %xmm1, %r14d
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
-; AVX2-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrw $6, %xmm2, %eax
-; AVX2-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm2, %edx
-; AVX2-NEXT: vmovd %xmm2, %esi
-; AVX2-NEXT: vpextrw $6, %xmm1, %edi
-; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX2-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX2-NEXT: vmovd %xmm1, %r10d
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX2-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX2-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX2-NEXT: vmovd %xmm1, %r14d
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: retq
-;
-; AVX512F-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vpmovdb %zmm0, %xmm1
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrw $6, %xmm2, %eax
-; AVX512F-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX512F-NEXT: vpextrw $2, %xmm2, %edx
-; AVX512F-NEXT: vmovd %xmm2, %esi
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX512F-NEXT: vpextrw $6, %xmm2, %edi
-; AVX512F-NEXT: vpextrw $4, %xmm2, %r8d
-; AVX512F-NEXT: vpextrw $2, %xmm2, %r9d
-; AVX512F-NEXT: vmovd %xmm2, %r10d
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512F-NEXT: vpextrw $4, %xmm0, %ebx
-; AVX512F-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512F-NEXT: vpinsrb $5, %ebp, %xmm1, %xmm0
-; AVX512F-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %rbp
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbp
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512VL-NEXT: vmovd %xmm1, %esi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %edi
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX512VL-NEXT: vmovd %xmm1, %r10d
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX512VL-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX512VL-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX512VL-NEXT: vmovd %xmm1, %r14d
-; AVX512VL-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %rbp
-; AVX512VL-NEXT: vzeroupper
-; AVX512VL-NEXT: retq
+; AVX2-SLOW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
+; AVX2-SLOW-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX2-SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-SLOW-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX2-SLOW-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vzeroupper
+; AVX2-SLOW-NEXT: retq
;
-; AVX512BW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: pushq %rbp
-; AVX512BW-NEXT: pushq %rbx
-; AVX512BW-NEXT: vpmovdb %zmm0, %xmm1
-; AVX512BW-NEXT: vextracti32x4 $3, %zmm0, %xmm2
-; AVX512BW-NEXT: vpextrw $6, %xmm2, %eax
-; AVX512BW-NEXT: vpextrw $4, %xmm2, %ecx
-; AVX512BW-NEXT: vpextrw $2, %xmm2, %edx
-; AVX512BW-NEXT: vmovd %xmm2, %esi
-; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, %xmm2
-; AVX512BW-NEXT: vpextrw $6, %xmm2, %edi
-; AVX512BW-NEXT: vpextrw $4, %xmm2, %r8d
-; AVX512BW-NEXT: vpextrw $2, %xmm2, %r9d
-; AVX512BW-NEXT: vmovd %xmm2, %r10d
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512BW-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512BW-NEXT: vpextrw $4, %xmm0, %ebx
-; AVX512BW-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512BW-NEXT: vpinsrb $5, %ebp, %xmm1, %xmm0
-; AVX512BW-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BW-NEXT: popq %rbx
-; AVX512BW-NEXT: popq %rbp
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
+; AVX2-FAST-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpmovsxdq {{.*#+}} ymm2 = [84148480,218892552,353636624,488380696]
+; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX2-FAST-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-FAST-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX2-FAST-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
;
-; AVX512BWVL-ONLY-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BWVL-ONLY: # %bb.0:
-; AVX512BWVL-ONLY-NEXT: pushq %rbp
-; AVX512BWVL-ONLY-NEXT: pushq %r14
-; AVX512BWVL-ONLY-NEXT: pushq %rbx
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %ecx
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %edx
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %esi
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %edi
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %r8d
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %r9d
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %r10d
-; AVX512BWVL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $6, %xmm1, %r11d
-; AVX512BWVL-ONLY-NEXT: vpextrw $4, %xmm1, %ebx
-; AVX512BWVL-ONLY-NEXT: vpextrw $2, %xmm1, %ebp
-; AVX512BWVL-ONLY-NEXT: vmovd %xmm1, %r14d
-; AVX512BWVL-ONLY-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: popq %rbx
-; AVX512BWVL-ONLY-NEXT: popq %r14
-; AVX512BWVL-ONLY-NEXT: popq %rbp
-; AVX512BWVL-ONLY-NEXT: vzeroupper
-; AVX512BWVL-ONLY-NEXT: retq
-;
-; AVX512VBMI-FAST-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-FAST: # %bb.0:
-; AVX512VBMI-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,79]
-; AVX512VBMI-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI-FAST-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
-; AVX512VBMI-FAST-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-FAST-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512VBMI-FAST-NEXT: vpinsrb $15, %eax, %xmm1, %xmm0
-; AVX512VBMI-FAST-NEXT: vzeroupper
-; AVX512VBMI-FAST-NEXT: retq
-;
-; AVX512VBMI-SLOW-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-SLOW: # %bb.0:
-; AVX512VBMI-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [0,4,8,12,16,20,24,28,32,36,40,44,48,77,78,79]
-; AVX512VBMI-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VBMI-SLOW-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
-; AVX512VBMI-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512VBMI-SLOW-NEXT: vpextrw $4, %xmm0, %ecx
-; AVX512VBMI-SLOW-NEXT: vpextrw $2, %xmm0, %edx
-; AVX512VBMI-SLOW-NEXT: vpinsrb $13, %edx, %xmm1, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vzeroupper
-; AVX512VBMI-SLOW-NEXT: retq
+; AVX512-LABEL: evenelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%n0 = shufflevector <32 x i16> %n2, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%n1 = trunc <16 x i16> %n0 to <16 x i8>
ret <16 x i8> %n1
@@ -1230,260 +1031,41 @@ define <16 x i8> @oddelts_v32i16_trunc_v16i16_to_v16i8(<32 x i16> %n2) nounwind
;
; AVX1-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %rbx
; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpextrw $7, %xmm2, %eax
-; AVX1-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX1-NEXT: vpextrw $3, %xmm2, %edx
-; AVX1-NEXT: vpextrw $1, %xmm2, %esi
-; AVX1-NEXT: vpextrw $7, %xmm1, %edi
-; AVX1-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX1-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX1-NEXT: vpextrw $1, %xmm1, %r10d
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX1-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX1-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX1-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpextrw $7, %xmm2, %eax
-; AVX2-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX2-NEXT: vpextrw $3, %xmm2, %edx
-; AVX2-NEXT: vpextrw $1, %xmm2, %esi
-; AVX2-NEXT: vpextrw $7, %xmm1, %edi
-; AVX2-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX2-NEXT: vpextrw $1, %xmm1, %r10d
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [2,3,6,7,10,11,14,15,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX2-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX2-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX2-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: pushq %r14
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512F-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512F-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512F-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512F-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512F-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512F-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512F-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512F-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512F-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512F-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %r14
-; AVX512F-NEXT: popq %rbp
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbp
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512VL-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512VL-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512VL-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512VL-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VL-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %rbp
-; AVX512VL-NEXT: vzeroupper
-; AVX512VL-NEXT: retq
-;
-; AVX512BW-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: pushq %rbp
-; AVX512BW-NEXT: pushq %r14
-; AVX512BW-NEXT: pushq %rbx
-; AVX512BW-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512BW-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512BW-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512BW-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BW-NEXT: popq %rbx
-; AVX512BW-NEXT: popq %r14
-; AVX512BW-NEXT: popq %rbp
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-ONLY-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512BWVL-ONLY: # %bb.0:
-; AVX512BWVL-ONLY-NEXT: pushq %rbp
-; AVX512BWVL-ONLY-NEXT: pushq %r14
-; AVX512BWVL-ONLY-NEXT: pushq %rbx
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %ecx
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %edx
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %esi
-; AVX512BWVL-ONLY-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %edi
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %r8d
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %r9d
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %r10d
-; AVX512BWVL-ONLY-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-ONLY-NEXT: vpextrw $7, %xmm1, %r11d
-; AVX512BWVL-ONLY-NEXT: vpextrw $5, %xmm1, %ebx
-; AVX512BWVL-ONLY-NEXT: vpextrw $3, %xmm1, %ebp
-; AVX512BWVL-ONLY-NEXT: vpextrw $1, %xmm1, %r14d
-; AVX512BWVL-ONLY-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512BWVL-ONLY-NEXT: vpinsrb $4, %r14d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $5, %ebp, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $6, %ebx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $7, %r11d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $8, %r10d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $10, %r8d, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $11, %edi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $12, %esi, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $13, %edx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512BWVL-ONLY-NEXT: popq %rbx
-; AVX512BWVL-ONLY-NEXT: popq %r14
-; AVX512BWVL-ONLY-NEXT: popq %rbp
-; AVX512BWVL-ONLY-NEXT: vzeroupper
-; AVX512BWVL-ONLY-NEXT: retq
-;
-; AVX512VBMI-FAST-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-FAST: # %bb.0:
-; AVX512VBMI-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]
-; AVX512VBMI-FAST-NEXT: vpermb %zmm0, %zmm1, %zmm0
-; AVX512VBMI-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512VBMI-FAST-NEXT: vzeroupper
-; AVX512VBMI-FAST-NEXT: retq
-;
-; AVX512VBMI-SLOW-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
-; AVX512VBMI-SLOW: # %bb.0:
-; AVX512VBMI-SLOW-NEXT: vmovdqa {{.*#+}} xmm1 = [2,6,10,14,18,22,26,30,34,38,42,46,50,u,u,u]
-; AVX512VBMI-SLOW-NEXT: vpermb %zmm0, %zmm1, %zmm1
-; AVX512VBMI-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512VBMI-SLOW-NEXT: vpextrw $5, %xmm0, %ecx
-; AVX512VBMI-SLOW-NEXT: vpextrw $3, %xmm0, %edx
-; AVX512VBMI-SLOW-NEXT: vpinsrb $13, %edx, %xmm1, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0
-; AVX512VBMI-SLOW-NEXT: vzeroupper
-; AVX512VBMI-SLOW-NEXT: retq
+; AVX512-LABEL: oddelts_v32i16_trunc_v16i16_to_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsrld $16, %zmm0, %zmm0
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%n0 = shufflevector <32 x i16> %n2, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
%n1 = trunc <16 x i16> %n0 to <16 x i8>
ret <16 x i8> %n1
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512VBMI: {{.*}}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 05c855ed90b3f..93ce094d7cf8c 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -1301,73 +1301,74 @@ define void @trunc_v4i64_to_v4i8(ptr %L, ptr %S) nounwind {
define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind {
; AVX1-LABEL: negative:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[u,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u],zero,zero,zero,zero,zero,zero,zero,xmm0[0,2,4,6,8,10,12,14]
-; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615]
-; AVX1-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vmovd %xmm1, %eax
+; AVX1-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: negative:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX2-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm1, %eax
+; AVX2-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: negative:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512F-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX512F-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT: vpbroadcastw {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX512F-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: negative:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm0[2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512VL-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm2 = (ymm1 & ~ymm2) | ymm0
-; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm2[0,3,2,3]
-; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX512VL-NEXT: vpand %xmm3, %xmm2, %xmm2
+; AVX512VL-NEXT: vpand %xmm3, %xmm0, %xmm0
+; AVX512VL-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512BW-LABEL: negative:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BW-NEXT: vpmovsxwq {{.*#+}} ymm2 = [18446744073709551360,18446744073709551615,18446744073709551360,18446744073709551615]
-; AVX512BW-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vmovd %xmm1, %eax
+; AVX512BW-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: negative:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BWVL-NEXT: movl $65537, %eax # imm = 0x10001
-; AVX512BWVL-NEXT: kmovd %eax, %k1
-; AVX512BWVL-NEXT: vmovdqu8 %ymm1, %ymm0 {%k1}
-; AVX512BWVL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX512BWVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vmovd %xmm1, %eax
+; AVX512BWVL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
;
; AVX512VBMIVL-LABEL: negative:
; AVX512VBMIVL: # %bb.0:
-; AVX512VBMIVL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [32,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
-; AVX512VBMIVL-NEXT: # ymm2 = mem[0,1,0,1]
-; AVX512VBMIVL-NEXT: vpermt2b %ymm1, %ymm2, %ymm0
-; AVX512VBMIVL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMIVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512VBMIVL-NEXT: vmovd %xmm1, %eax
+; AVX512VBMIVL-NEXT: vpinsrb $0, %eax, %xmm0, %xmm0
; AVX512VBMIVL-NEXT: vzeroupper
; AVX512VBMIVL-NEXT: retq
%strided.vec = shufflevector <32 x i8> %v, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
index e0410ae0cc5cb..7a146f78dafb7 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-2.ll
@@ -164,62 +164,62 @@ define void @load_i16_stride2_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i16_stride2_vf8:
; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i16_stride2_vf8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i16_stride2_vf8:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-FP-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FP-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-FP-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FP-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-FP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i16_stride2_vf8:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
-; AVX2-FCP-NEXT: vpsrld $16, %xmm2, %xmm2
+; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FCP-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
+; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
+; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; AVX2-FCP-NEXT: vpsrld $16, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FCP-NEXT: vpsrld $16, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i16_stride2_vf8:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
index 440e0187d69f3..08ce45ef615bd 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
@@ -416,130 +416,130 @@ define void @load_i8_stride2_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX-LABEL: load_i8_stride2_vf16:
; AVX: # %bb.0:
-; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX-NEXT: vmovdqa (%rdi), %xmm1
-; AVX-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX-NEXT: vmovdqa (%rdi), %xmm0
+; AVX-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX-NEXT: vmovdqa %xmm0, (%rdx)
; AVX-NEXT: retq
;
; AVX2-LABEL: load_i8_stride2_vf16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX2-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-NEXT: retq
;
; AVX2-FP-LABEL: load_i8_stride2_vf16:
; AVX2-FP: # %bb.0:
-; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-FP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-FP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX2-FP-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX2-FP-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX2-FP-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX2-FP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-FP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-FP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FP-NEXT: retq
;
; AVX2-FCP-LABEL: load_i8_stride2_vf16:
; AVX2-FCP: # %bb.0:
-; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX2-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX2-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX2-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX2-FCP-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX2-FCP-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX2-FCP-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX2-FCP-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX2-FCP-NEXT: retq
;
; AVX512-LABEL: load_i8_stride2_vf16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX512-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX512-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX512-NEXT: vmovdqa %xmm0, (%rdx)
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i8_stride2_vf16:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512-FCP-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512-FCP-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX512-FCP-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i8_stride2_vf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512DQ-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512DQ-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512DQ-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX512DQ-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX512DQ-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512DQ-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512DQ-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512DQ-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512DQ-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX512DQ-NEXT: vmovdqa %xmm0, (%rdx)
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i8_stride2_vf16:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT: vpand %xmm0, %xmm2, %xmm3
-; AVX512DQ-FCP-NEXT: vpand %xmm0, %xmm1, %xmm0
-; AVX512DQ-FCP-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512DQ-FCP-NEXT: vpand %xmm2, %xmm1, %xmm3
+; AVX512DQ-FCP-NEXT: vpand %xmm2, %xmm0, %xmm2
+; AVX512DQ-FCP-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0]
-; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2
; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rdx)
; AVX512DQ-FCP-NEXT: retq
;
; AVX512BW-LABEL: load_i8_stride2_vf16:
@@ -700,75 +700,75 @@ define void @load_i8_stride2_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
;
; AVX512-LABEL: load_i8_stride2_vf32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512-NEXT: vpshufb %ymm0, %ymm1, %ymm0
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
-; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm2
+; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
;
; AVX512-FCP-LABEL: load_i8_stride2_vf32:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm2
; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [0,2,5,7]
-; AVX512-FCP-NEXT: vpermt2q %ymm3, %ymm4, %ymm0
+; AVX512-FCP-NEXT: vpermt2q %ymm3, %ymm4, %ymm2
; AVX512-FCP-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-FCP-NEXT: vpermt2q %ymm2, %ymm4, %ymm1
-; AVX512-FCP-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512-FCP-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512-FCP-NEXT: vpermt2q %ymm1, %ymm4, %ymm0
+; AVX512-FCP-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512-FCP-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
; AVX512DQ-LABEL: load_i8_stride2_vf32:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-NEXT: vpshufb %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512DQ-NEXT: vpshufb %ymm2, %ymm0, %ymm2
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; AVX512DQ-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512DQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3],ymm1[4,5],ymm2[6,7]
-; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512DQ-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512DQ-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512DQ-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512DQ-NEXT: vzeroupper
; AVX512DQ-NEXT: retq
;
; AVX512DQ-FCP-LABEL: load_i8_stride2_vf32:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm2
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm3
-; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm2
; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm4 = [0,2,5,7]
-; AVX512DQ-FCP-NEXT: vpermt2q %ymm3, %ymm4, %ymm0
+; AVX512DQ-FCP-NEXT: vpermt2q %ymm3, %ymm4, %ymm2
; AVX512DQ-FCP-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm2
; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT: vpermt2q %ymm2, %ymm4, %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, (%rsi)
-; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, (%rdx)
+; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT: vpermt2q %ymm1, %ymm4, %ymm0
+; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, (%rsi)
+; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, (%rdx)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-narrow-binop.ll b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
index d9915d21186f9..63c7c7596eadb 100644
--- a/llvm/test/CodeGen/X86/vector-narrow-binop.ll
+++ b/llvm/test/CodeGen/X86/vector-narrow-binop.ll
@@ -60,15 +60,15 @@ define <8 x i32> @PR32790(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d
define <4 x i32> @do_not_use_256bit_op(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) {
; SSE-LABEL: do_not_use_256bit_op:
; SSE: # %bb.0:
-; SSE-NEXT: pand %xmm2, %xmm0
; SSE-NEXT: pand %xmm3, %xmm1
+; SSE-NEXT: pand %xmm2, %xmm0
; SSE-NEXT: psubd %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX-LABEL: do_not_use_256bit_op:
; AVX: # %bb.0:
-; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX-NEXT: vpand %xmm3, %xmm1, %xmm1
+; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%concat1 = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-subvector.ll b/llvm/test/CodeGen/X86/vector-reduce-add-subvector.ll
index 2a4967916c7cb..0249d3958399e 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-subvector.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-subvector.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,X86-SSE,X86-SSE2
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,X64-SSE
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,X64-SSE,X64-SSE2
; RUN: llc < %s -mtriple=i686-- -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X86-SSE,X86-SSE4
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X64-SSE
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,X64-SSE,X64-SSE4
; RUN: llc < %s -mtriple=i686-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,X86-SSE,X86-SSE4
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,X64-SSE
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,X64-SSE,X64-SSE4
; RUN: llc < %s -mtriple=i686-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,X86-AVX,AVX1,AVX1-SLOW,X86-AVX1
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,X64-AVX,AVX1,AVX1-SLOW,X64-AVX1
; RUN: llc < %s -mtriple=i686-- -mattr=+avx,+fast-hops | FileCheck %s --check-prefixes=AVX,X86-AVX,AVX1,AVX1-FAST,X86-AVX1
@@ -470,7 +470,8 @@ define i32 @test_v8i32_v4i32_mid(<8 x i32> %a0) nounwind {
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,2],xmm1[0,1]
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: vzeroupper
@@ -690,7 +691,7 @@ define i32 @test_v16i32_v8i32_mid(<16 x i32> %a0) nounwind {
; AVX1-FAST-LABEL: test_v16i32_v8i32_mid:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -985,23 +986,86 @@ define i8 @test_v16i8_v8i8_hi(<16 x i8> %a0) nounwind {
}
define i8 @test_v32i8_v16i8_mid(<32 x i8> %a0) nounwind {
-; SSE-LABEL: test_v32i8_v16i8_mid:
-; SSE: # %bb.0:
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; SSE-NEXT: paddb %xmm1, %xmm0
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: # kill: def $al killed $al killed $eax
-; SSE-NEXT: ret{{[l|q]}}
+; X86-SSE2-LABEL: test_v32i8_v16i8_mid:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm1[0,1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; X86-SSE2-NEXT: paddb %xmm0, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: psrld $16, %xmm1
+; X86-SSE2-NEXT: paddb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrlw $8, %xmm0
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v32i8_v16i8_mid:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm1[0,1]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; X64-SSE2-NEXT: paddb %xmm0, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE2-NEXT: paddb %xmm1, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT: psrld $16, %xmm1
+; X64-SSE2-NEXT: paddb %xmm0, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT: psrlw $8, %xmm0
+; X64-SSE2-NEXT: paddb %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v32i8_v16i8_mid:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,1,0,1]
+; X86-SSE4-NEXT: palignr {{.*#+}} xmm1 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
+; X86-SSE4-NEXT: paddb %xmm2, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE4-NEXT: paddb %xmm1, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: psrld $16, %xmm1
+; X86-SSE4-NEXT: paddb %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE4-NEXT: psrlw $8, %xmm0
+; X86-SSE4-NEXT: paddb %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: # kill: def $al killed $al killed $eax
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v32i8_v16i8_mid:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,1,0,1]
+; X64-SSE4-NEXT: palignr {{.*#+}} xmm1 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
+; X64-SSE4-NEXT: paddb %xmm2, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE4-NEXT: paddb %xmm1, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE4-NEXT: psrld $16, %xmm1
+; X64-SSE4-NEXT: paddb %xmm0, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE4-NEXT: psrlw $8, %xmm0
+; X64-SSE4-NEXT: paddb %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: # kill: def $al killed $al killed $eax
+; X64-SSE4-NEXT: retq
;
; AVX1-LABEL: test_v32i8_v16i8_mid:
; AVX1: # %bb.0:
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vmovd %xmm0, %eax
; AVX1-NEXT: # kill: def $al killed $al killed $eax
; AVX1-NEXT: vzeroupper
@@ -1009,11 +1073,16 @@ define i8 @test_v32i8_v16i8_mid(<32 x i8> %a0) nounwind {
;
; AVX2-LABEL: test_v32i8_v16i8_mid:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[1,2,2,3]
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpbroadcastq %xmm0, %xmm0
; AVX2-NEXT: vpaddb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $al killed $al killed $eax
; AVX2-NEXT: vzeroupper
@@ -1021,11 +1090,16 @@ define i8 @test_v32i8_v16i8_mid(<32 x i8> %a0) nounwind {
;
; AVX512-LABEL: test_v32i8_v16i8_mid:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm0[1,2,2,3]
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vpbroadcastq %xmm0, %xmm0
; AVX512-NEXT: vpaddb %xmm0, %xmm1, %xmm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $al killed $al killed $eax
; AVX512-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/vpdpwssd.ll b/llvm/test/CodeGen/X86/vpdpwssd.ll
index ea97800505bc2..62202b778b957 100644
--- a/llvm/test/CodeGen/X86/vpdpwssd.ll
+++ b/llvm/test/CodeGen/X86/vpdpwssd.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI,AVX512VL-VNNI-SLOW
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512-VNNI
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI,AVX512VL-VNNI-FAST
define <16 x i32> @vpdpwssd_test(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) {
; CHECK-LABEL: vpdpwssd_test:
@@ -31,23 +31,57 @@ define <16 x i32> @vpdpwssd_v16i32_accumulate(<32 x i16> %a0, <32 x i16> %a1, <1
}
define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x i32> %a2) {
-; AVX512VL-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
-; AVX512VL-VNNI: # %bb.0:
-; AVX512VL-VNNI-NEXT: vpdpwssd %ymm1, %ymm0, %ymm2
-; AVX512VL-VNNI-NEXT: vmovdqa %ymm2, %ymm0
-; AVX512VL-VNNI-NEXT: retq
+; AVX512VL-VNNI-SLOW-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI-SLOW: # %bb.0:
+; AVX512VL-VNNI-SLOW-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-VNNI-SLOW-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512VL-VNNI-SLOW-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0
+; AVX512VL-VNNI-SLOW-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512VL-VNNI-SLOW-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-VNNI-SLOW-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
+; AVX512VL-VNNI-SLOW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512VL-VNNI-SLOW-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX512VL-VNNI-SLOW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512VL-VNNI-SLOW-NEXT: retq
;
; AVX-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
; AVX-VNNI: # %bb.0:
-; AVX-VNNI-NEXT: {vex} vpdpwssd %ymm1, %ymm0, %ymm2
-; AVX-VNNI-NEXT: vmovdqa %ymm2, %ymm0
+; AVX-VNNI-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX-VNNI-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX-VNNI-NEXT: vpmaddwd %zmm1, %zmm0, %zmm0
+; AVX-VNNI-NEXT: vpmovqd %zmm0, %ymm1
+; AVX-VNNI-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX-VNNI-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
+; AVX-VNNI-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX-VNNI-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX-VNNI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX-VNNI-NEXT: retq
;
; AVX512-VNNI-LABEL: vpdpwssd_v8i32_accumulate:
; AVX512-VNNI: # %bb.0:
-; AVX512-VNNI-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
-; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-VNNI-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512-VNNI-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512-VNNI-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-VNNI-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512-VNNI-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
+; AVX512-VNNI-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512-VNNI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX512-VNNI-NEXT: retq
+;
+; AVX512VL-VNNI-FAST-LABEL: vpdpwssd_v8i32_accumulate:
+; AVX512VL-VNNI-FAST: # %bb.0:
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxwd %ymm0, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpmovsxwd %ymm1, %zmm1
+; AVX512VL-VNNI-FAST-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512VL-VNNI-FAST-NEXT: vpmovqd %zmm0, %ymm1
+; AVX512VL-VNNI-FAST-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-VNNI-FAST-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
+; AVX512VL-VNNI-FAST-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512VL-VNNI-FAST-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512VL-VNNI-FAST-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512VL-VNNI-FAST-NEXT: retq
%x0 = sext <16 x i16> %a0 to <16 x i32>
%x1 = sext <16 x i16> %a1 to <16 x i32>
%m = mul nsw <16 x i32> %x0, %x1
@@ -61,20 +95,41 @@ define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x
define <4 x i32> @vpdpwssd_v4i32_accumulate(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) {
; AVX512VL-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX512VL-VNNI: # %bb.0:
-; AVX512VL-VNNI-NEXT: vpdpwssd %xmm1, %xmm0, %xmm2
-; AVX512VL-VNNI-NEXT: vmovdqa %xmm2, %xmm0
+; AVX512VL-VNNI-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512VL-VNNI-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512VL-VNNI-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX512VL-VNNI-NEXT: vpmovqd %ymm0, %xmm1
+; AVX512VL-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512VL-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
+; AVX512VL-VNNI-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512VL-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512VL-VNNI-NEXT: vzeroupper
; AVX512VL-VNNI-NEXT: retq
;
; AVX-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX-VNNI: # %bb.0:
-; AVX-VNNI-NEXT: {vex} vpdpwssd %xmm1, %xmm0, %xmm2
-; AVX-VNNI-NEXT: vmovdqa %xmm2, %xmm0
+; AVX-VNNI-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX-VNNI-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX-VNNI-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX-VNNI-NEXT: vpmovqd %ymm0, %xmm1
+; AVX-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
+; AVX-VNNI-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX-VNNI-NEXT: vzeroupper
; AVX-VNNI-NEXT: retq
;
; AVX512-VNNI-LABEL: vpdpwssd_v4i32_accumulate:
; AVX512-VNNI: # %bb.0:
-; AVX512-VNNI-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0
-; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-VNNI-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512-VNNI-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512-VNNI-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX512-VNNI-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} xmm3 = xmm0[0,2],xmm1[0,2]
+; AVX512-VNNI-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
+; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm3, %xmm1
+; AVX512-VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-VNNI-NEXT: vzeroupper
; AVX512-VNNI-NEXT: retq
%x0 = sext <8 x i16> %a0 to <8 x i32>
%x1 = sext <8 x i16> %a1 to <8 x i32>
diff --git a/llvm/test/CodeGen/X86/vselect-avx.ll b/llvm/test/CodeGen/X86/vselect-avx.ll
index b684bb015ecd0..469a05a90f970 100644
--- a/llvm/test/CodeGen/X86/vselect-avx.ll
+++ b/llvm/test/CodeGen/X86/vselect-avx.ll
@@ -263,11 +263,11 @@ define <4 x i64> @vselect_concat_split_v16i8(<4 x i64> %a, <4 x i64> %b, <4 x i6
; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5
; AVX1-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
; AVX1-NEXT: vpcmpgtb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT: vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: vselect_concat_split_v16i8:
More information about the llvm-commits
mailing list