[llvm] [DAG] expandVecReduce - widen sub-legal vectors to not prematurely scalarize later reduction levels (PR #194672)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 06:30:56 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/194672
>From 4684729c5167f95e99437849286f0054da0cc918 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Tue, 28 Apr 2026 17:24:41 +0100
Subject: [PATCH] [DAG] expandVecReduce - widen sub-legal vectors to not
prematurely scalarize later reduction levels
When repeatedly splitting the pow2 vector source, we currently begin to scalarize as soon as drop below the legal vector op type.
This patch attempts to widen the source vectors back to legal op types to avoid additional scalarization or vector element extractions.
Fixes #194655
---
.../CodeGen/SelectionDAG/TargetLowering.cpp | 39 +-
llvm/test/CodeGen/AArch64/aarch64-mulv.ll | 78 ++--
llvm/test/CodeGen/AArch64/double_reduct.ll | 25 +-
llvm/test/CodeGen/AArch64/vecreduce-fmul.ll | 64 +--
llvm/test/CodeGen/PowerPC/cttz-elts.ll | 80 ++--
llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll | 21 +-
llvm/test/CodeGen/X86/vector-compress.ll | 432 +++++++-----------
.../CodeGen/X86/vector-extract-last-active.ll | 102 +++--
8 files changed, 406 insertions(+), 435 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 6e4c385c72edb..ac7a83f3ae998 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12240,16 +12240,51 @@ bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
- unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
+ ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
SDValue Op = Node->getOperand(0);
EVT VT = Op.getValueType();
// Try to use a shuffle reduction for power of two vectors.
if (VT.isPow2VectorType()) {
+ // See if the reduction opcode is safe to use with widened types.
+ bool WidenSrc = false;
+ switch (Node->getOpcode()) {
+ case ISD::VECREDUCE_FADD:
+ case ISD::VECREDUCE_FMUL:
+ case ISD::VECREDUCE_ADD:
+ case ISD::VECREDUCE_MUL:
+ case ISD::VECREDUCE_AND:
+ case ISD::VECREDUCE_OR:
+ case ISD::VECREDUCE_XOR:
+ case ISD::VECREDUCE_SMAX:
+ case ISD::VECREDUCE_SMIN:
+ case ISD::VECREDUCE_UMAX:
+ case ISD::VECREDUCE_UMIN:
+ WidenSrc = VT.isFixedLengthVector();
+ break;
+ }
+
while (VT.getVectorElementCount().isKnownMultipleOf(2)) {
EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
- if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))
+ if (!isOperationLegalOrCustom(BaseOpcode, HalfVT)) {
+ if (WidenSrc && Op.getOpcode() != ISD::BUILD_VECTOR) {
+ // Attempt to widen the source vectors to a legal op.
+ EVT WideVT = getTypeToTransformTo(*DAG.getContext(), HalfVT);
+ if (WideVT.isVector() &&
+ WideVT.getScalarType() == HalfVT.getScalarType() &&
+ isOperationLegalOrCustom(BaseOpcode, WideVT)) {
+ SDValue Lo, Hi;
+ std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
+ Lo = DAG.getInsertSubvector(dl, DAG.getPOISON(WideVT), Lo, 0);
+ Hi = DAG.getInsertSubvector(dl, DAG.getPOISON(WideVT), Hi, 0);
+ Op = DAG.getNode(BaseOpcode, dl, WideVT, Lo, Hi, Node->getFlags());
+ Op = DAG.getExtractSubvector(dl, HalfVT, Op, 0);
+ VT = HalfVT;
+ continue;
+ }
+ }
break;
+ }
SDValue Lo, Hi;
std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
index fecc6554667fa..954e8e422b55f 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
@@ -25,13 +25,20 @@ declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)
declare i128 @llvm.vector.reduce.mul.v2i128(<2 x i128>)
define i8 @mulv_v2i8(<2 x i8> %a) {
-; CHECK-LABEL: mulv_v2i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov w8, v0.s[1]
-; CHECK-NEXT: fmov w9, s0
-; CHECK-NEXT: mul w0, w9, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: mulv_v2i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: mulv_v2i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: fmov w9, s0
+; CHECK-GI-NEXT: mul w0, w9, w8
+; CHECK-GI-NEXT: ret
entry:
%arg1 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %a)
ret i8 %arg1
@@ -221,13 +228,20 @@ entry:
}
define i16 @mulv_v2i16(<2 x i16> %a) {
-; CHECK-LABEL: mulv_v2i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov w8, v0.s[1]
-; CHECK-NEXT: fmov w9, s0
-; CHECK-NEXT: mul w0, w9, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: mulv_v2i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: mulv_v2i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: fmov w9, s0
+; CHECK-GI-NEXT: mul w0, w9, w8
+; CHECK-GI-NEXT: ret
entry:
%arg1 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %a)
ret i16 %arg1
@@ -354,13 +368,20 @@ entry:
}
define i32 @mulv_v2i32(<2 x i32> %a) {
-; CHECK-LABEL: mulv_v2i32:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov w8, v0.s[1]
-; CHECK-NEXT: fmov w9, s0
-; CHECK-NEXT: mul w0, w9, w8
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: mulv_v2i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: mulv_v2i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: mov w8, v0.s[1]
+; CHECK-GI-NEXT: fmov w9, s0
+; CHECK-GI-NEXT: mul w0, w9, w8
+; CHECK-GI-NEXT: ret
entry:
%arg1 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %a)
ret i32 %arg1
@@ -374,9 +395,8 @@ define i32 @mulv_v3i32(<3 x i32> %a) {
; CHECK-NEXT: mov v1.s[3], w8
; CHECK-NEXT: ext v1.16b, v1.16b, v1.16b, #8
; CHECK-NEXT: mul v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: mov w8, v0.s[1]
-; CHECK-NEXT: fmov w9, s0
-; CHECK-NEXT: mul w0, w9, w8
+; CHECK-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
%arg1 = call i32 @llvm.vector.reduce.mul.v3i32(<3 x i32> %a)
@@ -388,9 +408,8 @@ define i32 @mulv_v4i32(<4 x i32> %a) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: mul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: mov w8, v0.s[1]
-; CHECK-SD-NEXT: fmov w9, s0
-; CHECK-SD-NEXT: mul w0, w9, w8
+; CHECK-SD-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: mulv_v4i32:
@@ -412,9 +431,8 @@ define i32 @mulv_v8i32(<8 x i32> %a) {
; CHECK-SD-NEXT: mul v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: mul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: mov w8, v0.s[1]
-; CHECK-SD-NEXT: fmov w9, s0
-; CHECK-SD-NEXT: mul w0, w9, w8
+; CHECK-SD-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: mulv_v8i32:
diff --git a/llvm/test/CodeGen/AArch64/double_reduct.ll b/llvm/test/CodeGen/AArch64/double_reduct.ll
index e231b91f1199b..3172735481d70 100644
--- a/llvm/test/CodeGen/AArch64/double_reduct.ll
+++ b/llvm/test/CodeGen/AArch64/double_reduct.ll
@@ -55,7 +55,8 @@ define float @fmul_f32(<8 x float> %a, <4 x float> %b) {
; CHECK-SD-NEXT: fmul v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: fmul_f32:
@@ -83,7 +84,8 @@ define float @fmul_f32_same(<4 x float> %a, <4 x float> %b) {
; CHECK-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: fmul_f32_same:
@@ -378,9 +380,8 @@ define i32 @mul_i32(<8 x i32> %a, <4 x i32> %b) {
; CHECK-SD-NEXT: mul v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: mul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: mov w8, v0.s[1]
-; CHECK-SD-NEXT: fmov w9, s0
-; CHECK-SD-NEXT: mul w0, w9, w8
+; CHECK-SD-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: mul_i32:
@@ -412,9 +413,8 @@ define i32 @mul_i32_same(<4 x i32> %a, <4 x i32> %b) {
; CHECK-SD-NEXT: mul v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: mul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: mov w8, v0.s[1]
-; CHECK-SD-NEXT: fmov w9, s0
-; CHECK-SD-NEXT: mul w0, w9, w8
+; CHECK-SD-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmov w0, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: mul_i32_same:
@@ -907,8 +907,8 @@ define float @nested_mul_f32(<4 x float> %a, <4 x float> %b, float %c, float %d)
; CHECK-SD-NEXT: fmul s2, s2, s3
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
-; CHECK-SD-NEXT: fmul s0, s0, s2
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmul s0, s2, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: nested_mul_f32:
@@ -1058,9 +1058,8 @@ define i32 @nested_mul_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
; CHECK-SD-NEXT: mul w8, w0, w1
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: mul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: mov w9, v0.s[1]
-; CHECK-SD-NEXT: fmov w10, s0
-; CHECK-SD-NEXT: mul w9, w10, w9
+; CHECK-SD-NEXT: mul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmov w9, s0
; CHECK-SD-NEXT: mul w0, w9, w8
; CHECK-SD-NEXT: ret
;
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
index f4825f065d316..dd633ca379941 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
@@ -8,7 +8,8 @@ define float @mul_HalfS(<2 x float> %bin.rdx) {
; CHECK-SD-LABEL: mul_HalfS:
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: mul_HalfS:
@@ -41,9 +42,9 @@ define half @mul_HalfH(<4 x half> %bin.rdx) {
; CHECK-SD-FP16-LABEL: mul_HalfH:
; CHECK-SD-FP16: // %bb.0:
; CHECK-SD-FP16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT: fmul h1, h0, v0.h[1]
-; CHECK-SD-FP16-NEXT: fmul h1, h1, v0.h[2]
-; CHECK-SD-FP16-NEXT: fmul h0, h1, v0.h[3]
+; CHECK-SD-FP16-NEXT: dup v1.2s, v0.s[1]
+; CHECK-SD-FP16-NEXT: fmul v0.4h, v0.4h, v1.4h
+; CHECK-SD-FP16-NEXT: fmul h0, h0, v0.h[1]
; CHECK-SD-FP16-NEXT: ret
;
; CHECK-GI-NOFP16-LABEL: mul_HalfH:
@@ -79,7 +80,7 @@ define half @mul_H(<8 x half> %bin.rdx) {
; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-SD-NOFP16-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NOFP16-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NOFP16-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NOFP16-NEXT: fmul v0.2s, v0.2s, v0.s[1]
; CHECK-SD-NOFP16-NEXT: fcvt h0, s0
; CHECK-SD-NOFP16-NEXT: ret
;
@@ -87,9 +88,9 @@ define half @mul_H(<8 x half> %bin.rdx) {
; CHECK-SD-FP16: // %bb.0:
; CHECK-SD-FP16-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-FP16-NEXT: fmul v0.4h, v0.4h, v1.4h
-; CHECK-SD-FP16-NEXT: fmul h1, h0, v0.h[1]
-; CHECK-SD-FP16-NEXT: fmul h1, h1, v0.h[2]
-; CHECK-SD-FP16-NEXT: fmul h0, h1, v0.h[3]
+; CHECK-SD-FP16-NEXT: dup v1.2s, v0.s[1]
+; CHECK-SD-FP16-NEXT: fmul v0.4h, v0.4h, v1.4h
+; CHECK-SD-FP16-NEXT: fmul h0, h0, v0.h[1]
; CHECK-SD-FP16-NEXT: ret
;
; CHECK-GI-NOFP16-LABEL: mul_H:
@@ -124,7 +125,8 @@ define float @mul_S(<4 x float> %bin.rdx) {
; CHECK-SD: // %bb.0:
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: mul_S:
@@ -163,7 +165,7 @@ define half @mul_2H(<16 x half> %bin.rdx) {
; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-SD-NOFP16-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NOFP16-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NOFP16-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NOFP16-NEXT: fmul v0.2s, v0.2s, v0.s[1]
; CHECK-SD-NOFP16-NEXT: fcvt h0, s0
; CHECK-SD-NOFP16-NEXT: ret
;
@@ -172,9 +174,9 @@ define half @mul_2H(<16 x half> %bin.rdx) {
; CHECK-SD-FP16-NEXT: fmul v0.8h, v0.8h, v1.8h
; CHECK-SD-FP16-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-FP16-NEXT: fmul v0.4h, v0.4h, v1.4h
-; CHECK-SD-FP16-NEXT: fmul h1, h0, v0.h[1]
-; CHECK-SD-FP16-NEXT: fmul h1, h1, v0.h[2]
-; CHECK-SD-FP16-NEXT: fmul h0, h1, v0.h[3]
+; CHECK-SD-FP16-NEXT: dup v1.2s, v0.s[1]
+; CHECK-SD-FP16-NEXT: fmul v0.4h, v0.4h, v1.4h
+; CHECK-SD-FP16-NEXT: fmul h0, h0, v0.h[1]
; CHECK-SD-FP16-NEXT: ret
;
; CHECK-GI-NOFP16-LABEL: mul_2H:
@@ -215,7 +217,8 @@ define float @mul_2S(<8 x float> %bin.rdx) {
; CHECK-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: mul_2S:
@@ -248,8 +251,8 @@ define float @mul_S_init_42(<4 x float> %bin.rdx) {
; CHECK-SD-NEXT: mov w8, #1109917696 // =0x42280000
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
; CHECK-SD-NEXT: fmov s1, w8
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
-; CHECK-SD-NEXT: fmul s0, s0, s1
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmul s0, s1, s0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: mul_S_init_42:
@@ -280,8 +283,8 @@ define half @fmul_reduct_reassoc_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-SD-NOFP16-NEXT: ext v3.16b, v1.16b, v1.16b, #8
; CHECK-SD-NOFP16-NEXT: fmul v0.2s, v0.2s, v2.2s
; CHECK-SD-NOFP16-NEXT: fmul v1.2s, v1.2s, v3.2s
-; CHECK-SD-NOFP16-NEXT: fmul s0, s0, v0.s[1]
-; CHECK-SD-NOFP16-NEXT: fmul s1, s1, v1.s[1]
+; CHECK-SD-NOFP16-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NOFP16-NEXT: fmul v1.2s, v1.2s, v1.s[1]
; CHECK-SD-NOFP16-NEXT: fcvt h0, s0
; CHECK-SD-NOFP16-NEXT: fcvt h1, s1
; CHECK-SD-NOFP16-NEXT: fcvt s0, h0
@@ -295,9 +298,9 @@ define half @fmul_reduct_reassoc_v8f16(<8 x half> %a, <8 x half> %b) {
; CHECK-SD-FP16-NEXT: fmul v0.8h, v0.8h, v1.8h
; CHECK-SD-FP16-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-FP16-NEXT: fmul v0.4h, v0.4h, v1.4h
-; CHECK-SD-FP16-NEXT: fmul h1, h0, v0.h[1]
-; CHECK-SD-FP16-NEXT: fmul h1, h1, v0.h[2]
-; CHECK-SD-FP16-NEXT: fmul h0, h1, v0.h[3]
+; CHECK-SD-FP16-NEXT: dup v1.2s, v0.s[1]
+; CHECK-SD-FP16-NEXT: fmul v0.4h, v0.4h, v1.4h
+; CHECK-SD-FP16-NEXT: fmul h0, h0, v0.h[1]
; CHECK-SD-FP16-NEXT: ret
;
; CHECK-GI-NOFP16-LABEL: fmul_reduct_reassoc_v8f16:
@@ -358,7 +361,8 @@ define float @fmul_reduct_reassoc_v8f32(<8 x float> %a, <8 x float> %b) {
; CHECK-SD-NEXT: fmul v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: fmul_reduct_reassoc_v8f32:
@@ -387,7 +391,8 @@ define float @fmul_reduct_reassoc_v4f32(<4 x float> %a, <4 x float> %b) {
; CHECK-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: fmul_reduct_reassoc_v4f32:
@@ -414,10 +419,10 @@ define float @fmul_reduct_reassoc_v4f32_init(float %i, <4 x float> %a, <4 x floa
; CHECK-SD-NEXT: ext v3.16b, v1.16b, v1.16b, #8
; CHECK-SD-NEXT: fmul v1.2s, v1.2s, v3.2s
; CHECK-SD-NEXT: ext v3.16b, v2.16b, v2.16b, #8
-; CHECK-SD-NEXT: fmul s1, s1, v1.s[1]
+; CHECK-SD-NEXT: fmul v1.2s, v1.2s, v1.s[1]
; CHECK-SD-NEXT: fmul v2.2s, v2.2s, v3.2s
; CHECK-SD-NEXT: fmul s0, s0, s1
-; CHECK-SD-NEXT: fmul s1, s2, v2.s[1]
+; CHECK-SD-NEXT: fmul v1.2s, v2.2s, v2.s[1]
; CHECK-SD-NEXT: fmul s0, s0, s1
; CHECK-SD-NEXT: ret
;
@@ -447,7 +452,8 @@ define float @fmul_reduct_reassoc_v4v8f32(<4 x float> %a, <8 x float> %b) {
; CHECK-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: fmul_reduct_reassoc_v4v8f32:
@@ -499,9 +505,9 @@ define float @fmul_reduct_reassoc_v4f32_extrause(<4 x float> %a, <4 x float> %b)
; CHECK-SD-NEXT: ext v3.16b, v1.16b, v1.16b, #8
; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v2.2s
; CHECK-SD-NEXT: fmul v1.2s, v1.2s, v3.2s
-; CHECK-SD-NEXT: fmul s0, s0, v0.s[1]
-; CHECK-SD-NEXT: fmul s1, s1, v1.s[1]
-; CHECK-SD-NEXT: fmul s1, s0, s1
+; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v0.s[1]
+; CHECK-SD-NEXT: fmul v1.2s, v1.2s, v1.s[1]
+; CHECK-SD-NEXT: fmul s1, s0, v1.s[0]
; CHECK-SD-NEXT: fmul s0, s1, s0
; CHECK-SD-NEXT: ret
;
diff --git a/llvm/test/CodeGen/PowerPC/cttz-elts.ll b/llvm/test/CodeGen/PowerPC/cttz-elts.ll
index 301466657487f..d84ab409f3d9f 100644
--- a/llvm/test/CodeGen/PowerPC/cttz-elts.ll
+++ b/llvm/test/CodeGen/PowerPC/cttz-elts.ll
@@ -11,20 +11,16 @@ define i32 @v4i1(<4 x i1> %x) {
; CHECK-NEXT: vsraw 2, 2, 4
; CHECK-NEXT: lxvd2x 0, 0, 3
; CHECK-NEXT: xxswapd 35, 0
-; CHECK-NEXT: xxland 0, 34, 35
-; CHECK-NEXT: xxsldwi 1, 0, 0, 1
-; CHECK-NEXT: xxswapd 2, 0
-; CHECK-NEXT: xxsldwi 3, 0, 0, 3
-; CHECK-NEXT: mffprwz 3, 1
-; CHECK-NEXT: mffprwz 4, 2
+; CHECK-NEXT: xxland 34, 34, 35
+; CHECK-NEXT: xxmrghw 0, 34, 34
+; CHECK-NEXT: xxmrghw 35, 0, 34
+; CHECK-NEXT: vmaxuw 2, 2, 3
+; CHECK-NEXT: xxsldwi 0, 34, 34, 1
+; CHECK-NEXT: xxswapd 1, 34
+; CHECK-NEXT: mffprwz 3, 0
+; CHECK-NEXT: mffprwz 4, 1
; CHECK-NEXT: cmplw 4, 3
; CHECK-NEXT: iselgt 3, 4, 3
-; CHECK-NEXT: mffprwz 4, 0
-; CHECK-NEXT: cmplw 3, 4
-; CHECK-NEXT: iselgt 3, 3, 4
-; CHECK-NEXT: mffprwz 4, 3
-; CHECK-NEXT: cmplw 3, 4
-; CHECK-NEXT: iselgt 3, 3, 4
; CHECK-NEXT: subfic 3, 3, 4
; CHECK-NEXT: blr
%y = call i32 @llvm.experimental.cttz.elts(<4 x i1> %x, i1 false)
@@ -40,20 +36,16 @@ define i32 @v4i32(<4 x i32> %x) {
; CHECK-NEXT: vcmpgtuw 2, 2, 4
; CHECK-NEXT: lxvd2x 0, 0, 3
; CHECK-NEXT: xxswapd 35, 0
-; CHECK-NEXT: xxland 0, 34, 35
-; CHECK-NEXT: xxsldwi 1, 0, 0, 1
-; CHECK-NEXT: xxswapd 2, 0
-; CHECK-NEXT: xxsldwi 3, 0, 0, 3
-; CHECK-NEXT: mffprwz 3, 1
-; CHECK-NEXT: mffprwz 4, 2
+; CHECK-NEXT: xxland 34, 34, 35
+; CHECK-NEXT: xxmrghw 0, 34, 34
+; CHECK-NEXT: xxmrghw 35, 0, 34
+; CHECK-NEXT: vmaxuw 2, 2, 3
+; CHECK-NEXT: xxsldwi 0, 34, 34, 1
+; CHECK-NEXT: xxswapd 1, 34
+; CHECK-NEXT: mffprwz 3, 0
+; CHECK-NEXT: mffprwz 4, 1
; CHECK-NEXT: cmplw 4, 3
; CHECK-NEXT: iselgt 3, 4, 3
-; CHECK-NEXT: mffprwz 4, 0
-; CHECK-NEXT: cmplw 3, 4
-; CHECK-NEXT: iselgt 3, 3, 4
-; CHECK-NEXT: mffprwz 4, 3
-; CHECK-NEXT: cmplw 3, 4
-; CHECK-NEXT: iselgt 3, 3, 4
; CHECK-NEXT: subfic 3, 3, 4
; CHECK-NEXT: blr
%y = call i32 @llvm.experimental.cttz.elts(<4 x i32> %x, i1 false)
@@ -70,20 +62,16 @@ define i32 @v4i1_zero_is_poison(<4 x i1> %x) {
; CHECK-NEXT: vsraw 2, 2, 4
; CHECK-NEXT: lxvd2x 0, 0, 3
; CHECK-NEXT: xxswapd 35, 0
-; CHECK-NEXT: xxland 0, 34, 35
-; CHECK-NEXT: xxsldwi 1, 0, 0, 1
-; CHECK-NEXT: xxswapd 2, 0
-; CHECK-NEXT: xxsldwi 3, 0, 0, 3
-; CHECK-NEXT: mffprwz 3, 1
-; CHECK-NEXT: mffprwz 4, 2
+; CHECK-NEXT: xxland 34, 34, 35
+; CHECK-NEXT: xxmrghw 0, 34, 34
+; CHECK-NEXT: xxmrghw 35, 0, 34
+; CHECK-NEXT: vmaxuw 2, 2, 3
+; CHECK-NEXT: xxsldwi 0, 34, 34, 1
+; CHECK-NEXT: xxswapd 1, 34
+; CHECK-NEXT: mffprwz 3, 0
+; CHECK-NEXT: mffprwz 4, 1
; CHECK-NEXT: cmplw 4, 3
; CHECK-NEXT: iselgt 3, 4, 3
-; CHECK-NEXT: mffprwz 4, 0
-; CHECK-NEXT: cmplw 3, 4
-; CHECK-NEXT: iselgt 3, 3, 4
-; CHECK-NEXT: mffprwz 4, 3
-; CHECK-NEXT: cmplw 3, 4
-; CHECK-NEXT: iselgt 3, 3, 4
; CHECK-NEXT: subfic 3, 3, 4
; CHECK-NEXT: blr
%y = call i32 @llvm.experimental.cttz.elts(<4 x i1> %x, i1 true)
@@ -99,20 +87,16 @@ define i32 @v4i32_zero_is_poison(<4 x i32> %x) {
; CHECK-NEXT: vcmpgtuw 2, 2, 4
; CHECK-NEXT: lxvd2x 0, 0, 3
; CHECK-NEXT: xxswapd 35, 0
-; CHECK-NEXT: xxland 0, 34, 35
-; CHECK-NEXT: xxsldwi 1, 0, 0, 1
-; CHECK-NEXT: xxswapd 2, 0
-; CHECK-NEXT: xxsldwi 3, 0, 0, 3
-; CHECK-NEXT: mffprwz 3, 1
-; CHECK-NEXT: mffprwz 4, 2
+; CHECK-NEXT: xxland 34, 34, 35
+; CHECK-NEXT: xxmrghw 0, 34, 34
+; CHECK-NEXT: xxmrghw 35, 0, 34
+; CHECK-NEXT: vmaxuw 2, 2, 3
+; CHECK-NEXT: xxsldwi 0, 34, 34, 1
+; CHECK-NEXT: xxswapd 1, 34
+; CHECK-NEXT: mffprwz 3, 0
+; CHECK-NEXT: mffprwz 4, 1
; CHECK-NEXT: cmplw 4, 3
; CHECK-NEXT: iselgt 3, 4, 3
-; CHECK-NEXT: mffprwz 4, 0
-; CHECK-NEXT: cmplw 3, 4
-; CHECK-NEXT: iselgt 3, 3, 4
-; CHECK-NEXT: mffprwz 4, 3
-; CHECK-NEXT: cmplw 3, 4
-; CHECK-NEXT: iselgt 3, 3, 4
; CHECK-NEXT: subfic 3, 3, 4
; CHECK-NEXT: blr
%y = call i32 @llvm.experimental.cttz.elts(<4 x i32> %x, i1 true)
diff --git a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
index 61191fd2637e8..8159468722596 100644
--- a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
+++ b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
@@ -45,19 +45,20 @@ define i16 @ctz_v4i32(<4 x i32> %a) {
; CHECK-NEXT: pxor %xmm1, %xmm1
; CHECK-NEXT: pcmpeqd %xmm0, %xmm1
; CHECK-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; CHECK-NEXT: movdqa %xmm1, %xmm2
+; CHECK-NEXT: por %xmm0, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; CHECK-NEXT: por %xmm3, %xmm0
+; CHECK-NEXT: pcmpgtd %xmm0, %xmm2
+; CHECK-NEXT: pand %xmm2, %xmm1
+; CHECK-NEXT: pandn %xmm3, %xmm2
+; CHECK-NEXT: por %xmm1, %xmm2
+; CHECK-NEXT: movd %xmm2, %eax
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
; CHECK-NEXT: movd %xmm0, %ecx
; CHECK-NEXT: cmpl %ecx, %eax
; CHECK-NEXT: cmoval %eax, %ecx
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: cmpl %eax, %ecx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: cmpl %eax, %ecx
-; CHECK-NEXT: cmovbel %eax, %ecx
; CHECK-NEXT: movl $4, %eax
; CHECK-NEXT: subl %ecx, %eax
; CHECK-NEXT: # kill: def $ax killed $ax killed $eax
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 01bdf0a098e7a..1839eefcd264e 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -253,12 +253,10 @@ define <8 x i32> @test_compress_v8i32(<8 x i32> %vec, <8 x i1> %mask, <8 x i32>
; AVX2-NEXT: vpsrld $31, %ymm2, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm2, %edx
-; AVX2-NEXT: vpextrd $3, %xmm2, %eax
-; AVX2-NEXT: addl %edx, %eax
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpextrd $1, %xmm2, %ecx
+; AVX2-NEXT: vmovd %xmm2, %eax
; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
@@ -351,14 +349,12 @@ define <8 x float> @test_compress_v8f32(<8 x float> %vec, <8 x i1> %mask, <8 x f
; AVX2-NEXT: vpsrld $31, %ymm2, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
; AVX2-NEXT: vpextrd $1, %xmm2, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm2, %eax
-; AVX2-NEXT: vpextrd $3, %xmm2, %edx
-; AVX2-NEXT: addl %eax, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: andl $7, %edx
+; AVX2-NEXT: andl $7, %ecx
; AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, (%rsp)
; AVX2-NEXT: vmovd %xmm3, %eax
@@ -596,9 +592,9 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: andq $-32, %rsp
-; AVX2-NEXT: subq $128, %rsp
+; AVX2-NEXT: subq $160, %rsp
; AVX2-NEXT: vmovaps %ymm4, {{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps %ymm3, (%rsp)
+; AVX2-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3
; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
@@ -606,15 +602,13 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
; AVX2-NEXT: vpextrd $1, %xmm3, %eax
; AVX2-NEXT: vmovd %xmm3, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm3, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %edx
-; AVX2-NEXT: addl %eax, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: andl $15, %edx
-; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
@@ -657,59 +651,60 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpextrb $12, %xmm2, %r13d
; AVX2-NEXT: andl $1, %r13d
; AVX2-NEXT: addq %r12, %r13
-; AVX2-NEXT: vpextrb $13, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %r13, %rcx
-; AVX2-NEXT: vpextrb $14, %xmm2, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: vpextrb $15, %xmm2, %edx
+; AVX2-NEXT: vpextrb $13, %xmm2, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rax, %rdx
+; AVX2-NEXT: addq %r13, %rdx
+; AVX2-NEXT: vpextrb $14, %xmm2, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrb $15, %xmm2, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rcx, %rdi
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX2-NEXT: cmpq $16, %rdx
-; AVX2-NEXT: vextractps $3, %xmm2, %esi
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; AVX2-NEXT: cmovbl (%rsp,%rdi,4), %esi
-; AVX2-NEXT: movl %esi, %edi
-; AVX2-NEXT: vmovss %xmm0, (%rsp)
+; AVX2-NEXT: cmpq $16, %rdi
+; AVX2-NEXT: vextractps $3, %xmm2, %eax
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rsi,4)
+; AVX2-NEXT: cmovbl 32(%rsp,%rsi,4), %eax
+; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX2-NEXT: vmovss %xmm0, {{[0-9]+}}(%rsp)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rsi,4)
+; AVX2-NEXT: vextractps $1, %xmm0, 32(%rsp,%rsi,4)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rsi,4)
-; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%rsi,4)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: vmovss %xmm0, (%rsp,%rsi,4)
+; AVX2-NEXT: vextractps $3, %xmm0, 32(%rsp,%rsi,4)
+; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT: vmovss %xmm0, 32(%rsp,%rax,4)
; AVX2-NEXT: andl $15, %r8d
-; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%r8,4)
+; AVX2-NEXT: vextractps $1, %xmm0, 32(%rsp,%r8,4)
; AVX2-NEXT: andl $15, %r9d
-; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%r9,4)
+; AVX2-NEXT: vextractps $2, %xmm0, 32(%rsp,%r9,4)
; AVX2-NEXT: andl $15, %r10d
-; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%r10,4)
+; AVX2-NEXT: vextractps $3, %xmm0, 32(%rsp,%r10,4)
; AVX2-NEXT: andl $15, %r11d
-; AVX2-NEXT: vmovss %xmm1, (%rsp,%r11,4)
+; AVX2-NEXT: vmovss %xmm1, 32(%rsp,%r11,4)
; AVX2-NEXT: andl $15, %ebx
-; AVX2-NEXT: vextractps $1, %xmm1, (%rsp,%rbx,4)
+; AVX2-NEXT: vextractps $1, %xmm1, 32(%rsp,%rbx,4)
; AVX2-NEXT: andl $15, %r14d
-; AVX2-NEXT: vextractps $2, %xmm1, (%rsp,%r14,4)
+; AVX2-NEXT: vextractps $2, %xmm1, 32(%rsp,%r14,4)
; AVX2-NEXT: andl $15, %r15d
-; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%r15,4)
+; AVX2-NEXT: vextractps $3, %xmm1, 32(%rsp,%r15,4)
; AVX2-NEXT: andl $15, %r12d
-; AVX2-NEXT: vmovss %xmm2, (%rsp,%r12,4)
+; AVX2-NEXT: vmovss %xmm2, 32(%rsp,%r12,4)
; AVX2-NEXT: andl $15, %r13d
-; AVX2-NEXT: vextractps $1, %xmm2, (%rsp,%r13,4)
+; AVX2-NEXT: vextractps $1, %xmm2, 32(%rsp,%r13,4)
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vextractps $2, %xmm2, 32(%rsp,%rdx,4)
; AVX2-NEXT: andl $15, %ecx
-; AVX2-NEXT: vextractps $2, %xmm2, (%rsp,%rcx,4)
-; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vextractps $3, %xmm2, (%rsp,%rax,4)
-; AVX2-NEXT: cmpq $15, %rdx
+; AVX2-NEXT: vextractps $3, %xmm2, 32(%rsp,%rcx,4)
+; AVX2-NEXT: cmpq $15, %rdi
; AVX2-NEXT: movl $15, %eax
-; AVX2-NEXT: cmovbq %rdx, %rax
+; AVX2-NEXT: cmovbq %rdi, %rax
; AVX2-NEXT: movl %eax, %eax
-; AVX2-NEXT: movl %edi, (%rsp,%rax,4)
-; AVX2-NEXT: vmovaps (%rsp), %ymm0
+; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
+; AVX2-NEXT: movl %ecx, 32(%rsp,%rax,4)
+; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm0
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1
; AVX2-NEXT: leaq -40(%rbp), %rsp
; AVX2-NEXT: popq %rbx
@@ -756,14 +751,12 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
; AVX2-NEXT: vpextrd $1, %xmm3, %eax
; AVX2-NEXT: vmovd %xmm3, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrd $2, %xmm3, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %edx
-; AVX2-NEXT: addl %eax, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, (%rsp)
; AVX2-NEXT: vmovd %xmm2, %eax
@@ -1342,38 +1335,16 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm4
; AVX2-NEXT: vpand %xmm2, %xmm3, %xmm2
; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm2[8,9,10,11,12,13,14,15,12,13],zero,zero,xmm2[14,15],zero,zero
+; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
+; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpsrld $16, %xmm2, %xmm4
+; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: vpextrb $2, %xmm2, %eax
-; AVX2-NEXT: vpextrb $3, %xmm2, %edx
-; AVX2-NEXT: addb %al, %dl
-; AVX2-NEXT: addb %cl, %dl
-; AVX2-NEXT: vpextrb $4, %xmm2, %eax
-; AVX2-NEXT: vpextrb $5, %xmm2, %ecx
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: vpextrb $6, %xmm2, %eax
-; AVX2-NEXT: addb %cl, %al
-; AVX2-NEXT: addb %dl, %al
-; AVX2-NEXT: vpextrb $7, %xmm2, %ecx
-; AVX2-NEXT: vpextrb $8, %xmm2, %edx
-; AVX2-NEXT: addb %cl, %dl
-; AVX2-NEXT: vpextrb $9, %xmm2, %ecx
-; AVX2-NEXT: addb %dl, %cl
-; AVX2-NEXT: vpextrb $10, %xmm2, %edx
-; AVX2-NEXT: addb %cl, %dl
-; AVX2-NEXT: addb %al, %dl
-; AVX2-NEXT: vpextrb $11, %xmm2, %eax
-; AVX2-NEXT: vpextrb $12, %xmm2, %ecx
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: vpextrb $13, %xmm2, %eax
-; AVX2-NEXT: addb %cl, %al
-; AVX2-NEXT: vpextrb $14, %xmm2, %ecx
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: vpextrb $15, %xmm2, %eax
-; AVX2-NEXT: addb %cl, %al
-; AVX2-NEXT: addb %dl, %al
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movzbl %cl, %eax
; AVX2-NEXT: andl $31, %eax
; AVX2-NEXT: movzbl (%rsp,%rax), %eax
; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)
@@ -1556,12 +1527,13 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $31, %ecx
; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rcx)
+; AVX2-NEXT: vpextrb $15, %xmm0, %ecx
+; AVX2-NEXT: cmpq $32, %rdx
+; AVX2-NEXT: cmovbl %eax, %ecx
; AVX2-NEXT: cmpq $31, %rdx
-; AVX2-NEXT: movl $31, %ecx
-; AVX2-NEXT: cmovbq %rdx, %rcx
-; AVX2-NEXT: vpextrb $15, %xmm0, %edx
-; AVX2-NEXT: cmovbel %eax, %edx
-; AVX2-NEXT: movb %dl, (%rsp,%rcx)
+; AVX2-NEXT: movl $31, %eax
+; AVX2-NEXT: cmovbq %rdx, %rax
+; AVX2-NEXT: movb %cl, (%rsp,%rax)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: movq %rbp, %rsp
; AVX2-NEXT: popq %rbp
@@ -1589,18 +1561,16 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512F-NEXT: vpaddd %ymm4, %ymm3, %ymm3
; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
; AVX512F-NEXT: vmovd %xmm3, %ecx
; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm3, %eax
-; AVX512F-NEXT: vpextrd $3, %xmm3, %edx
-; AVX512F-NEXT: addl %eax, %edx
-; AVX512F-NEXT: addl %ecx, %edx
-; AVX512F-NEXT: andl $31, %edx
+; AVX512F-NEXT: andl $31, %ecx
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdb %zmm0, (%rsp,%rdx)
+; AVX512F-NEXT: vpmovdb %zmm0, (%rsp,%rcx)
; AVX512F-NEXT: vpsllw $7, %ymm1, %ymm0
; AVX512F-NEXT: vpblendvb %ymm0, (%rsp), %ymm2, %ymm0
; AVX512F-NEXT: movq %rbp, %rsp
@@ -1637,21 +1607,13 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpsrlw $15, %ymm3, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpextrw $1, %xmm2, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm2, %eax
-; AVX2-NEXT: vpextrw $3, %xmm2, %edx
-; AVX2-NEXT: addl %eax, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: vpextrw $4, %xmm2, %eax
-; AVX2-NEXT: vpextrw $5, %xmm2, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrw $6, %xmm2, %eax
-; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: addl %edx, %eax
-; AVX2-NEXT: vpextrw $7, %xmm2, %ecx
-; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrw $1, %xmm1, %eax
@@ -1793,10 +1755,10 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: andq $-32, %rsp
; AVX2-NEXT: subq $96, %rsp
-; AVX2-NEXT: movl %r9d, %r11d
-; AVX2-NEXT: movl %r8d, %r10d
-; AVX2-NEXT: movl %ecx, %r9d
-; AVX2-NEXT: movl %edx, %r8d
+; AVX2-NEXT: movl %r9d, %r10d
+; AVX2-NEXT: movl %r8d, %r9d
+; AVX2-NEXT: movl %ecx, %r8d
+; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
; AVX2-NEXT: # kill: def $esi killed $esi def $rsi
; AVX2-NEXT: # kill: def $edi killed $edi def $rdi
; AVX2-NEXT: movzbl 360(%rbp), %eax
@@ -1867,21 +1829,21 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vmovd %edi, %xmm5
; AVX2-NEXT: vpinsrb $1, %esi, %xmm5, %xmm5
; AVX2-NEXT: vpinsrb $2, %edx, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $3, %r9d, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $4, %r10d, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $5, %r11d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 16(%rbp), %ebx
-; AVX2-NEXT: vpinsrb $6, %ebx, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 24(%rbp), %r14d
-; AVX2-NEXT: vpinsrb $7, %r14d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 32(%rbp), %r15d
-; AVX2-NEXT: vpinsrb $8, %r15d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 40(%rbp), %r12d
-; AVX2-NEXT: vpinsrb $9, %r12d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 48(%rbp), %r13d
-; AVX2-NEXT: vpinsrb $10, %r13d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 56(%rbp), %eax
-; AVX2-NEXT: vpinsrb $11, %eax, %xmm5, %xmm5
+; AVX2-NEXT: vpinsrb $3, %r8d, %xmm5, %xmm5
+; AVX2-NEXT: vpinsrb $4, %r9d, %xmm5, %xmm5
+; AVX2-NEXT: vpinsrb $5, %r10d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 16(%rbp), %r11d
+; AVX2-NEXT: vpinsrb $6, %r11d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 24(%rbp), %ebx
+; AVX2-NEXT: vpinsrb $7, %ebx, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 32(%rbp), %r14d
+; AVX2-NEXT: vpinsrb $8, %r14d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 40(%rbp), %r15d
+; AVX2-NEXT: vpinsrb $9, %r15d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 48(%rbp), %r12d
+; AVX2-NEXT: vpinsrb $10, %r12d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 56(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $11, %r13d, %xmm5, %xmm5
; AVX2-NEXT: movzbl 64(%rbp), %eax
; AVX2-NEXT: vpinsrb $12, %eax, %xmm5, %xmm5
; AVX2-NEXT: movzbl 72(%rbp), %eax
@@ -1929,92 +1891,70 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vpaddb %ymm4, %ymm5, %ymm4
; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpextrb $1, %xmm4, %eax
-; AVX2-NEXT: vmovd %xmm4, %ecx
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: vpextrb $2, %xmm4, %edx
-; AVX2-NEXT: vpextrb $3, %xmm4, %eax
-; AVX2-NEXT: addb %dl, %al
-; AVX2-NEXT: addb %cl, %al
-; AVX2-NEXT: vpextrb $4, %xmm4, %ecx
-; AVX2-NEXT: vpextrb $5, %xmm4, %edx
-; AVX2-NEXT: addb %cl, %dl
-; AVX2-NEXT: vpextrb $6, %xmm4, %ecx
-; AVX2-NEXT: addb %dl, %cl
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: vpextrb $7, %xmm4, %eax
-; AVX2-NEXT: vpextrb $8, %xmm4, %edx
-; AVX2-NEXT: addb %al, %dl
-; AVX2-NEXT: vpextrb $9, %xmm4, %eax
-; AVX2-NEXT: addb %dl, %al
-; AVX2-NEXT: vpextrb $10, %xmm4, %edx
-; AVX2-NEXT: addb %al, %dl
-; AVX2-NEXT: addb %cl, %dl
-; AVX2-NEXT: vpextrb $11, %xmm4, %eax
-; AVX2-NEXT: vpextrb $12, %xmm4, %ecx
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: vpextrb $13, %xmm4, %eax
-; AVX2-NEXT: addb %cl, %al
-; AVX2-NEXT: vpextrb $14, %xmm4, %ecx
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: vpextrb $15, %xmm4, %eax
+; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm4[8,9,10,11,12,13,14,15,12,13],zero,zero,xmm4[14,15],zero,zero
+; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpsrld $16, %xmm4, %xmm5
+; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpextrb $1, %xmm4, %ecx
+; AVX2-NEXT: vmovd %xmm4, %eax
; AVX2-NEXT: addb %cl, %al
-; AVX2-NEXT: addb %dl, %al
; AVX2-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %ymm2, (%rsp)
; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: movzbl (%rsp,%rax), %edx
+; AVX2-NEXT: movzbl (%rsp,%rax), %eax
+; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)
; AVX2-NEXT: andl $1, %edi
; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rdi)
; AVX2-NEXT: andl $1, %esi
; AVX2-NEXT: addq %rdi, %rsi
; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rsi)
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rsi, %rdx
+; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rdx)
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rsi, %r8
-; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%r8)
+; AVX2-NEXT: addq %rdx, %r8
+; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%r8)
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %r8, %r9
-; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%r9)
+; AVX2-NEXT: movl %r9d, %eax
+; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
; AVX2-NEXT: andl $1, %r10d
; AVX2-NEXT: addq %r9, %r10
-; AVX2-NEXT: movl %r10d, %eax
-; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: andl $1, %r11d
-; AVX2-NEXT: addq %r10, %r11
-; AVX2-NEXT: movzbl %bl, %eax
+; AVX2-NEXT: movzbl %r11b, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %r11, %rax
-; AVX2-NEXT: # kill: def $r11d killed $r11d killed $r11 def $r11
-; AVX2-NEXT: andl $63, %r11d
-; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%r11)
-; AVX2-NEXT: movzbl %r14b, %ecx
+; AVX2-NEXT: addq %r10, %rax
+; AVX2-NEXT: # kill: def $r10d killed $r10d killed $r10 def $r10
+; AVX2-NEXT: andl $63, %r10d
+; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%r10)
+; AVX2-NEXT: movzbl %bl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl %r15b, %eax
+; AVX2-NEXT: movzbl %r14b, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl %r12b, %ecx
+; AVX2-NEXT: movzbl %r15b, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl %r13b, %eax
+; AVX2-NEXT: movzbl %r12b, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl 56(%rbp), %ecx
-; AVX2-NEXT: movzbl %cl, %ecx
+; AVX2-NEXT: movzbl %r13b, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
@@ -2388,7 +2328,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)
; AVX2-NEXT: vpextrb $15, %xmm0, %eax
; AVX2-NEXT: cmpq $64, %rcx
-; AVX2-NEXT: cmovbl %edx, %eax
+; AVX2-NEXT: cmovbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
; AVX2-NEXT: cmpq $63, %rcx
; AVX2-NEXT: movl $63, %edx
; AVX2-NEXT: cmovbq %rcx, %rdx
@@ -2840,19 +2780,19 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm2, %zmm4 {%k2} {z}
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm7
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
; AVX512F-NEXT: vpcompressd %zmm3, %zmm5 {%k2} {z}
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -1
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm0, %zmm6 {%k1} {z}
+; AVX512F-NEXT: vpcompressd %zmm0, %zmm7 {%k1} {z}
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
-; AVX512F-NEXT: vextracti128 $1, %ymm7, %xmm7
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero,xmm7[8],zero,zero,zero,xmm7[9],zero,zero,zero,xmm7[10],zero,zero,zero,xmm7[11],zero,zero,zero,xmm7[12],zero,zero,zero,xmm7[13],zero,zero,zero,xmm7[14],zero,zero,zero,xmm7[15],zero,zero,zero
+; AVX512F-NEXT: vextracti128 $1, %ymm6, %xmm6
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero,xmm6[8],zero,zero,zero,xmm6[9],zero,zero,zero,xmm6[10],zero,zero,zero,xmm6[11],zero,zero,zero,xmm6[12],zero,zero,zero,xmm6[13],zero,zero,zero,xmm6[14],zero,zero,zero,xmm6[15],zero,zero,zero
; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
-; AVX512F-NEXT: vpcompressd %zmm7, %zmm7 {%k1} {z}
+; AVX512F-NEXT: vpcompressd %zmm6, %zmm6 {%k1} {z}
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm8 {%k1} {z} = -1
; AVX512F-NEXT: vpmovdb %zmm4, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpsrld $31, %zmm2, %zmm4
@@ -2860,30 +2800,26 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpaddd %ymm4, %ymm9, %ymm4
; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm9
; AVX512F-NEXT: vpaddd %xmm4, %xmm9, %xmm4
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm9 = xmm4[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm4, %xmm9, %xmm4
; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
; AVX512F-NEXT: vmovd %xmm4, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm4, %edx
-; AVX512F-NEXT: vpextrd $3, %xmm4, %esi
; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: addl %edx, %esi
-; AVX512F-NEXT: addl %ecx, %esi
-; AVX512F-NEXT: andl $31, %esi
-; AVX512F-NEXT: vpmovdb %zmm6, 64(%rsp,%rsi)
+; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpmovdb %zmm7, 64(%rsp,%rcx)
; AVX512F-NEXT: vpmovdb %zmm5, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpsrld $31, %zmm3, %zmm4
; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
; AVX512F-NEXT: vmovd %xmm4, %ecx
; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm4, %eax
-; AVX512F-NEXT: vpextrd $3, %xmm4, %edx
-; AVX512F-NEXT: addl %eax, %edx
-; AVX512F-NEXT: addl %ecx, %edx
-; AVX512F-NEXT: andl $31, %edx
-; AVX512F-NEXT: vpmovdb %zmm7, 96(%rsp,%rdx)
+; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpmovdb %zmm6, 96(%rsp,%rcx)
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm4
; AVX512F-NEXT: vmovaps %ymm4, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm4
@@ -2892,16 +2828,14 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
; AVX512F-NEXT: vmovd %xmm4, %ecx
; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm4, %eax
-; AVX512F-NEXT: vpextrd $3, %xmm4, %edx
-; AVX512F-NEXT: addl %eax, %edx
-; AVX512F-NEXT: addl %ecx, %edx
-; AVX512F-NEXT: andl $63, %edx
+; AVX512F-NEXT: andl $63, %ecx
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm4
-; AVX512F-NEXT: vmovaps %ymm4, 128(%rsp,%rdx)
+; AVX512F-NEXT: vmovaps %ymm4, 128(%rsp,%rcx)
; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
; AVX512F-NEXT: vpmovdb %zmm8, %xmm4
; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
@@ -2950,21 +2884,13 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: vpaddw %ymm4, %ymm5, %ymm4
; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
; AVX2-NEXT: vpextrw $1, %xmm4, %eax
; AVX2-NEXT: vmovd %xmm4, %ecx
; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrw $2, %xmm4, %eax
-; AVX2-NEXT: vpextrw $3, %xmm4, %edx
-; AVX2-NEXT: addl %eax, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: vpextrw $4, %xmm4, %eax
-; AVX2-NEXT: vpextrw $5, %xmm4, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: vpextrw $6, %xmm4, %eax
-; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: addl %edx, %eax
-; AVX2-NEXT: vpextrw $7, %xmm4, %ecx
-; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: andl $31, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
@@ -3208,18 +3134,16 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX512F-NEXT: vpaddd %ymm5, %ymm1, %ymm1
; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm5
; AVX512F-NEXT: vpaddd %xmm5, %xmm1, %xmm1
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm5, %xmm1, %xmm1
; AVX512F-NEXT: vpextrd $1, %xmm1, %eax
; AVX512F-NEXT: vmovd %xmm1, %ecx
; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm1, %eax
-; AVX512F-NEXT: vpextrd $3, %xmm1, %edx
-; AVX512F-NEXT: addl %eax, %edx
-; AVX512F-NEXT: addl %ecx, %edx
-; AVX512F-NEXT: andl $31, %edx
+; AVX512F-NEXT: andl $31, %ecx
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdw %zmm0, (%rsp,%rdx,2)
+; AVX512F-NEXT: vpmovdw %zmm0, (%rsp,%rcx,2)
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512F-NEXT: vpsllw $15, %ymm4, %ymm1
; AVX512F-NEXT: vpsraw $15, %ymm1, %ymm1
@@ -4151,30 +4075,26 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: vpaddd %ymm7, %ymm4, %ymm4
; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm7
; AVX512F-NEXT: vpaddd %xmm7, %xmm4, %xmm4
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm7, %xmm4, %xmm4
; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
; AVX512F-NEXT: vmovd %xmm4, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm4, %edx
-; AVX512F-NEXT: vpextrd $3, %xmm4, %esi
; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: addl %edx, %esi
-; AVX512F-NEXT: addl %ecx, %esi
-; AVX512F-NEXT: andl $31, %esi
-; AVX512F-NEXT: vmovdqa64 %zmm1, 64(%rsp,%rsi,4)
+; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vmovdqa64 %zmm1, 64(%rsp,%rcx,4)
; AVX512F-NEXT: vmovdqa64 %zmm2, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpsrld $31, %zmm5, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2
; AVX512F-NEXT: vpaddd %ymm2, %ymm1, %ymm1
; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX512F-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm2, %xmm1, %xmm1
; AVX512F-NEXT: vpextrd $1, %xmm1, %eax
; AVX512F-NEXT: vmovd %xmm1, %ecx
; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm1, %eax
-; AVX512F-NEXT: vpextrd $3, %xmm1, %edx
-; AVX512F-NEXT: addl %eax, %edx
-; AVX512F-NEXT: addl %ecx, %edx
-; AVX512F-NEXT: andl $31, %edx
-; AVX512F-NEXT: vmovdqa64 %zmm3, 192(%rsp,%rdx,4)
+; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vmovdqa64 %zmm3, 192(%rsp,%rcx,4)
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
; AVX512F-NEXT: vmovaps %zmm1, {{[0-9]+}}(%rsp)
@@ -4184,19 +4104,17 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
; AVX512F-NEXT: vmovd %xmm0, %ecx
; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512F-NEXT: vpextrd $3, %xmm0, %edx
-; AVX512F-NEXT: addl %eax, %edx
-; AVX512F-NEXT: addl %ecx, %edx
-; AVX512F-NEXT: andl $63, %edx
+; AVX512F-NEXT: andl $63, %ecx
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
-; AVX512F-NEXT: vmovaps %zmm0, 320(%rsp,%rdx,4)
+; AVX512F-NEXT: vmovaps %zmm0, 320(%rsp,%rcx,4)
; AVX512F-NEXT: vmovaps %zmm2, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovaps %zmm1, 384(%rsp,%rdx,4)
+; AVX512F-NEXT: vmovaps %zmm1, 384(%rsp,%rcx,4)
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
@@ -4223,17 +4141,15 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512VL-NEXT: vpaddd %ymm5, %ymm1, %ymm1
; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm5
; AVX512VL-NEXT: vpaddd %xmm5, %xmm1, %xmm1
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX512VL-NEXT: vpaddd %xmm5, %xmm1, %xmm1
; AVX512VL-NEXT: vpextrd $1, %xmm1, %eax
; AVX512VL-NEXT: vmovd %xmm1, %ecx
; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: vpextrd $2, %xmm1, %eax
-; AVX512VL-NEXT: vpextrd $3, %xmm1, %edx
-; AVX512VL-NEXT: addl %eax, %edx
-; AVX512VL-NEXT: addl %ecx, %edx
-; AVX512VL-NEXT: andl $31, %edx
+; AVX512VL-NEXT: andl $31, %ecx
; AVX512VL-NEXT: kshiftrq $16, %k2, %k2
; AVX512VL-NEXT: vpcompressd %zmm2, %zmm1 {%k2} {z}
-; AVX512VL-NEXT: vmovdqa64 %zmm1, (%rsp,%rdx,4)
+; AVX512VL-NEXT: vmovdqa64 %zmm1, (%rsp,%rcx,4)
; AVX512VL-NEXT: vpcompressd %zmm3, %zmm1 {%k3} {z}
; AVX512VL-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 {%k3} {z} = -1
@@ -4242,16 +4158,14 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512VL-NEXT: vpaddd %ymm2, %ymm1, %ymm1
; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX512VL-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX512VL-NEXT: vpaddd %xmm2, %xmm1, %xmm1
; AVX512VL-NEXT: vpextrd $1, %xmm1, %eax
; AVX512VL-NEXT: vmovd %xmm1, %ecx
; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: vpextrd $2, %xmm1, %eax
-; AVX512VL-NEXT: vpextrd $3, %xmm1, %edx
-; AVX512VL-NEXT: addl %eax, %edx
-; AVX512VL-NEXT: addl %ecx, %edx
-; AVX512VL-NEXT: andl $31, %edx
+; AVX512VL-NEXT: andl $31, %ecx
; AVX512VL-NEXT: vpcompressd %zmm4, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rdx,4)
+; AVX512VL-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rcx,4)
; AVX512VL-NEXT: vmovdqa64 (%rsp), %zmm1
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
; AVX512VL-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
@@ -4262,19 +4176,17 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512VL-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VL-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpextrd $1, %xmm0, %eax
; AVX512VL-NEXT: vmovd %xmm0, %ecx
; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: vpextrd $2, %xmm0, %eax
-; AVX512VL-NEXT: vpextrd $3, %xmm0, %edx
-; AVX512VL-NEXT: addl %eax, %edx
-; AVX512VL-NEXT: addl %ecx, %edx
-; AVX512VL-NEXT: andl $63, %edx
+; AVX512VL-NEXT: andl $63, %ecx
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
-; AVX512VL-NEXT: vmovaps %zmm0, 256(%rsp,%rdx,4)
+; AVX512VL-NEXT: vmovaps %zmm0, 256(%rsp,%rcx,4)
; AVX512VL-NEXT: vmovaps %zmm2, {{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps %zmm1, 320(%rsp,%rdx,4)
+; AVX512VL-NEXT: vmovaps %zmm1, 320(%rsp,%rcx,4)
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
diff --git a/llvm/test/CodeGen/X86/vector-extract-last-active.ll b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
index 3f622c5c98077..a4ed74887c556 100644
--- a/llvm/test/CodeGen/X86/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
@@ -13,24 +13,29 @@ define i32 @extract_last_active_v4i32(<4 x i32> %a, <4 x i1> %c) {
; CHECK-NEXT: pslld $31, %xmm1
; CHECK-NEXT: psrad $31, %xmm1
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; CHECK-NEXT: movdqa %xmm1, %xmm3
+; CHECK-NEXT: por %xmm0, %xmm3
+; CHECK-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; CHECK-NEXT: por %xmm4, %xmm0
+; CHECK-NEXT: pcmpgtd %xmm0, %xmm3
+; CHECK-NEXT: pand %xmm3, %xmm1
+; CHECK-NEXT: pandn %xmm4, %xmm3
+; CHECK-NEXT: por %xmm1, %xmm3
+; CHECK-NEXT: movd %xmm3, %eax
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
+; CHECK-NEXT: movd %xmm0, %ecx
+; CHECK-NEXT: cmpl %ecx, %eax
+; CHECK-NEXT: cmoval %eax, %ecx
+; CHECK-NEXT: andl $3, %ecx
; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
; CHECK-NEXT: por %xmm2, %xmm0
-; CHECK-NEXT: movd %xmm0, %ecx
-; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: movd %xmm0, %edx
+; CHECK-NEXT: andb $1, %dl
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: cmpb $1, %cl
+; CHECK-NEXT: cmpb $1, %dl
; CHECK-NEXT: sbbl %eax, %eax
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; CHECK-NEXT: movd %xmm0, %ecx
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; CHECK-NEXT: movd %xmm0, %edx
-; CHECK-NEXT: cmpl %ecx, %edx
-; CHECK-NEXT: cmoval %edx, %ecx
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
-; CHECK-NEXT: movd %xmm0, %edx
-; CHECK-NEXT: cmpl %edx, %ecx
-; CHECK-NEXT: cmovbel %edx, %ecx
; CHECK-NEXT: orl -24(%rsp,%rcx,4), %eax
; CHECK-NEXT: retq
%res = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> %a, <4 x i1> %c, i32 -1)
@@ -44,17 +49,22 @@ define i32 @extract_last_active_v4i32_no_default(<4 x i32> %a, <4 x i1> %c) {
; CHECK-NEXT: psrad $31, %xmm1
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; CHECK-NEXT: movd %xmm0, %ecx
-; CHECK-NEXT: cmpl %eax, %ecx
-; CHECK-NEXT: cmoval %ecx, %eax
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; CHECK-NEXT: movdqa %xmm1, %xmm2
+; CHECK-NEXT: por %xmm0, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; CHECK-NEXT: por %xmm3, %xmm0
+; CHECK-NEXT: pcmpgtd %xmm0, %xmm2
+; CHECK-NEXT: pand %xmm2, %xmm1
+; CHECK-NEXT: pandn %xmm3, %xmm2
+; CHECK-NEXT: por %xmm1, %xmm2
+; CHECK-NEXT: movd %xmm2, %eax
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
; CHECK-NEXT: movd %xmm0, %ecx
; CHECK-NEXT: cmpl %ecx, %eax
-; CHECK-NEXT: cmovbel %ecx, %eax
-; CHECK-NEXT: movl -24(%rsp,%rax,4), %eax
+; CHECK-NEXT: cmoval %eax, %ecx
+; CHECK-NEXT: andl $3, %ecx
+; CHECK-NEXT: movl -24(%rsp,%rcx,4), %eax
; CHECK-NEXT: retq
%res = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> %a, <4 x i1> %c, i32 poison)
ret i32 %res
@@ -90,28 +100,35 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
; CHECK: # %bb.0:
; CHECK-NEXT: movd %esi, %xmm1
; CHECK-NEXT: movd %edi, %xmm2
-; CHECK-NEXT: movdqa %xmm2, %xmm3
-; CHECK-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; CHECK-NEXT: movd %edx, %xmm4
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; CHECK-NEXT: movdqa %xmm2, %xmm4
+; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-NEXT: movd %edx, %xmm3
+; CHECK-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: pslld $31, %xmm3
-; CHECK-NEXT: psrad $31, %xmm3
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; CHECK-NEXT: pslld $31, %xmm4
+; CHECK-NEXT: psrad $31, %xmm4
+; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; CHECK-NEXT: movdqa %xmm4, %xmm5
+; CHECK-NEXT: por %xmm0, %xmm5
+; CHECK-NEXT: pshufd {{.*#+}} xmm6 = xmm4[2,3,2,3]
+; CHECK-NEXT: por %xmm6, %xmm0
+; CHECK-NEXT: pcmpgtd %xmm0, %xmm5
+; CHECK-NEXT: pand %xmm5, %xmm4
+; CHECK-NEXT: pandn %xmm6, %xmm5
+; CHECK-NEXT: por %xmm4, %xmm5
+; CHECK-NEXT: movd %xmm5, %eax
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,1,1]
; CHECK-NEXT: movd %xmm0, %ecx
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
-; CHECK-NEXT: movd %xmm0, %eax
; CHECK-NEXT: cmpl %ecx, %eax
; CHECK-NEXT: cmoval %eax, %ecx
-; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: cmpl $0, %ecx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: por %xmm4, %xmm2
+; CHECK-NEXT: andl $3, %ecx
+; CHECK-NEXT: por %xmm3, %xmm2
; CHECK-NEXT: por %xmm1, %xmm2
; CHECK-NEXT: movd %xmm2, %edx
; CHECK-NEXT: andb $1, %dl
+; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: cmpb $1, %dl
; CHECK-NEXT: sbbl %eax, %eax
; CHECK-NEXT: orl -24(%rsp,%rcx,4), %eax
@@ -212,13 +229,12 @@ define i32 @extract_last_active_v4i32_penryn(<4 x i32> %a, <4 x i1> %c) "target-
; CHECK-NEXT: pslld $31, %xmm1
; CHECK-NEXT: psrad $31, %xmm1
; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: pextrd $2, %xmm1, %eax
-; CHECK-NEXT: pextrd $1, %xmm1, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; CHECK-NEXT: pmaxud %xmm1, %xmm2
+; CHECK-NEXT: pextrd $1, %xmm2, %eax
+; CHECK-NEXT: movd %xmm2, %ecx
; CHECK-NEXT: cmpl %eax, %ecx
; CHECK-NEXT: cmoval %ecx, %eax
-; CHECK-NEXT: pextrd $3, %xmm1, %ecx
-; CHECK-NEXT: cmpl %ecx, %eax
-; CHECK-NEXT: cmovbel %ecx, %eax
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movl -24(%rsp,%rax,4), %eax
; CHECK-NEXT: retq
More information about the llvm-commits
mailing list