[llvm] 507caaf - [X86] Add custom ISD::VEC_REDUCE_*MIN/MAX lowering (#194848)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 08:41:04 PDT 2026
Author: Simon Pilgrim
Date: 2026-04-29T15:40:58Z
New Revision: 507caafe0fe7e838dc879bdfad2420d77fe5799e
URL: https://github.com/llvm/llvm-project/commit/507caafe0fe7e838dc879bdfad2420d77fe5799e
DIFF: https://github.com/llvm/llvm-project/commit/507caafe0fe7e838dc879bdfad2420d77fe5799e.diff
LOG: [X86] Add custom ISD::VEC_REDUCE_*MIN/MAX lowering (#194848)
Pulled out of #194473 - update combineMinMaxReduction to fold to a
ISD::VECREDUCE_SMAX/SMIN/UMAX/UMIN node and then perform the lowering
later on.
combineMinMaxReduction will go away once we can use
shouldExpandReduction, rely on the middle-end to recognise reductions
and not have to recreate them from the expanded patterns.
I've added pre-SSE41 handling using vector unrolling - hopefully this
will go away once #194672 is in place.
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
llvm/test/CodeGen/X86/vector-extract-last-active.ll
llvm/test/CodeGen/X86/vector-reduce-smax.ll
llvm/test/CodeGen/X86/vector-reduce-smin.ll
llvm/test/CodeGen/X86/vector-reduce-umax.ll
llvm/test/CodeGen/X86/vector-reduce-umin.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6b926b8131db5..78a1dc9e790ce 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1161,6 +1161,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
}
+ // SSE2 can use basic vector unrolling.
+ // SSE41 can use PHMINPOS to perform v16i8/v8i16 minmax reductions.
+ for (auto VT : {MVT::v16i8, MVT::v8i16}) {
+ setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+ }
+
setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
@@ -29826,6 +29835,88 @@ static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget,
return SDValue();
}
+// Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
+// PHMINPOSUW.
+static SDValue LowerMINMAX_REDUCE(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ EVT ExtractVT = Op.getValueType();
+ if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
+ return SDValue();
+
+ // Check for SMAX/SMIN/UMAX/UMIN horizontal reduction patterns.
+ ISD::NodeType BinOp = ISD::getVecReduceBaseOpcode(Op.getOpcode());
+
+ SDValue Src = Op.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ EVT SrcSVT = SrcVT.getScalarType();
+ if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
+ return SDValue();
+
+ SDLoc DL(Op);
+ SDValue MinPos = Src;
+
+ // First, reduce the source down to 128-bit, applying BinOp to lo/hi.
+ while (SrcVT.getSizeInBits() > 128) {
+ SDValue Lo, Hi;
+ std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
+ SrcVT = Lo.getValueType();
+ MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
+ }
+ assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||
+ (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
+ "Unexpected value type");
+
+ // Without SSE41, we need to unroll.
+ if (!Subtarget.hasSSE41()) {
+ unsigned NumSrcElts = SrcVT.getVectorNumElements();
+ for (unsigned NumElts = NumSrcElts; NumElts != 1; NumElts /= 2) {
+ SmallVector<int, 16> Mask(NumSrcElts, -1);
+ std::iota(Mask.begin(), Mask.begin() + (NumElts / 2), NumElts / 2);
+ SDValue Upper =
+ DAG.getVectorShuffle(SrcVT, DL, MinPos, DAG.getUNDEF(SrcVT), Mask);
+ MinPos = DAG.getNode(BinOp, DL, SrcVT, MinPos, Upper);
+ }
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
+ DAG.getVectorIdxConstant(0, DL));
+ }
+
+ // PHMINPOSUW applies to UMIN(v8i16), for SMIN/SMAX/UMAX we must apply a mask
+ // to flip the value accordingly.
+ SDValue Mask;
+ unsigned MaskEltsBits = ExtractVT.getSizeInBits();
+ if (BinOp == ISD::SMAX)
+ Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
+ else if (BinOp == ISD::SMIN)
+ Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
+ else if (BinOp == ISD::UMAX)
+ Mask = DAG.getAllOnesConstant(DL, SrcVT);
+
+ if (Mask)
+ MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
+
+ // For v16i8 cases we need to perform UMIN on pairs of byte elements,
+ // shuffling each upper element down and insert zeros. This means that the
+ // v16i8 UMIN will leave the upper element as zero, performing zero-extension
+ // ready for the PHMINPOS.
+ if (ExtractVT == MVT::i8) {
+ SDValue Upper = DAG.getVectorShuffle(
+ SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
+ {1, 16, 3, 16, 5, 16, 7, 16, 9, 16, 11, 16, 13, 16, 15, 16});
+ MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
+ }
+
+ // Perform the PHMINPOS on a v8i16 vector,
+ MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
+ MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
+ MinPos = DAG.getBitcast(SrcVT, MinPos);
+
+ if (Mask)
+ MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -34385,6 +34476,10 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SMIN:
case ISD::UMAX:
case ISD::UMIN: return LowerMINMAX(Op, Subtarget, DAG);
+ case ISD::VECREDUCE_SMAX:
+ case ISD::VECREDUCE_SMIN:
+ case ISD::VECREDUCE_UMAX:
+ case ISD::VECREDUCE_UMIN: return LowerMINMAX_REDUCE(Op, Subtarget, DAG);
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
case ISD::FMINIMUMNUM:
@@ -46955,14 +47050,10 @@ static SDValue createPSADBW(SelectionDAG &DAG, SDValue N0, SDValue N1,
PSADBWBuilder);
}
-// Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
-// PHMINPOSUW.
+// Attempt to replace an integer min/max horizontal reduction with
+// ISD::VECREDUCE_SMIN/SMAX/UMIN/UMAX.
static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
- // Bail without SSE41.
- if (!Subtarget.hasSSE41())
- return SDValue();
-
EVT ExtractVT = Extract->getValueType(0);
if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
return SDValue();
@@ -46979,55 +47070,25 @@ static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
return SDValue();
- SDLoc DL(Extract);
- SDValue MinPos = Src;
-
- // First, reduce the source down to 128-bit, applying BinOp to lo/hi.
- while (SrcVT.getSizeInBits() > 128) {
- SDValue Lo, Hi;
- std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
- SrcVT = Lo.getValueType();
- MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
- }
- assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||
- (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
- "Unexpected value type");
-
- // PHMINPOSUW applies to UMIN(v8i16), for SMIN/SMAX/UMAX we must apply a mask
- // to flip the value accordingly.
- SDValue Mask;
- unsigned MaskEltsBits = ExtractVT.getSizeInBits();
- if (BinOp == ISD::SMAX)
- Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
- else if (BinOp == ISD::SMIN)
- Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
- else if (BinOp == ISD::UMAX)
- Mask = DAG.getAllOnesConstant(DL, SrcVT);
-
- if (Mask)
- MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
-
- // For v16i8 cases we need to perform UMIN on pairs of byte elements,
- // shuffling each upper element down and insert zeros. This means that the
- // v16i8 UMIN will leave the upper element as zero, performing zero-extension
- // ready for the PHMINPOS.
- if (ExtractVT == MVT::i8) {
- SDValue Upper = DAG.getVectorShuffle(
- SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
- {1, 16, 3, 16, 5, 16, 7, 16, 9, 16, 11, 16, 13, 16, 15, 16});
- MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
+ ISD::NodeType RdxOp;
+ switch (BinOp) {
+ case ISD::SMAX:
+ RdxOp = ISD::VECREDUCE_SMAX;
+ break;
+ case ISD::SMIN:
+ RdxOp = ISD::VECREDUCE_SMIN;
+ break;
+ case ISD::UMAX:
+ RdxOp = ISD::VECREDUCE_UMAX;
+ break;
+ case ISD::UMIN:
+ RdxOp = ISD::VECREDUCE_UMIN;
+ break;
+ default:
+ llvm_unreachable("Unexpected reduction");
}
- // Perform the PHMINPOS on a v8i16 vector,
- MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
- MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
- MinPos = DAG.getBitcast(SrcVT, MinPos);
-
- if (Mask)
- MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
-
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
- DAG.getVectorIdxConstant(0, DL));
+ return DAG.getNode(RdxOp, SDLoc(Extract), ExtractVT, Src);
}
// Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
@@ -47982,9 +48043,10 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
return Cmp;
- // Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW.
- if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
- return MinMax;
+ // Attempt to replace min/max v8i16/v16i8 reductions with ISD::VECREDUCE.
+ if (DCI.isBeforeLegalizeOps())
+ if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
+ return MinMax;
// Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
if (SDValue V = combineArithReduction(N, DAG, Subtarget))
diff --git a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
index 7c124324302c5..61191fd2637e8 100644
--- a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
+++ b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
@@ -15,28 +15,17 @@ define i8 @ctz_v8i16(<8 x i16> %a) {
; CHECK-NEXT: pxor %xmm1, %xmm1
; CHECK-NEXT: pcmpeqw %xmm0, %xmm1
; CHECK-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: pextrw $1, %xmm1, %ecx
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: cmpw %cx, %ax
-; CHECK-NEXT: cmoval %eax, %ecx
-; CHECK-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $4, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: paddw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %ecx
; CHECK-NEXT: movl $8, %eax
; CHECK-NEXT: subl %ecx, %eax
; CHECK-NEXT: # kill: def $al killed $al killed $eax
@@ -94,28 +83,17 @@ define i8 @ctz_v8i16_poison(<8 x i16> %a) {
; CHECK-NEXT: pxor %xmm1, %xmm1
; CHECK-NEXT: pcmpeqw %xmm0, %xmm1
; CHECK-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: pextrw $1, %xmm1, %ecx
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: cmpw %cx, %ax
-; CHECK-NEXT: cmoval %eax, %ecx
-; CHECK-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $4, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: paddw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %ecx
; CHECK-NEXT: movl $8, %eax
; CHECK-NEXT: subl %ecx, %eax
; CHECK-NEXT: # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/vector-extract-last-active.ll b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
index 690c91aaeee67..c8a59562f9506 100644
--- a/llvm/test/CodeGen/X86/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
@@ -127,56 +127,39 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
define i32 @extract_last_active_v8i32(<8 x i32> %a, <8 x i1> %c) {
; CHECK-LABEL: extract_last_active_v8i32:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset %rbx, -16
-; CHECK-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-NEXT: pextrw $6, %xmm2, %edx
-; CHECK-NEXT: pextrw $5, %xmm2, %r8d
-; CHECK-NEXT: pextrw $4, %xmm2, %ecx
-; CHECK-NEXT: pextrw $3, %xmm2, %r9d
-; CHECK-NEXT: pextrw $2, %xmm2, %edi
-; CHECK-NEXT: pextrw $1, %xmm2, %r11d
-; CHECK-NEXT: movd %xmm2, %r10d
; CHECK-NEXT: psllw $15, %xmm2
; CHECK-NEXT: psraw $15, %xmm2
; CHECK-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; CHECK-NEXT: pextrw $2, %xmm2, %esi
-; CHECK-NEXT: pextrw $1, %xmm2, %ebx
-; CHECK-NEXT: cmpw %si, %bx
-; CHECK-NEXT: cmoval %ebx, %esi
-; CHECK-NEXT: pextrw $3, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: pextrw $4, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: pextrw $5, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: pextrw $6, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: pextrw $7, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: andl $7, %esi
-; CHECK-NEXT: orl %r11d, %r10d
-; CHECK-NEXT: orl %r9d, %edi
-; CHECK-NEXT: orl %r10d, %edi
-; CHECK-NEXT: orl %r8d, %ecx
-; CHECK-NEXT: orl %edx, %ecx
-; CHECK-NEXT: orl %edi, %ecx
-; CHECK-NEXT: orl %eax, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; CHECK-NEXT: psubusw %xmm2, %xmm0
+; CHECK-NEXT: paddw %xmm2, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: paddw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %ecx
; CHECK-NEXT: andb $1, %cl
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: cmpb $1, %cl
; CHECK-NEXT: sbbl %eax, %eax
-; CHECK-NEXT: orl -32(%rsp,%rsi,4), %eax
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; CHECK-NEXT: psubusw %xmm2, %xmm0
+; CHECK-NEXT: paddw %xmm2, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: paddw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %ecx
+; CHECK-NEXT: andl $7, %ecx
+; CHECK-NEXT: orl -40(%rsp,%rcx,4), %eax
; CHECK-NEXT: retq
%res = call i32 @llvm.experimental.vector.extract.last.active.v8i32(<8 x i32> %a, <8 x i1> %c, i32 -1)
ret i32 %res
@@ -186,81 +169,42 @@ define i32 @extract_last_active_v8i32(<8 x i32> %a, <8 x i1> %c) {
define i32 @extract_last_active_v16i32(<16 x i32> %a, <16 x i1> %c) {
; CHECK-LABEL: extract_last_active_v16i32:
; CHECK: # %bb.0:
-; CHECK-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: psllw $7, %xmm4
; CHECK-NEXT: pxor %xmm5, %xmm5
; CHECK-NEXT: pcmpgtb %xmm4, %xmm5
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; CHECK-NEXT: movdqa %xmm5, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %dl, %al
-; CHECK-NEXT: cmoval %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: andl $15, %edx
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; CHECK-NEXT: pmaxub %xmm5, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %ecx
; CHECK-NEXT: andb $1, %cl
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: cmpb $1, %cl
; CHECK-NEXT: sbbl %eax, %eax
-; CHECK-NEXT: orl -72(%rsp,%rdx,4), %eax
+; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; CHECK-NEXT: pmaxub %xmm5, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %ecx
+; CHECK-NEXT: andl $15, %ecx
+; CHECK-NEXT: orl -72(%rsp,%rcx,4), %eax
; CHECK-NEXT: retq
%res = call i32 @llvm.experimental.vector.extract.last.active.v16i32(<16 x i32> %a, <16 x i1> %c, i32 -1)
ret i32 %res
@@ -291,148 +235,70 @@ define i32 @extract_last_active_v4i32_penryn(<4 x i32> %a, <4 x i1> %c) "target-
define i8 @extract_last_active_split(<32 x i8> %data, <32 x i8> %mask, i8 %passthru) {
; CHECK-LABEL: extract_last_active_split:
; CHECK: # %bb.0:
-; CHECK-NEXT: pxor %xmm4, %xmm4
-; CHECK-NEXT: movdqa %xmm2, %xmm5
-; CHECK-NEXT: por %xmm3, %xmm5
-; CHECK-NEXT: pcmpeqb %xmm4, %xmm3
+; CHECK-NEXT: movdqa %xmm2, %xmm4
+; CHECK-NEXT: por %xmm3, %xmm4
+; CHECK-NEXT: pxor %xmm5, %xmm5
+; CHECK-NEXT: pcmpeqb %xmm5, %xmm3
; CHECK-NEXT: pcmpeqd %xmm6, %xmm6
; CHECK-NEXT: movdqa %xmm3, %xmm7
; CHECK-NEXT: pxor %xmm6, %xmm7
-; CHECK-NEXT: movdqa %xmm7, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: pcmpeqb %xmm4, %xmm2
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: movdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; CHECK-NEXT: pandn %xmm7, %xmm2
-; CHECK-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: pandn %xmm7, %xmm3
-; CHECK-NEXT: movdqa %xmm3, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT: pcmpeqb %xmm5, %xmm2
; CHECK-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: pcmpeqb %xmm4, %xmm5
-; CHECK-NEXT: pxor %xmm6, %xmm5
-; CHECK-NEXT: movdqa %xmm5, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %cl, %dl
-; CHECK-NEXT: cmoval %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: cmpb %sil, %cl
-; CHECK-NEXT: cmovbel %esi, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: cmpb %dl, %cl
-; CHECK-NEXT: cmovbel %edx, %ecx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %dl, %sil
-; CHECK-NEXT: cmoval %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %r8d
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: cmpb %r8b, %dl
-; CHECK-NEXT: cmovbel %r8d, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: addl $16, %edx
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %al
-; CHECK-NEXT: testb $1, %al
-; CHECK-NEXT: cmoveq %rcx, %rdx
-; CHECK-NEXT: andl $31, %edx
-; CHECK-NEXT: movzbl -40(%rsp,%rdx), %eax
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; CHECK-NEXT: pandn %xmm0, %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; CHECK-NEXT: pmaxub %xmm2, %xmm1
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm1, %xmm2
+; CHECK-NEXT: movdqa %xmm2, %xmm1
+; CHECK-NEXT: psrld $16, %xmm1
+; CHECK-NEXT: pmaxub %xmm2, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm2
+; CHECK-NEXT: psrlw $8, %xmm2
+; CHECK-NEXT: pmaxub %xmm1, %xmm2
+; CHECK-NEXT: movd %xmm2, %eax
+; CHECK-NEXT: pandn %xmm0, %xmm3
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; CHECK-NEXT: pmaxub %xmm3, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %ecx
+; CHECK-NEXT: addl $16, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm7[2,3,2,3]
+; CHECK-NEXT: pmaxub %xmm7, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %edx
+; CHECK-NEXT: testb $1, %dl
+; CHECK-NEXT: cmoveq %rax, %rcx
+; CHECK-NEXT: andl $31, %ecx
+; CHECK-NEXT: movzbl -40(%rsp,%rcx), %eax
+; CHECK-NEXT: pcmpeqb %xmm5, %xmm4
+; CHECK-NEXT: pxor %xmm6, %xmm4
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
+; CHECK-NEXT: pmaxub %xmm4, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %ecx
; CHECK-NEXT: testb $1, %cl
; CHECK-NEXT: cmovel %edi, %eax
; CHECK-NEXT: # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smax.ll b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
index 95aea6f524023..fdf394e14eeef 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
@@ -1426,21 +1426,21 @@ define i16 @test_v32i16(<32 x i16> %a0) {
define i16 @test_v64i16(<64 x i16> %a0) {
; SSE2-LABEL: test_v64i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pmaxsw %xmm6, %xmm2
-; SSE2-NEXT: pmaxsw %xmm4, %xmm0
-; SSE2-NEXT: pmaxsw %xmm2, %xmm0
; SSE2-NEXT: pmaxsw %xmm7, %xmm3
; SSE2-NEXT: pmaxsw %xmm5, %xmm1
; SSE2-NEXT: pmaxsw %xmm3, %xmm1
-; SSE2-NEXT: pmaxsw %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: pmaxsw %xmm6, %xmm2
+; SSE2-NEXT: pmaxsw %xmm4, %xmm0
+; SSE2-NEXT: pmaxsw %xmm2, %xmm0
; SSE2-NEXT: pmaxsw %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: pmaxsw %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; SSE2-NEXT: pmaxsw %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: pmaxsw %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: retq
;
@@ -2023,35 +2023,35 @@ define i8 @test_v64i8(<64 x i8> %a0) {
define i8 @test_v128i8(<128 x i8> %a0) {
; SSE2-LABEL: test_v128i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: pcmpgtb %xmm6, %xmm8
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: pandn %xmm6, %xmm8
-; SSE2-NEXT: por %xmm2, %xmm8
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm4, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: pcmpgtb %xmm7, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm0
-; SSE2-NEXT: por %xmm3, %xmm0
+; SSE2-NEXT: movdqa %xmm3, %xmm8
+; SSE2-NEXT: pcmpgtb %xmm7, %xmm8
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm3, %xmm8
; SSE2-NEXT: movdqa %xmm1, %xmm3
; SSE2-NEXT: pcmpgtb %xmm5, %xmm3
; SSE2-NEXT: pand %xmm3, %xmm1
; SSE2-NEXT: pandn %xmm5, %xmm3
; SSE2-NEXT: por %xmm1, %xmm3
; SSE2-NEXT: movdqa %xmm3, %xmm1
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm8, %xmm1
; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pandn %xmm0, %xmm1
+; SSE2-NEXT: pandn %xmm8, %xmm1
; SSE2-NEXT: por %xmm3, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: pcmpgtb %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pandn %xmm6, %xmm3
+; SSE2-NEXT: por %xmm2, %xmm3
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: pcmpgtb %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm4, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm0
-; SSE2-NEXT: pcmpgtb %xmm8, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm0
; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: pandn %xmm8, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm0
; SSE2-NEXT: por %xmm2, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: pcmpgtb %xmm1, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smin.ll b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
index f86d3c1dd6e3f..84bbf6c8c03db 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
@@ -1427,21 +1427,21 @@ define i16 @test_v32i16(<32 x i16> %a0) {
define i16 @test_v64i16(<64 x i16> %a0) {
; SSE2-LABEL: test_v64i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pminsw %xmm6, %xmm2
-; SSE2-NEXT: pminsw %xmm4, %xmm0
-; SSE2-NEXT: pminsw %xmm2, %xmm0
; SSE2-NEXT: pminsw %xmm7, %xmm3
; SSE2-NEXT: pminsw %xmm5, %xmm1
; SSE2-NEXT: pminsw %xmm3, %xmm1
-; SSE2-NEXT: pminsw %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: pminsw %xmm6, %xmm2
+; SSE2-NEXT: pminsw %xmm4, %xmm0
+; SSE2-NEXT: pminsw %xmm2, %xmm0
; SSE2-NEXT: pminsw %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: pminsw %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; SSE2-NEXT: pminsw %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: pminsw %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: retq
;
@@ -2024,41 +2024,41 @@ define i8 @test_v64i8(<64 x i8> %a0) {
define i8 @test_v128i8(<128 x i8> %a0) {
; SSE2-LABEL: test_v128i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm5, %xmm8
-; SSE2-NEXT: pcmpgtb %xmm1, %xmm8
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: pandn %xmm5, %xmm8
-; SSE2-NEXT: por %xmm1, %xmm8
+; SSE2-NEXT: movdqa %xmm4, %xmm8
+; SSE2-NEXT: pcmpgtb %xmm0, %xmm8
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm0, %xmm8
+; SSE2-NEXT: movdqa %xmm6, %xmm4
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm4
+; SSE2-NEXT: pand %xmm4, %xmm2
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: por %xmm2, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm8, %xmm0
+; SSE2-NEXT: pand %xmm0, %xmm8
+; SSE2-NEXT: pandn %xmm4, %xmm0
+; SSE2-NEXT: por %xmm8, %xmm0
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT: pand %xmm2, %xmm1
+; SSE2-NEXT: pandn %xmm5, %xmm2
+; SSE2-NEXT: por %xmm1, %xmm2
; SSE2-NEXT: movdqa %xmm7, %xmm1
; SSE2-NEXT: pcmpgtb %xmm3, %xmm1
; SSE2-NEXT: pand %xmm1, %xmm3
; SSE2-NEXT: pandn %xmm7, %xmm1
; SSE2-NEXT: por %xmm3, %xmm1
-; SSE2-NEXT: movdqa %xmm4, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm3
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm4, %xmm3
-; SSE2-NEXT: por %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm6, %xmm0
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: pandn %xmm6, %xmm0
-; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm3
-; SSE2-NEXT: pandn %xmm0, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: pcmpgtb %xmm8, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm8
-; SSE2-NEXT: pandn %xmm1, %xmm0
-; SSE2-NEXT: por %xmm8, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm1
-; SSE2-NEXT: pand %xmm1, %xmm2
-; SSE2-NEXT: pandn %xmm0, %xmm1
-; SSE2-NEXT: por %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm3, %xmm2
+; SSE2-NEXT: pandn %xmm1, %xmm3
+; SSE2-NEXT: por %xmm2, %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm0, %xmm1
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm1
+; SSE2-NEXT: por %xmm0, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: pcmpgtb %xmm1, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umax.ll b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
index 2f3c1e09ea78e..588053cd05379 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
@@ -1648,16 +1648,16 @@ define i16 @test_v32i16(<32 x i16> %a0) {
define i16 @test_v64i16(<64 x i16> %a0) {
; SSE2-LABEL: test_v64i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: psubusw %xmm1, %xmm5
-; SSE2-NEXT: paddw %xmm1, %xmm5
-; SSE2-NEXT: psubusw %xmm3, %xmm7
-; SSE2-NEXT: paddw %xmm3, %xmm7
; SSE2-NEXT: psubusw %xmm0, %xmm4
; SSE2-NEXT: paddw %xmm0, %xmm4
; SSE2-NEXT: psubusw %xmm2, %xmm6
; SSE2-NEXT: paddw %xmm2, %xmm6
; SSE2-NEXT: psubusw %xmm4, %xmm6
; SSE2-NEXT: paddw %xmm4, %xmm6
+; SSE2-NEXT: psubusw %xmm1, %xmm5
+; SSE2-NEXT: paddw %xmm1, %xmm5
+; SSE2-NEXT: psubusw %xmm3, %xmm7
+; SSE2-NEXT: paddw %xmm3, %xmm7
; SSE2-NEXT: psubusw %xmm5, %xmm7
; SSE2-NEXT: paddw %xmm5, %xmm7
; SSE2-NEXT: psubusw %xmm6, %xmm7
@@ -2145,24 +2145,24 @@ define i8 @test_v64i8(<64 x i8> %a0) {
define i8 @test_v128i8(<128 x i8> %a0) {
; SSE2-LABEL: test_v128i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: pmaxub %xmm6, %xmm2
-; SSE2-NEXT: pmaxub %xmm4, %xmm0
-; SSE2-NEXT: pmaxub %xmm2, %xmm0
; SSE2-NEXT: pmaxub %xmm7, %xmm3
; SSE2-NEXT: pmaxub %xmm5, %xmm1
; SSE2-NEXT: pmaxub %xmm3, %xmm1
-; SSE2-NEXT: pmaxub %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: pmaxub %xmm6, %xmm2
+; SSE2-NEXT: pmaxub %xmm4, %xmm0
+; SSE2-NEXT: pmaxub %xmm2, %xmm0
; SSE2-NEXT: pmaxub %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: pmaxub %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; SSE2-NEXT: pmaxub %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrlw $8, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
; SSE2-NEXT: pmaxub %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: psrlw $8, %xmm0
+; SSE2-NEXT: pmaxub %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $al killed $al killed $eax
; SSE2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umin.ll b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
index dc8d0d53a91a1..ce2175637781c 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
@@ -1602,21 +1602,21 @@ define i16 @test_v32i16(<32 x i16> %a0) {
define i16 @test_v64i16(<64 x i16> %a0) {
; SSE2-LABEL: test_v64i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm2, %xmm8
-; SSE2-NEXT: psubusw %xmm6, %xmm8
-; SSE2-NEXT: psubw %xmm8, %xmm2
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: psubusw %xmm4, %xmm6
-; SSE2-NEXT: psubw %xmm6, %xmm0
-; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: psubusw %xmm7, %xmm4
-; SSE2-NEXT: psubw %xmm4, %xmm3
-; SSE2-NEXT: movdqa %xmm1, %xmm4
-; SSE2-NEXT: psubusw %xmm5, %xmm4
-; SSE2-NEXT: psubw %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm4
-; SSE2-NEXT: psubusw %xmm3, %xmm4
-; SSE2-NEXT: psubw %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm3, %xmm8
+; SSE2-NEXT: psubusw %xmm7, %xmm8
+; SSE2-NEXT: psubw %xmm8, %xmm3
+; SSE2-NEXT: movdqa %xmm1, %xmm7
+; SSE2-NEXT: psubusw %xmm5, %xmm7
+; SSE2-NEXT: psubw %xmm7, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm5
+; SSE2-NEXT: psubusw %xmm3, %xmm5
+; SSE2-NEXT: psubw %xmm5, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: psubusw %xmm6, %xmm3
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: psubusw %xmm4, %xmm3
+; SSE2-NEXT: psubw %xmm3, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm3
; SSE2-NEXT: psubusw %xmm2, %xmm3
; SSE2-NEXT: psubw %xmm3, %xmm0
@@ -2010,24 +2010,24 @@ define i8 @test_v64i8(<64 x i8> %a0) {
define i8 @test_v128i8(<128 x i8> %a0) {
; SSE2-LABEL: test_v128i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: pminub %xmm6, %xmm2
-; SSE2-NEXT: pminub %xmm4, %xmm0
-; SSE2-NEXT: pminub %xmm2, %xmm0
; SSE2-NEXT: pminub %xmm7, %xmm3
; SSE2-NEXT: pminub %xmm5, %xmm1
; SSE2-NEXT: pminub %xmm3, %xmm1
-; SSE2-NEXT: pminub %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: pminub %xmm6, %xmm2
+; SSE2-NEXT: pminub %xmm4, %xmm0
+; SSE2-NEXT: pminub %xmm2, %xmm0
; SSE2-NEXT: pminub %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: pminub %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; SSE2-NEXT: pminub %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrlw $8, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
; SSE2-NEXT: pminub %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: psrlw $8, %xmm0
+; SSE2-NEXT: pminub %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $al killed $al killed $eax
; SSE2-NEXT: retq
;
More information about the llvm-commits
mailing list