[llvm] [X86] Add custom ISD::VEC_REDUCE_*MIN/MAX lowering (PR #194848)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 07:00:27 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Simon Pilgrim (RKSimon)
<details>
<summary>Changes</summary>
Pulled out of #<!-- -->194473 - update combineMinMaxReduction to fold to a ISD::VECREDUCE_SMAX/SMIN/UMAX/UMIN node and then perform the lowering later on.
combineMinMaxReduction will go away once we can use shouldExpandReduction, rely on the middle-end to recognise reductions and not have to recreate them from the expanded patterns.
I've added pre-SSE41 handling using vector unrolling - hopefully this will go away once #<!-- -->194672 is in place.
---
Patch is 41.07 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/194848.diff
7 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+118-56)
- (modified) llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll (+22-44)
- (modified) llvm/test/CodeGen/X86/vector-extract-last-active.ll (+109-243)
- (modified) llvm/test/CodeGen/X86/vector-reduce-smax.ll (+28-28)
- (modified) llvm/test/CodeGen/X86/vector-reduce-smin.ll (+39-39)
- (modified) llvm/test/CodeGen/X86/vector-reduce-umax.ll (+14-14)
- (modified) llvm/test/CodeGen/X86/vector-reduce-umin.ll (+25-25)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6b926b8131db5..78a1dc9e790ce 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1161,6 +1161,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
}
+ // SSE2 can use basic vector unrolling.
+ // SSE41 can use PHMINPOS to perform v16i8/v8i16 minmax reductions.
+ for (auto VT : {MVT::v16i8, MVT::v8i16}) {
+ setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+ }
+
setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
@@ -29826,6 +29835,88 @@ static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget,
return SDValue();
}
+// Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
+// PHMINPOSUW.
+static SDValue LowerMINMAX_REDUCE(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ EVT ExtractVT = Op.getValueType();
+ if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
+ return SDValue();
+
+ // Check for SMAX/SMIN/UMAX/UMIN horizontal reduction patterns.
+ ISD::NodeType BinOp = ISD::getVecReduceBaseOpcode(Op.getOpcode());
+
+ SDValue Src = Op.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ EVT SrcSVT = SrcVT.getScalarType();
+ if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
+ return SDValue();
+
+ SDLoc DL(Op);
+ SDValue MinPos = Src;
+
+ // First, reduce the source down to 128-bit, applying BinOp to lo/hi.
+ while (SrcVT.getSizeInBits() > 128) {
+ SDValue Lo, Hi;
+ std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
+ SrcVT = Lo.getValueType();
+ MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
+ }
+ assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||
+ (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
+ "Unexpected value type");
+
+ // Without SSE41, we need to unroll.
+ if (!Subtarget.hasSSE41()) {
+ unsigned NumSrcElts = SrcVT.getVectorNumElements();
+ for (unsigned NumElts = NumSrcElts; NumElts != 1; NumElts /= 2) {
+ SmallVector<int, 16> Mask(NumSrcElts, -1);
+ std::iota(Mask.begin(), Mask.begin() + (NumElts / 2), NumElts / 2);
+ SDValue Upper =
+ DAG.getVectorShuffle(SrcVT, DL, MinPos, DAG.getUNDEF(SrcVT), Mask);
+ MinPos = DAG.getNode(BinOp, DL, SrcVT, MinPos, Upper);
+ }
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
+ DAG.getVectorIdxConstant(0, DL));
+ }
+
+ // PHMINPOSUW applies to UMIN(v8i16), for SMIN/SMAX/UMAX we must apply a mask
+ // to flip the value accordingly.
+ SDValue Mask;
+ unsigned MaskEltsBits = ExtractVT.getSizeInBits();
+ if (BinOp == ISD::SMAX)
+ Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
+ else if (BinOp == ISD::SMIN)
+ Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
+ else if (BinOp == ISD::UMAX)
+ Mask = DAG.getAllOnesConstant(DL, SrcVT);
+
+ if (Mask)
+ MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
+
+ // For v16i8 cases we need to perform UMIN on pairs of byte elements,
+ // shuffling each upper element down and insert zeros. This means that the
+ // v16i8 UMIN will leave the upper element as zero, performing zero-extension
+ // ready for the PHMINPOS.
+ if (ExtractVT == MVT::i8) {
+ SDValue Upper = DAG.getVectorShuffle(
+ SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
+ {1, 16, 3, 16, 5, 16, 7, 16, 9, 16, 11, 16, 13, 16, 15, 16});
+ MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
+ }
+
+ // Perform the PHMINPOS on a v8i16 vector,
+ MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
+ MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
+ MinPos = DAG.getBitcast(SrcVT, MinPos);
+
+ if (Mask)
+ MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
+
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -34385,6 +34476,10 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SMIN:
case ISD::UMAX:
case ISD::UMIN: return LowerMINMAX(Op, Subtarget, DAG);
+ case ISD::VECREDUCE_SMAX:
+ case ISD::VECREDUCE_SMIN:
+ case ISD::VECREDUCE_UMAX:
+ case ISD::VECREDUCE_UMIN: return LowerMINMAX_REDUCE(Op, Subtarget, DAG);
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
case ISD::FMINIMUMNUM:
@@ -46955,14 +47050,10 @@ static SDValue createPSADBW(SelectionDAG &DAG, SDValue N0, SDValue N1,
PSADBWBuilder);
}
-// Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
-// PHMINPOSUW.
+// Attempt to replace an integer min/max horizontal reduction with
+// ISD::VECREDUCE_SMIN/SMAX/UMIN/UMAX.
static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
- // Bail without SSE41.
- if (!Subtarget.hasSSE41())
- return SDValue();
-
EVT ExtractVT = Extract->getValueType(0);
if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
return SDValue();
@@ -46979,55 +47070,25 @@ static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
return SDValue();
- SDLoc DL(Extract);
- SDValue MinPos = Src;
-
- // First, reduce the source down to 128-bit, applying BinOp to lo/hi.
- while (SrcVT.getSizeInBits() > 128) {
- SDValue Lo, Hi;
- std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
- SrcVT = Lo.getValueType();
- MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
- }
- assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||
- (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
- "Unexpected value type");
-
- // PHMINPOSUW applies to UMIN(v8i16), for SMIN/SMAX/UMAX we must apply a mask
- // to flip the value accordingly.
- SDValue Mask;
- unsigned MaskEltsBits = ExtractVT.getSizeInBits();
- if (BinOp == ISD::SMAX)
- Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
- else if (BinOp == ISD::SMIN)
- Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
- else if (BinOp == ISD::UMAX)
- Mask = DAG.getAllOnesConstant(DL, SrcVT);
-
- if (Mask)
- MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
-
- // For v16i8 cases we need to perform UMIN on pairs of byte elements,
- // shuffling each upper element down and insert zeros. This means that the
- // v16i8 UMIN will leave the upper element as zero, performing zero-extension
- // ready for the PHMINPOS.
- if (ExtractVT == MVT::i8) {
- SDValue Upper = DAG.getVectorShuffle(
- SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
- {1, 16, 3, 16, 5, 16, 7, 16, 9, 16, 11, 16, 13, 16, 15, 16});
- MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
+ ISD::NodeType RdxOp;
+ switch (BinOp) {
+ case ISD::SMAX:
+ RdxOp = ISD::VECREDUCE_SMAX;
+ break;
+ case ISD::SMIN:
+ RdxOp = ISD::VECREDUCE_SMIN;
+ break;
+ case ISD::UMAX:
+ RdxOp = ISD::VECREDUCE_UMAX;
+ break;
+ case ISD::UMIN:
+ RdxOp = ISD::VECREDUCE_UMIN;
+ break;
+ default:
+ llvm_unreachable("Unexpected reduction");
}
- // Perform the PHMINPOS on a v8i16 vector,
- MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
- MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
- MinPos = DAG.getBitcast(SrcVT, MinPos);
-
- if (Mask)
- MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
-
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
- DAG.getVectorIdxConstant(0, DL));
+ return DAG.getNode(RdxOp, SDLoc(Extract), ExtractVT, Src);
}
// Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
@@ -47982,9 +48043,10 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
return Cmp;
- // Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW.
- if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
- return MinMax;
+ // Attempt to replace min/max v8i16/v16i8 reductions with ISD::VECREDUCE.
+ if (DCI.isBeforeLegalizeOps())
+ if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
+ return MinMax;
// Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
if (SDValue V = combineArithReduction(N, DAG, Subtarget))
diff --git a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
index 7c124324302c5..61191fd2637e8 100644
--- a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
+++ b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
@@ -15,28 +15,17 @@ define i8 @ctz_v8i16(<8 x i16> %a) {
; CHECK-NEXT: pxor %xmm1, %xmm1
; CHECK-NEXT: pcmpeqw %xmm0, %xmm1
; CHECK-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: pextrw $1, %xmm1, %ecx
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: cmpw %cx, %ax
-; CHECK-NEXT: cmoval %eax, %ecx
-; CHECK-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $4, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: paddw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %ecx
; CHECK-NEXT: movl $8, %eax
; CHECK-NEXT: subl %ecx, %eax
; CHECK-NEXT: # kill: def $al killed $al killed $eax
@@ -94,28 +83,17 @@ define i8 @ctz_v8i16_poison(<8 x i16> %a) {
; CHECK-NEXT: pxor %xmm1, %xmm1
; CHECK-NEXT: pcmpeqw %xmm0, %xmm1
; CHECK-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: pextrw $1, %xmm1, %ecx
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: cmpw %cx, %ax
-; CHECK-NEXT: cmoval %eax, %ecx
-; CHECK-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $4, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
-; CHECK-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-NEXT: cmpw %ax, %cx
-; CHECK-NEXT: cmovbel %eax, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: paddw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %ecx
; CHECK-NEXT: movl $8, %eax
; CHECK-NEXT: subl %ecx, %eax
; CHECK-NEXT: # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/vector-extract-last-active.ll b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
index 690c91aaeee67..c8a59562f9506 100644
--- a/llvm/test/CodeGen/X86/vector-extract-last-active.ll
+++ b/llvm/test/CodeGen/X86/vector-extract-last-active.ll
@@ -127,56 +127,39 @@ define i32 @extract_last_active_v3i32(<3 x i32> %a, <3 x i1> %c) {
define i32 @extract_last_active_v8i32(<8 x i32> %a, <8 x i1> %c) {
; CHECK-LABEL: extract_last_active_v8i32:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset %rbx, -16
-; CHECK-NEXT: pextrw $7, %xmm2, %eax
-; CHECK-NEXT: pextrw $6, %xmm2, %edx
-; CHECK-NEXT: pextrw $5, %xmm2, %r8d
-; CHECK-NEXT: pextrw $4, %xmm2, %ecx
-; CHECK-NEXT: pextrw $3, %xmm2, %r9d
-; CHECK-NEXT: pextrw $2, %xmm2, %edi
-; CHECK-NEXT: pextrw $1, %xmm2, %r11d
-; CHECK-NEXT: movd %xmm2, %r10d
; CHECK-NEXT: psllw $15, %xmm2
; CHECK-NEXT: psraw $15, %xmm2
; CHECK-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; CHECK-NEXT: pextrw $2, %xmm2, %esi
-; CHECK-NEXT: pextrw $1, %xmm2, %ebx
-; CHECK-NEXT: cmpw %si, %bx
-; CHECK-NEXT: cmoval %ebx, %esi
-; CHECK-NEXT: pextrw $3, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: pextrw $4, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: pextrw $5, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: pextrw $6, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: pextrw $7, %xmm2, %ebx
-; CHECK-NEXT: cmpw %bx, %si
-; CHECK-NEXT: cmovbel %ebx, %esi
-; CHECK-NEXT: andl $7, %esi
-; CHECK-NEXT: orl %r11d, %r10d
-; CHECK-NEXT: orl %r9d, %edi
-; CHECK-NEXT: orl %r10d, %edi
-; CHECK-NEXT: orl %r8d, %ecx
-; CHECK-NEXT: orl %edx, %ecx
-; CHECK-NEXT: orl %edi, %ecx
-; CHECK-NEXT: orl %eax, %ecx
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; CHECK-NEXT: psubusw %xmm2, %xmm0
+; CHECK-NEXT: paddw %xmm2, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: paddw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %ecx
; CHECK-NEXT: andb $1, %cl
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: cmpb $1, %cl
; CHECK-NEXT: sbbl %eax, %eax
-; CHECK-NEXT: orl -32(%rsp,%rsi,4), %eax
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; CHECK-NEXT: psubusw %xmm2, %xmm0
+; CHECK-NEXT: paddw %xmm2, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: paddw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: psubusw %xmm1, %xmm0
+; CHECK-NEXT: paddw %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %ecx
+; CHECK-NEXT: andl $7, %ecx
+; CHECK-NEXT: orl -40(%rsp,%rcx,4), %eax
; CHECK-NEXT: retq
%res = call i32 @llvm.experimental.vector.extract.last.active.v8i32(<8 x i32> %a, <8 x i1> %c, i32 -1)
ret i32 %res
@@ -186,81 +169,42 @@ define i32 @extract_last_active_v8i32(<8 x i32> %a, <8 x i1> %c) {
define i32 @extract_last_active_v16i32(<16 x i32> %a, <16 x i1> %c) {
; CHECK-LABEL: extract_last_active_v16i32:
; CHECK: # %bb.0:
-; CHECK-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: psllw $7, %xmm4
; CHECK-NEXT: pxor %xmm5, %xmm5
; CHECK-NEXT: pcmpgtb %xmm4, %xmm5
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; CHECK-NEXT: movdqa %xmm5, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %dl, %al
-; CHECK-NEXT: cmoval %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %esi
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: cmpb %sil, %dl
-; CHECK-NEXT: cmovbel %esi, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
-; CHECK-NEXT: cmpb %al, %dl
-; CHECK-NEXT: cmovbel %eax, %edx
-; CHECK-NEXT: andl $15, %edx
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
-; CHECK-NEXT: orb -{{[0-9]+}}(%rsp), %cl
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; CHECK-NEXT: pmaxub %xmm5, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %ecx
; CHECK-NEXT: andb $1, %cl
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: cmpb $1, %cl
; CHECK-NEXT: sbbl %eax, %eax
-; CHECK-NEXT: orl -72(%rsp,%rdx,4), %eax
+; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; CHECK-NEXT: pmaxub %xmm5, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %ecx
+; CHECK-NEXT: andl $15, %ecx
+; CHECK-NEXT: orl -72(%...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/194848
More information about the llvm-commits
mailing list