[llvm] r202301 - Lower unsigned vsetcc to psubus in certain cases
Quentin Colombet
qcolombet at apple.com
Wed Feb 26 13:39:12 PST 2014
Author: qcolombet
Date: Wed Feb 26 15:39:12 2014
New Revision: 202301
URL: http://llvm.org/viewvc/llvm-project?rev=202301&view=rev
Log:
Lower unsigned vsetcc to psubus in certain cases
The current approach to lower a vsetult is to flip the sign bit of the
operands, swap the operands and then use a (signed) pcmpgt. psubus (unsigned
saturating subtract) can be used to emulate a vsetult more efficiently:
+ case ISD::SETULT: {
+ // If the comparison is against a constant we can turn this into a
+ // setule. With psubus, setule does not require a swap. This is
+ // beneficial because the constant in the register is no longer
+ // destructed as the destination so it can be hoisted out of a loop.
I also enable lowering via psubus in a few other cases where it's clearly
beneficial: setule and setuge if minu/maxu cannot be used.
rdar://problem/14338765
Patch by Adam Nemet <anemet at apple.com>.
Added:
llvm/trunk/test/CodeGen/X86/vec_setcc-2.ll
Modified:
llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
llvm/trunk/test/CodeGen/X86/vec_setcc.ll
Modified: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=202301&r1=202300&r2=202301&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp Wed Feb 26 15:39:12 2014
@@ -10009,6 +10009,37 @@ static SDValue LowerIntVSETCC_AVX512(SDV
DAG.getConstant(SSECC, MVT::i8));
}
+/// \brief Try to turn a VSETULT into a VSETULE by modifying its second
+/// operand \p Op1. If non-trivial (for example because it's not constant)
+/// return an empty value.
+static SDValue ChangeVSETULTtoVSETULE(SDValue Op1, SelectionDAG &DAG)
+{
+ BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op1.getNode());
+ if (!BV)
+ return SDValue();
+
+ MVT VT = Op1.getSimpleValueType();
+ MVT EVT = VT.getVectorElementType();
+ unsigned n = VT.getVectorNumElements();
+ SmallVector<SDValue, 8> ULTOp1;
+
+ for (unsigned i = 0; i < n; ++i) {
+ ConstantSDNode *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
+ if (!Elt || Elt->isOpaque() || Elt->getValueType(0) != EVT)
+ return SDValue();
+
+ // Avoid underflow.
+ APInt Val = Elt->getAPIntValue();
+ if (Val == 0)
+ return SDValue();
+
+ ULTOp1.push_back(DAG.getConstant(Val - 1, EVT));
+ }
+
+ return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op1), VT, ULTOp1.data(),
+ ULTOp1.size());
+}
+
static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
SelectionDAG &DAG) {
SDValue Op0 = Op.getOperand(0);
@@ -10080,6 +10111,7 @@ static SDValue LowerVSETCC(SDValue Op, c
// operations may be required for some comparisons.
unsigned Opc;
bool Swap = false, Invert = false, FlipSigns = false, MinMax = false;
+ bool Subus = false;
switch (SetCCOpcode) {
default: llvm_unreachable("Unexpected SETCC condition");
@@ -10114,6 +10146,40 @@ static SDValue LowerVSETCC(SDValue Op, c
if (MinMax) { Swap = false; Invert = false; FlipSigns = false; }
}
+ bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16);
+ if (!MinMax && hasSubus) {
+ // As another special case, use PSUBUS[BW] when it's profitable. E.g. for
+ // Op0 u<= Op1:
+ // t = psubus Op0, Op1
+ // pcmpeq t, <0..0>
+ switch (SetCCOpcode) {
+ default: break;
+ case ISD::SETULT: {
+ // If the comparison is against a constant we can turn this into a
+ // setule. With psubus, setule does not require a swap. This is
+ // beneficial because the constant in the register is no longer
+ // destructed as the destination so it can be hoisted out of a loop.
+ // Only do this pre-AVX since vpcmp* is no longer destructive.
+ if (Subtarget->hasAVX())
+ break;
+ SDValue ULEOp1 = ChangeVSETULTtoVSETULE(Op1, DAG);
+ if (ULEOp1.getNode()) {
+ Op1 = ULEOp1;
+ Subus = true; Invert = false; Swap = false;
+ }
+ break;
+ }
+ // Psubus is better than flip-sign because it requires no inversion.
+ case ISD::SETUGE: Subus = true; Invert = false; Swap = true; break;
+ case ISD::SETULE: Subus = true; Invert = false; Swap = false; break;
+ }
+
+ if (Subus) {
+ Opc = X86ISD::SUBUS;
+ FlipSigns = false;
+ }
+ }
+
if (Swap)
std::swap(Op0, Op1);
@@ -10204,6 +10270,10 @@ static SDValue LowerVSETCC(SDValue Op, c
if (MinMax)
Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
+ if (Subus)
+ Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
+ getZeroVector(VT, Subtarget, DAG, dl));
+
return Result;
}
Added: llvm/trunk/test/CodeGen/X86/vec_setcc-2.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/vec_setcc-2.ll?rev=202301&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/X86/vec_setcc-2.ll (added)
+++ llvm/trunk/test/CodeGen/X86/vec_setcc-2.ll Wed Feb 26 15:39:12 2014
@@ -0,0 +1,95 @@
+; RUN: llc < %s -o - -mcpu=generic -mtriple=x86_64-apple-darwin -mattr=+sse2 | FileCheck %s
+; RUN: llc < %s -o - -mcpu=generic -mtriple=x86_64-apple-darwin -mattr=+sse4.2 | FileCheck %s
+
+; For a setult against a constant, turn it into a setule and lower via psubusw.
+
+define void @loop_no_const_reload(<2 x i64>* %in, <2 x i64>* %out, i32 %n) {
+; CHECK: .short 25
+; CHECK-NEXT: .short 25
+; CHECK-NEXT: .short 25
+; CHECK-NEXT: .short 25
+; CHECK-NEXT: .short 25
+; CHECK-NEXT: .short 25
+; CHECK-NEXT: .short 25
+; CHECK-NEXT: .short 25
+; CHECK-LABEL: loop_no_const_reload:
+; CHECK: psubusw
+
+; Constant is no longer clobbered so no need to reload it in the loop.
+
+; CHECK-NOT: movdqa {{%xmm[0-9]+}}, {{%xmm[0-9]+}}
+entry:
+ %cmp9 = icmp eq i32 %n, 0
+ br i1 %cmp9, label %for.end, label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx1 = getelementptr inbounds <2 x i64>* %in, i64 %indvars.iv
+ %arrayidx1.val = load <2 x i64>* %arrayidx1, align 16
+ %0 = bitcast <2 x i64> %arrayidx1.val to <8 x i16>
+ %cmp.i.i = icmp ult <8 x i16> %0, <i16 26, i16 26, i16 26, i16 26, i16 26, i16 26, i16 26, i16 26>
+ %sext.i.i = sext <8 x i1> %cmp.i.i to <8 x i16>
+ %1 = bitcast <8 x i16> %sext.i.i to <2 x i64>
+ %arrayidx5 = getelementptr inbounds <2 x i64>* %out, i64 %indvars.iv
+ store <2 x i64> %1, <2 x i64>* %arrayidx5, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, %n
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end: ; preds = %for.body, %entry
+ ret void
+}
+
+; Be careful if decrementing the constant would undeflow.
+
+define void @loop_const_folding_underflow(<2 x i64>* %in, <2 x i64>* %out, i32 %n) {
+; CHECK-NOT: .short 25
+; CHECK-LABEL: loop_const_folding_underflow:
+; CHECK-NOT: psubusw
+entry:
+ %cmp9 = icmp eq i32 %n, 0
+ br i1 %cmp9, label %for.end, label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx1 = getelementptr inbounds <2 x i64>* %in, i64 %indvars.iv
+ %arrayidx1.val = load <2 x i64>* %arrayidx1, align 16
+ %0 = bitcast <2 x i64> %arrayidx1.val to <8 x i16>
+ %cmp.i.i = icmp ult <8 x i16> %0, <i16 0, i16 26, i16 26, i16 26, i16 26, i16 26, i16 26, i16 26>
+ %sext.i.i = sext <8 x i1> %cmp.i.i to <8 x i16>
+ %1 = bitcast <8 x i16> %sext.i.i to <2 x i64>
+ %arrayidx5 = getelementptr inbounds <2 x i64>* %out, i64 %indvars.iv
+ store <2 x i64> %1, <2 x i64>* %arrayidx5, align 16
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, %n
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end: ; preds = %for.body, %entry
+ ret void
+}
+
+; Test for PSUBUSB
+
+define <16 x i8> @test_ult_byte(<16 x i8> %a) {
+; CHECK: .space 16,10
+; CHECK-LABEL: test_ult_byte:
+; CHECK: psubus
+entry:
+ %icmp = icmp ult <16 x i8> %a, <i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11>
+ %sext = sext <16 x i1> %icmp to <16 x i8>
+ ret <16 x i8> %sext
+}
+
+; Only do this when we can turn the comparison into a setule. I.e. not for
+; register operands.
+
+define <8 x i16> @test_ult_register(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_ult_register:
+; CHECK-NOT: psubus
+entry:
+ %icmp = icmp ult <8 x i16> %a, %b
+ %sext = sext <8 x i1> %icmp to <8 x i16>
+ ret <8 x i16> %sext
+}
Modified: llvm/trunk/test/CodeGen/X86/vec_setcc.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/vec_setcc.ll?rev=202301&r1=202300&r2=202301&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/vec_setcc.ll (original)
+++ llvm/trunk/test/CodeGen/X86/vec_setcc.ll Wed Feb 26 15:39:12 2014
@@ -42,12 +42,9 @@ define <8 x i16> @v8i16_icmp_uge(<8 x i1
%2 = sext <8 x i1> %1 to <8 x i16>
ret <8 x i16> %2
; SSE2-LABEL: v8i16_icmp_uge:
-; SSE2: movdqa {{.*}}(%rip), %xmm2
-; SEE2: pxor %xmm2, %xmm0
-; SSE2: pxor %xmm1, %xmm2
-; SSE2: pcmpgtw %xmm0, %xmm2
-; SSE2: pcmpeqd %xmm0, %xmm0
-; SSE2: pxor %xmm2, %xmm0
+; SSE2: psubusw %xmm0, %xmm1
+; SEE2: pxor %xmm0, %xmm0
+; SSE2: pcmpeqw %xmm1, %xmm0
; SSE41-LABEL: v8i16_icmp_uge:
; SSE41: pmaxuw %xmm0, %xmm1
@@ -63,12 +60,9 @@ define <8 x i16> @v8i16_icmp_ule(<8 x i1
%2 = sext <8 x i1> %1 to <8 x i16>
ret <8 x i16> %2
; SSE2-LABEL: v8i16_icmp_ule:
-; SSE2: movdqa {{.*}}(%rip), %xmm2
-; SSE2: pxor %xmm2, %xmm1
-; SSE2: pxor %xmm2, %xmm0
-; SSE2: pcmpgtw %xmm1, %xmm0
-; SSE2: pcmpeqd %xmm1, %xmm1
-; SSE2: pxor %xmm0, %xmm1
+; SSE2: psubusw %xmm1, %xmm0
+; SSE2: pxor %xmm1, %xmm1
+; SSE2: pcmpeqw %xmm0, %xmm1
; SSE2: movdqa %xmm1, %xmm0
; SSE41-LABEL: v8i16_icmp_ule:
More information about the llvm-commits
mailing list