<div dir="ltr">There's a lot more whitespace change than actual patch here, could you split those up in the future?<br><br><div>Thanks.</div><div><br></div><div>-eric</div></div><br><div class="gmail_quote">On Wed, Mar 18, 2015 at 3:24 PM Simon Pilgrim <<a href="mailto:llvm-dev@redking.me.uk">llvm-dev@redking.me.uk</a>> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: rksimon<br>
Date: Wed Mar 18 17:18:51 2015<br>
New Revision: 232682<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=232682&view=rev" target="_blank">http://llvm.org/viewvc/llvm-<u></u>project?rev=232682&view=rev</a><br>
Log:<br>
[X86][SSE] Avoid scalarization of v2i64 vector shifts (REAPPLIED)<br>
<br>
Fixed broken tests.<br>
<br>
Differential Revision: <a href="http://reviews.llvm.org/D8416" target="_blank">http://reviews.llvm.org/D8416</a><br>
<br>
Modified:<br>
llvm/trunk/lib/Target/X86/<u></u>X86ISelLowering.cpp<br>
llvm/trunk/test/Analysis/<u></u>CostModel/X86/testshiftlshr.ll<br>
llvm/trunk/test/Analysis/<u></u>CostModel/X86/testshiftshl.ll<br>
llvm/trunk/test/CodeGen/X86/<u></u>vshift-4.ll<br>
llvm/trunk/test/CodeGen/X86/<u></u>x86-shifts.ll<br>
<br>
Modified: llvm/trunk/lib/Target/X86/<u></u>X86ISelLowering.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86ISelLowering.cpp?rev=232682&r1=232681&r2=232682&view=diff" target="_blank">http://llvm.org/viewvc/llvm-<u></u>project/llvm/trunk/lib/Target/<u></u>X86/X86ISelLowering.cpp?rev=<u></u>232682&r1=232681&r2=232682&<u></u>view=diff</a><br>
==============================<u></u>==============================<u></u>==================<br>
--- llvm/trunk/lib/Target/X86/<u></u>X86ISelLowering.cpp (original)<br>
+++ llvm/trunk/lib/Target/X86/<u></u>X86ISelLowering.cpp Wed Mar 18 17:18:51 2015<br>
@@ -5906,7 +5906,7 @@ static SDValue LowerCONCAT_VECTORSvXi1(S<br>
return DAG.getNode(ISD::OR, dl, ResVT, V1, V2);<br>
}<br>
<br>
-static SDValue LowerCONCAT_VECTORS(SDValue Op,<br>
+static SDValue LowerCONCAT_VECTORS(SDValue Op,<br>
const X86Subtarget *Subtarget,<br>
SelectionDAG &DAG) {<br>
MVT VT = Op.getSimpleValueType();<br>
@@ -13255,11 +13255,11 @@ SDValue X86TargetLowering::<u></u>LowerSELECT(S<br>
// If we have AVX, we can use a variable vector select (VBLENDV) instead<br>
// of 3 logic instructions for size savings and potentially speed.<br>
// Unfortunately, there is no scalar form of VBLENDV.<br>
-<br>
+<br>
// If either operand is a constant, don't try this. We can expect to<br>
// optimize away at least one of the logic instructions later in that<br>
// case, so that sequence would be faster than a variable blend.<br>
-<br>
+<br>
// BLENDV was introduced with SSE 4.1, but the 2 register form implicitly<br>
// uses XMM0 as the selection register. That may need just as many<br>
// instructions as the AND/ANDN/OR sequence due to register moves, so<br>
@@ -13267,10 +13267,10 @@ SDValue X86TargetLowering::<u></u>LowerSELECT(S<br>
<br>
if (Subtarget->hasAVX() &&<br>
!isa<ConstantFPSDNode>(Op1) && !isa<ConstantFPSDNode>(Op2)) {<br>
-<br>
+<br>
// Convert to vectors, do a VSELECT, and convert back to scalar.<br>
// All of the conversions should be optimized away.<br>
-<br>
+<br>
EVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64;<br>
SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_<u></u>VECTOR, DL, VecVT, Op1);<br>
SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_<u></u>VECTOR, DL, VecVT, Op2);<br>
@@ -13278,9 +13278,9 @@ SDValue X86TargetLowering::<u></u>LowerSELECT(S<br>
<br>
EVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64;<br>
VCmp = DAG.getNode(ISD::BITCAST, DL, VCmpVT, VCmp);<br>
-<br>
+<br>
SDValue VSel = DAG.getNode(ISD::VSELECT, DL, VecVT, VCmp, VOp1, VOp2);<br>
-<br>
+<br>
return DAG.getNode(ISD::EXTRACT_<u></u>VECTOR_ELT, DL, VT,<br>
VSel, DAG.getIntPtrConstant(0));<br>
}<br>
@@ -16189,6 +16189,17 @@ static SDValue LowerShift(SDValue Op, co<br>
return Op;<br>
}<br>
<br>
+ // 2i64 vector logical shifts can efficiently avoid scalarization - do the<br>
+ // shifts per-lane and then shuffle the partial results back together.<br>
+ if (VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) {<br>
+ // Splat the shift amounts so the scalar shifts above will catch it.<br>
+ SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});<br>
+ SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});<br>
+ SDValue R0 = DAG.getNode(Op->getOpcode(), dl, VT, R, Amt0);<br>
+ SDValue R1 = DAG.getNode(Op->getOpcode(), dl, VT, R, Amt1);<br>
+ return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});<br>
+ }<br>
+<br>
// If possible, lower this packed shift into a vector multiply instead of<br>
// expanding it into a sequence of scalar shifts.<br>
// Do this only if the vector shift count is a constant build_vector.<br>
@@ -21960,7 +21971,7 @@ static SDValue VectorZextCombine(SDNode<br>
// an and with a mask.<br>
// We'd like to try to combine that into a shuffle with zero<br>
// plus a bitcast, removing the and.<br>
- if (N0.getOpcode() != ISD::BITCAST ||<br>
+ if (N0.getOpcode() != ISD::BITCAST ||<br>
N0.getOperand(0).getOpcode() != ISD::VECTOR_SHUFFLE)<br>
return SDValue();<br>
<br>
@@ -21990,7 +22001,7 @@ static SDValue VectorZextCombine(SDNode<br>
<br>
unsigned ResSize = N1.getValueType().<u></u>getScalarSizeInBits();<br>
// Make sure the splat matches the mask we expect<br>
- if (SplatBitSize > ResSize ||<br>
+ if (SplatBitSize > ResSize ||<br>
(SplatValue + 1).exactLogBase2() != (int)SrcSize)<br>
return SDValue();<br>
<br>
@@ -22948,7 +22959,7 @@ static SDValue PerformFANDCombine(SDNode<br>
if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N-><u></u>getOperand(1)))<br>
if (C->getValueAPF().isPosZero())<br>
return N->getOperand(1);<br>
-<br>
+<br>
return SDValue();<br>
}<br>
<br>
@@ -23222,7 +23233,7 @@ static SDValue PerformISDSETCCCombine(SD<br>
return DAG.getConstant(1, VT);<br>
if (CC == ISD::SETEQ || CC == ISD::SETGE)<br>
return DAG.getNOT(DL, LHS.getOperand(0), VT);<br>
-<br>
+<br>
assert((CC == ISD::SETNE || CC == ISD::SETLT) &&<br>
"Unexpected condition code!");<br>
return LHS.getOperand(0);<br>
@@ -23264,7 +23275,7 @@ static SDValue PerformINSERTPSCombine(SD<br>
// countS and just gets an f32 from that address.<br>
unsigned DestIndex =<br>
cast<ConstantSDNode>(N-><u></u>getOperand(2))->getZExtValue() >> 6;<br>
-<br>
+<br>
Ld = NarrowVectorLoadToElement(<u></u>cast<LoadSDNode>(Ld), DestIndex, DAG);<br>
<br>
// Create this as a scalar to vector to match the instruction pattern.<br>
@@ -23288,7 +23299,7 @@ static SDValue PerformBLENDICombine(SDNo<br>
// pattern-matching possibilities related to scalar math ops in SSE/AVX.<br>
// x86InstrInfo knows how to commute this back after instruction selection<br>
// if it would help register allocation.<br>
-<br>
+<br>
// TODO: If optimizing for size or a processor that doesn't suffer from<br>
// partial register update stalls, this should be transformed into a MOVSD<br>
// instruction because a MOVSD is 1-2 bytes smaller than a BLENDPD.<br>
<br>
Modified: llvm/trunk/test/Analysis/<u></u>CostModel/X86/testshiftlshr.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Analysis/CostModel/X86/testshiftlshr.ll?rev=232682&r1=232681&r2=232682&view=diff" target="_blank">http://llvm.org/viewvc/llvm-<u></u>project/llvm/trunk/test/<u></u>Analysis/CostModel/X86/<u></u>testshiftlshr.ll?rev=232682&<u></u>r1=232681&r2=232682&view=diff</a><br>
==============================<u></u>==============================<u></u>==================<br>
--- llvm/trunk/test/Analysis/<u></u>CostModel/X86/testshiftlshr.ll (original)<br>
+++ llvm/trunk/test/Analysis/<u></u>CostModel/X86/testshiftlshr.ll Wed Mar 18 17:18:51 2015<br>
@@ -7,7 +7,7 @@ entry:<br>
; SSE2: shift2i16<br>
; SSE2: cost of 20 {{.*}} lshr<br>
; SSE2-CODEGEN: shift2i16<br>
- ; SSE2-CODEGEN: shrq %cl<br>
+ ; SSE2-CODEGEN: psrlq<br>
<br>
%0 = lshr %shifttype %a , %b<br>
ret %shifttype %0<br>
@@ -67,7 +67,7 @@ entry:<br>
; SSE2: shift2i32<br>
; SSE2: cost of 20 {{.*}} lshr<br>
; SSE2-CODEGEN: shift2i32<br>
- ; SSE2-CODEGEN: shrq %cl<br>
+ ; SSE2-CODEGEN: psrlq<br>
<br>
%0 = lshr %shifttype2i32 %a , %b<br>
ret %shifttype2i32 %0<br>
@@ -127,7 +127,7 @@ entry:<br>
; SSE2: shift2i64<br>
; SSE2: cost of 20 {{.*}} lshr<br>
; SSE2-CODEGEN: shift2i64<br>
- ; SSE2-CODEGEN: shrq %cl<br>
+ ; SSE2-CODEGEN: psrlq<br>
<br>
%0 = lshr %shifttype2i64 %a , %b<br>
ret %shifttype2i64 %0<br>
@@ -139,7 +139,7 @@ entry:<br>
; SSE2: shift4i64<br>
; SSE2: cost of 40 {{.*}} lshr<br>
; SSE2-CODEGEN: shift4i64<br>
- ; SSE2-CODEGEN: shrq %cl<br>
+ ; SSE2-CODEGEN: psrlq<br>
<br>
%0 = lshr %shifttype4i64 %a , %b<br>
ret %shifttype4i64 %0<br>
@@ -151,7 +151,7 @@ entry:<br>
; SSE2: shift8i64<br>
; SSE2: cost of 80 {{.*}} lshr<br>
; SSE2-CODEGEN: shift8i64<br>
- ; SSE2-CODEGEN: shrq %cl<br>
+ ; SSE2-CODEGEN: psrlq<br>
<br>
%0 = lshr %shifttype8i64 %a , %b<br>
ret %shifttype8i64 %0<br>
@@ -163,7 +163,7 @@ entry:<br>
; SSE2: shift16i64<br>
; SSE2: cost of 160 {{.*}} lshr<br>
; SSE2-CODEGEN: shift16i64<br>
- ; SSE2-CODEGEN: shrq %cl<br>
+ ; SSE2-CODEGEN: psrlq<br>
<br>
%0 = lshr %shifttype16i64 %a , %b<br>
ret %shifttype16i64 %0<br>
@@ -175,7 +175,7 @@ entry:<br>
; SSE2: shift32i64<br>
; SSE2: cost of 320 {{.*}} lshr<br>
; SSE2-CODEGEN: shift32i64<br>
- ; SSE2-CODEGEN: shrq %cl<br>
+ ; SSE2-CODEGEN: psrlq<br>
<br>
%0 = lshr %shifttype32i64 %a , %b<br>
ret %shifttype32i64 %0<br>
@@ -187,7 +187,7 @@ entry:<br>
; SSE2: shift2i8<br>
; SSE2: cost of 20 {{.*}} lshr<br>
; SSE2-CODEGEN: shift2i8<br>
- ; SSE2-CODEGEN: shrq %cl<br>
+ ; SSE2-CODEGEN: psrlq<br>
<br>
%0 = lshr %shifttype2i8 %a , %b<br>
ret %shifttype2i8 %0<br>
<br>
Modified: llvm/trunk/test/Analysis/<u></u>CostModel/X86/testshiftshl.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Analysis/CostModel/X86/testshiftshl.ll?rev=232682&r1=232681&r2=232682&view=diff" target="_blank">http://llvm.org/viewvc/llvm-<u></u>project/llvm/trunk/test/<u></u>Analysis/CostModel/X86/<u></u>testshiftshl.ll?rev=232682&r1=<u></u>232681&r2=232682&view=diff</a><br>
==============================<u></u>==============================<u></u>==================<br>
--- llvm/trunk/test/Analysis/<u></u>CostModel/X86/testshiftshl.ll (original)<br>
+++ llvm/trunk/test/Analysis/<u></u>CostModel/X86/testshiftshl.ll Wed Mar 18 17:18:51 2015<br>
@@ -7,7 +7,7 @@ entry:<br>
; SSE2: shift2i16<br>
; SSE2: cost of 20 {{.*}} shl<br>
; SSE2-CODEGEN: shift2i16<br>
- ; SSE2-CODEGEN: shlq %cl<br>
+ ; SSE2-CODEGEN: psllq<br>
<br>
%0 = shl %shifttype %a , %b<br>
ret %shifttype %0<br>
@@ -67,7 +67,7 @@ entry:<br>
; SSE2: shift2i32<br>
; SSE2: cost of 20 {{.*}} shl<br>
; SSE2-CODEGEN: shift2i32<br>
- ; SSE2-CODEGEN: shlq %cl<br>
+ ; SSE2-CODEGEN: psllq<br>
<br>
%0 = shl %shifttype2i32 %a , %b<br>
ret %shifttype2i32 %0<br>
@@ -127,7 +127,7 @@ entry:<br>
; SSE2: shift2i64<br>
; SSE2: cost of 20 {{.*}} shl<br>
; SSE2-CODEGEN: shift2i64<br>
- ; SSE2-CODEGEN: shlq %cl<br>
+ ; SSE2-CODEGEN: psllq<br>
<br>
%0 = shl %shifttype2i64 %a , %b<br>
ret %shifttype2i64 %0<br>
@@ -139,7 +139,7 @@ entry:<br>
; SSE2: shift4i64<br>
; SSE2: cost of 40 {{.*}} shl<br>
; SSE2-CODEGEN: shift4i64<br>
- ; SSE2-CODEGEN: shlq %cl<br>
+ ; SSE2-CODEGEN: psllq<br>
<br>
%0 = shl %shifttype4i64 %a , %b<br>
ret %shifttype4i64 %0<br>
@@ -151,7 +151,7 @@ entry:<br>
; SSE2: shift8i64<br>
; SSE2: cost of 80 {{.*}} shl<br>
; SSE2-CODEGEN: shift8i64<br>
- ; SSE2-CODEGEN: shlq %cl<br>
+ ; SSE2-CODEGEN: psllq<br>
<br>
%0 = shl %shifttype8i64 %a , %b<br>
ret %shifttype8i64 %0<br>
@@ -163,7 +163,7 @@ entry:<br>
; SSE2: shift16i64<br>
; SSE2: cost of 160 {{.*}} shl<br>
; SSE2-CODEGEN: shift16i64<br>
- ; SSE2-CODEGEN: shlq %cl<br>
+ ; SSE2-CODEGEN: psllq<br>
<br>
%0 = shl %shifttype16i64 %a , %b<br>
ret %shifttype16i64 %0<br>
@@ -175,7 +175,7 @@ entry:<br>
; SSE2: shift32i64<br>
; SSE2: cost of 320 {{.*}} shl<br>
; SSE2-CODEGEN: shift32i64<br>
- ; SSE2-CODEGEN: shlq %cl<br>
+ ; SSE2-CODEGEN: psllq<br>
<br>
%0 = shl %shifttype32i64 %a , %b<br>
ret %shifttype32i64 %0<br>
@@ -187,7 +187,7 @@ entry:<br>
; SSE2: shift2i8<br>
; SSE2: cost of 20 {{.*}} shl<br>
; SSE2-CODEGEN: shift2i8<br>
- ; SSE2-CODEGEN: shlq %cl<br>
+ ; SSE2-CODEGEN: psllq<br>
<br>
%0 = shl %shifttype2i8 %a , %b<br>
ret %shifttype2i8 %0<br>
<br>
Modified: llvm/trunk/test/CodeGen/X86/<u></u>vshift-4.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/vshift-4.ll?rev=232682&r1=232681&r2=232682&view=diff" target="_blank">http://llvm.org/viewvc/llvm-<u></u>project/llvm/trunk/test/<u></u>CodeGen/X86/vshift-4.ll?rev=<u></u>232682&r1=232681&r2=232682&<u></u>view=diff</a><br>
==============================<u></u>==============================<u></u>==================<br>
--- llvm/trunk/test/CodeGen/X86/<u></u>vshift-4.ll (original)<br>
+++ llvm/trunk/test/CodeGen/X86/<u></u>vshift-4.ll Wed Mar 18 17:18:51 2015<br>
@@ -13,11 +13,16 @@ entry:<br>
ret void<br>
}<br>
<br>
-; shift1b can't use a packed shift<br>
+; shift1b can't use a packed shift but can shift lanes separately and shuffle back together<br>
define void @shift1b(<2 x i64> %val, <2 x i64>* %dst, <2 x i64> %sh) nounwind {<br>
entry:<br>
; CHECK-LABEL: shift1b:<br>
-; CHECK: shll<br>
+; CHECK: pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]<br>
+; CHECK-NEXT: movdqa %xmm0, %xmm3<br>
+; CHECK-NEXT: psllq %xmm2, %xmm3<br>
+; CHECK-NEXT: movq {{.*#+}} xmm1 = xmm1[0],zero<br>
+; CHECK-NEXT: psllq %xmm1, %xmm0<br>
+; CHECK-NEXT: movsd {{.*#+}} xmm3 = xmm0[0],xmm3[1]<br>
%shamt = shufflevector <2 x i64> %sh, <2 x i64> undef, <2 x i32> <i32 0, i32 1><br>
%shl = shl <2 x i64> %val, %shamt<br>
store <2 x i64> %shl, <2 x i64>* %dst<br>
<br>
Modified: llvm/trunk/test/CodeGen/X86/<u></u>x86-shifts.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/x86-shifts.ll?rev=232682&r1=232681&r2=232682&view=diff" target="_blank">http://llvm.org/viewvc/llvm-<u></u>project/llvm/trunk/test/<u></u>CodeGen/X86/x86-shifts.ll?rev=<u></u>232682&r1=232681&r2=232682&<u></u>view=diff</a><br>
==============================<u></u>==============================<u></u>==================<br>
--- llvm/trunk/test/CodeGen/X86/<u></u>x86-shifts.ll (original)<br>
+++ llvm/trunk/test/CodeGen/X86/<u></u>x86-shifts.ll Wed Mar 18 17:18:51 2015<br>
@@ -118,10 +118,15 @@ entry:<br>
<br>
define <2 x i64> @shr2_nosplat(<2 x i64> %A) nounwind {<br>
entry:<br>
-; CHECK: shr2_nosplat<br>
-; CHECK-NOT: psrlq<br>
-; CHECK-NOT: psrlq<br>
-; CHECK: ret<br>
+; CHECK-LABEL: shr2_nosplat<br>
+; CHECK: movdqa %xmm1, %xmm2<br>
+; CHECK-NEXT: psrlq $8, %xmm2<br>
+; CHECK-NEXT: movdqa %xmm1, %xmm0<br>
+; CHECK-NEXT: psrlq $1, %xmm0<br>
+; CHECK-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]<br>
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]<br>
+; CHECK-NEXT: xorpd %xmm1, %xmm0<br>
+; CHECK-NEXT: ret<br>
%B = lshr <2 x i64> %A, < i64 8, i64 1><br>
%C = lshr <2 x i64> %A, < i64 1, i64 0><br>
%K = xor <2 x i64> %B, %C<br>
<br>
<br>
______________________________<u></u>_________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@cs.uiuc.edu" target="_blank">llvm-commits@cs.uiuc.edu</a><br>
<a href="http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits" target="_blank">http://lists.cs.uiuc.edu/<u></u>mailman/listinfo/llvm-commits</a><br>
</blockquote></div>