[llvm] abbe80f - [X86][SSE] Fold HOP(HOP(X,X),HOP(Y,Y)) -> HOP(PERMUTE(HOP(X,Y)),PERMUTE(HOP(X,Y))
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 1 03:54:26 PDT 2021
Author: Simon Pilgrim
Date: 2021-04-01T11:54:10+01:00
New Revision: abbe80fa52c5286ec929116df6a58d793d6f0d0e
URL: https://github.com/llvm/llvm-project/commit/abbe80fa52c5286ec929116df6a58d793d6f0d0e
DIFF: https://github.com/llvm/llvm-project/commit/abbe80fa52c5286ec929116df6a58d793d6f0d0e.diff
LOG: [X86][SSE] Fold HOP(HOP(X,X),HOP(Y,Y)) -> HOP(PERMUTE(HOP(X,Y)),PERMUTE(HOP(X,Y))
For slow-hop targets, attempt to merge HADD/SUB pairs used in chains.
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/test/CodeGen/X86/horizontal-sum.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 25eac02a53a8..fa26c972c919 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -43587,9 +43587,9 @@ static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG,
X86ISD::HSUB == N->getOpcode() || X86ISD::FHSUB == N->getOpcode()) &&
"Unexpected horizontal add/sub opcode");
- // For slow-hop targets, if we have a hop with a single op, see if we already
- // have another user that we can reuse and shuffle the result.
if (!shouldUseHorizontalOp(true, DAG, Subtarget)) {
+ // For slow-hop targets, if we have a hop with a single op, see if we already
+ // have another user that we can reuse and shuffle the result.
MVT VT = N->getSimpleValueType(0);
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
@@ -43614,6 +43614,34 @@ static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG,
}
}
}
+
+ // HOP(HOP'(X,X),HOP'(Y,Y)) -> HOP(PERMUTE(HOP'(X,Y)),PERMUTE(HOP'(X,Y)).
+ if (LHS != RHS && LHS.getOpcode() == N->getOpcode() &&
+ LHS.getOpcode() == RHS.getOpcode() &&
+ LHS.getValueType() == RHS.getValueType()) {
+ SDValue LHS0 = LHS.getOperand(0);
+ SDValue RHS0 = LHS.getOperand(1);
+ SDValue LHS1 = RHS.getOperand(0);
+ SDValue RHS1 = RHS.getOperand(1);
+ if ((LHS0 == RHS0 || LHS0.isUndef() || RHS0.isUndef()) &&
+ (LHS1 == RHS1 || LHS1.isUndef() || RHS1.isUndef())) {
+ SDLoc DL(N);
+ SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(),
+ LHS0.isUndef() ? RHS0 : LHS0,
+ LHS1.isUndef() ? RHS1 : LHS1);
+ MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
+ Res = DAG.getBitcast(ShufVT, Res);
+ SDValue NewLHS =
+ DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
+ getV4X86ShuffleImm8ForMask({0, 1, 0, 1}, DL, DAG));
+ SDValue NewRHS =
+ DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
+ getV4X86ShuffleImm8ForMask({2, 3, 2, 3}, DL, DAG));
+ DAG.ReplaceAllUsesOfValueWith(LHS, DAG.getBitcast(VT, NewLHS));
+ DAG.ReplaceAllUsesOfValueWith(RHS, DAG.getBitcast(VT, NewRHS));
+ return SDValue(N, 0);
+ }
+ }
}
// Try to fold HOP(SHUFFLE(),SHUFFLE()) -> SHUFFLE(HOP()).
diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll
index de0286ed27a4..0b41e6bbad0d 100644
--- a/llvm/test/CodeGen/X86/horizontal-sum.ll
+++ b/llvm/test/CodeGen/X86/horizontal-sum.ll
@@ -206,9 +206,10 @@ define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl
; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,1],xmm5[3,1]
; SSSE3-SLOW-NEXT: addps %xmm1, %xmm3
; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SSSE3-SLOW-NEXT: haddps %xmm6, %xmm6
-; SSSE3-SLOW-NEXT: haddps %xmm7, %xmm7
; SSSE3-SLOW-NEXT: haddps %xmm7, %xmm6
+; SSSE3-SLOW-NEXT: movaps %xmm6, %xmm1
+; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm6[1]
+; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm6
; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,3],xmm6[0,2]
; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm1
; SSSE3-SLOW-NEXT: retq
@@ -250,8 +251,8 @@ define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl
; AVX1-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX1-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-SLOW-NEXT: vhaddps %xmm6, %xmm6, %xmm2
-; AVX1-SLOW-NEXT: vhaddps %xmm7, %xmm7, %xmm3
+; AVX1-SLOW-NEXT: vhaddps %xmm7, %xmm6, %xmm2
+; AVX1-SLOW-NEXT: vpermilpd {{.*#+}} xmm3 = xmm2[1,1]
; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2
; AVX1-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -300,8 +301,8 @@ define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl
; AVX2-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX2-SLOW-NEXT: vhaddps %xmm6, %xmm6, %xmm2
-; AVX2-SLOW-NEXT: vhaddps %xmm7, %xmm7, %xmm3
+; AVX2-SLOW-NEXT: vhaddps %xmm7, %xmm6, %xmm2
+; AVX2-SLOW-NEXT: vpermilpd {{.*#+}} xmm3 = xmm2[1,1]
; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
@@ -393,10 +394,10 @@ define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2,
; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm3[2,0]
; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm2
; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SSSE3-SLOW-NEXT: phaddd %xmm6, %xmm6
-; SSSE3-SLOW-NEXT: phaddd %xmm7, %xmm7
; SSSE3-SLOW-NEXT: phaddd %xmm7, %xmm6
-; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,3],xmm6[0,2]
+; SSSE3-SLOW-NEXT: phaddd %xmm6, %xmm6
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,1,3,3]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,3],xmm1[0,2]
; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm1
; SSSE3-SLOW-NEXT: retq
;
@@ -442,10 +443,8 @@ define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2,
; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-SLOW-NEXT: vphaddd %xmm6, %xmm6, %xmm2
-; AVX1-SLOW-NEXT: vphaddd %xmm7, %xmm7, %xmm3
-; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX1-SLOW-NEXT: vphaddd %xmm7, %xmm6, %xmm2
+; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm2
; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
; AVX1-SLOW-NEXT: retq
@@ -494,10 +493,8 @@ define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2,
; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vphaddd %xmm6, %xmm6, %xmm1
-; AVX2-SLOW-NEXT: vphaddd %xmm7, %xmm7, %xmm2
-; AVX2-SLOW-NEXT: vphaddd %xmm2, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT: vphaddd %xmm7, %xmm6, %xmm1
+; AVX2-SLOW-NEXT: vphaddd %xmm0, %xmm1, %xmm1
; AVX2-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX2-SLOW-NEXT: retq
More information about the llvm-commits
mailing list