[PATCH][X86] Add target combine rules for horizontal add/sub.

Andrea Di Biagio andrea.dibiagio at gmail.com
Fri Jun 6 12:50:43 PDT 2014


Hi,

This patch adds a new target specific combine rule to identify
horizontal add/sub idioms from BUILD_VECTOR dag nodes where operands
are integer/float add/sub between adjacent elements extracted from a
same vector.

Here is an example:

////
typedef double double2 __attribute__((ext_vector_type(2)));

double2 test(double2 A, double2 B) {
  return (double2){A[0] + A[1], B[0] + B[1]};
}
////

Produces the following IR:

define <2 x double> @test(<2 x double> %A, <2 x double> %B) {
  %vecext = extractelement <2 x double> %A, i32 0
  %vecext1 = extractelement <2 x double> %A, i32 1
  %add = fadd double %vecext, %vecext1
  %vecinit = insertelement <2 x double> undef, double %add, i32 0
  %vecext2 = extractelement <2 x double> %B, i32 0
  %vecext3 = extractelement <2 x double> %B, i32 1
  %add2 = fadd double %vecext2, %vecext3
  %vecinit2 = insertelement <2 x double> %vecinit, double %add2, i32 1
  ret <2 x double> %vecinit2
}

Before this patch, function @test was emitted as a long sequence of
extract+fadd+insert instructions. With this patch, the backend
produces a single horizontal float add.

To simplify the logic of the new target combine rule, I added a target
independent combine rule to canonicalize sequences of
insert_vector_elt dag nodes:

canonicalize: (insert_vector_elt (insert_vector_elt A, Idx0), Idx1) ->
              (insert_vector_elt (insert_vector_elt A, Idx1), Idx0)

This new canonicalization rule only triggers if the inner
insert_vector node has exactly one use; also, both indices must be
known constants, and 'Idx1' has to be less than 'Idx0'. This strongly
reduces the complexity of the new target combine rule algorithm; it
makes easier to identify horizontal add/sub from build_vector dag
nodes.

Ideally, this new target combine rule should be run immediately after
"type legalize". However, the backend never re-runs the dag combiner
if all nodes already have a 'Legal' type.
Unfortunately, there are many cases (see for example the tests added
by this patch) where the dag combiner is not re-run on the
"type-legalized" dag. It is easy to match horizontal add/sub before
operations are legalized; it is hard instead to write generic rules to
match the already legalized dag due to the presence of target specific
dag nodes (which may vary depending on the Subtarget features and the
type of horizontal operation being performed).

Please let me know if ok to submit.

Thanks!
Andrea Di Biagio
-------------- next part --------------
Index: lib/Target/X86/X86ISelLowering.cpp
===================================================================
--- lib/Target/X86/X86ISelLowering.cpp	(revision 210366)
+++ lib/Target/X86/X86ISelLowering.cpp	(working copy)
@@ -1560,6 +1560,7 @@
   setTargetDAGCombine(ISD::SINT_TO_FP);
   setTargetDAGCombine(ISD::SETCC);
   setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
+  setTargetDAGCombine(ISD::BUILD_VECTOR);
   if (Subtarget->is64Bit())
     setTargetDAGCombine(ISD::MUL);
   setTargetDAGCombine(ISD::XOR);
@@ -6047,6 +6048,89 @@
   return DAG.getNode(ISD::BITCAST, dl, VT, Select);
 }
 
+static SDValue PerformBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
+                                          const X86Subtarget *Subtarget) {
+  EVT VT = N->getValueType(0);
+
+  // Try to match a horizontal ADD or SUB.
+  if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) ||
+      ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget->hasAVX()) ||
+      ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) ||
+      ((VT == MVT::v8i32 || VT == MVT::v16i16) && Subtarget->hasAVX2())) {
+    unsigned NumOperands = N->getNumOperands();
+    unsigned Opcode = N->getOperand(0)->getOpcode();
+    bool isCommutable = false;
+    bool CanFold = false;
+    switch (Opcode) {
+    default : break;
+    case ISD::ADD :
+    case ISD::FADD :
+      isCommutable = true;
+      // FALL-THROUGH
+    case ISD::SUB :
+    case ISD::FSUB :
+      CanFold = true;
+    }
+
+    // Verify that operands have the same opcode; also, the opcode can only
+    // be either of: ADD, FADD, SUB, FSUB.
+    SDValue InVec0, InVec1;
+    for (unsigned i = 0, e = NumOperands; i != e && CanFold; ++i) {
+      SDValue Op = N->getOperand(i);
+      CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
+
+      if (!CanFold)
+        break;
+
+      SDValue Op0 = Op.getOperand(0);
+      SDValue Op1 = Op.getOperand(1);
+
+      // Try to match the following pattern:
+      // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
+      CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+          Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+          Op0.getOperand(0) == Op1.getOperand(0) &&
+          isa<ConstantSDNode>(Op0.getOperand(1)) &&
+          isa<ConstantSDNode>(Op1.getOperand(1)));
+      if (!CanFold)
+        break;
+
+      unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
+      unsigned I1 = cast<ConstantSDNode>(Op1.getOperand(1))->getZExtValue();
+      unsigned ExpectedIndex = (i * 2) % NumOperands;
+ 
+      if (i == 0)
+        InVec0 = Op0.getOperand(0);
+      else if (i * 2 == NumOperands)
+        InVec1 = Op0.getOperand(0);
+
+      SDValue Expected = (i * 2 < NumOperands) ? InVec0 : InVec1;
+      if (I0 == ExpectedIndex)
+        CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
+      else if (isCommutable && I1 == ExpectedIndex) {
+        // Try to see if we can match the following dag sequence:
+        // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
+        CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
+      }
+    }
+
+    if (CanFold) {
+      unsigned NewOpcode;
+      switch (Opcode) {
+      default : llvm_unreachable("Unexpected opcode found!");
+      case ISD::ADD : NewOpcode = X86ISD::HADD; break;
+      case ISD::FADD : NewOpcode = X86ISD::FHADD; break;
+      case ISD::SUB : NewOpcode = X86ISD::HSUB; break;
+      case ISD::FSUB : NewOpcode = X86ISD::FHSUB; break;
+      }
+ 
+      return DAG.getNode(NewOpcode, SDLoc(N), VT, InVec0, InVec1);
+    }
+  }
+
+  return SDValue();
+}
+
 SDValue
 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   SDLoc dl(Op);
@@ -20721,6 +20805,7 @@
     return PerformINTRINSIC_WO_CHAINCombine(N, DAG, Subtarget);
   case X86ISD::INSERTPS:
     return PerformINSERTPSCombine(N, DAG, Subtarget);
+  case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DAG, Subtarget);
   }
 
   return SDValue();
Index: lib/CodeGen/SelectionDAG/DAGCombiner.cpp
===================================================================
--- lib/CodeGen/SelectionDAG/DAGCombiner.cpp	(revision 210366)
+++ lib/CodeGen/SelectionDAG/DAGCombiner.cpp	(working copy)
@@ -9714,6 +9714,27 @@
     return SDValue();
   unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
 
+  // Canonicalize insert_vector_elt dag nodes.
+  // Example:
+  // (insert_vector_elt (insert_vector_elt A, Idx0), Idx1)
+  // -> (insert_vector_elt (insert_vector_elt A, Idx1), Idx0)
+  //
+  // Do this only if the child insert_vector node has one use; also
+  // do this only if indices are both constants and Idx1 < Idx0.
+  if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT && InVec.hasOneUse()
+      && isa<ConstantSDNode>(InVec.getOperand(2))) {
+    unsigned OtherElt =
+      cast<ConstantSDNode>(InVec.getOperand(2))->getZExtValue();
+    if (Elt < OtherElt) {
+      // Swap nodes.
+      SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), VT,
+                                  InVec.getOperand(0), InVal, EltNo);
+      AddToWorkList(NewOp.getNode());
+      return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()),
+                         VT, NewOp, InVec.getOperand(1), InVec.getOperand(2));
+    }
+  }
+
   // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
   // be converted to a BUILD_VECTOR).  Fill in the Ops vector with the
   // vector elements.
Index: test/CodeGen/X86/haddsub-2.ll
===================================================================
--- test/CodeGen/X86/haddsub-2.ll	(revision 0)
+++ test/CodeGen/X86/haddsub-2.ll	(working copy)
@@ -0,0 +1,376 @@
+; RUN: llc < %s -march=x86-64 -mattr=+sse2,+sse3 | FileCheck %s -check-prefix=CHECK -check-prefix=SSE3
+; RUN: llc < %s -march=x86-64 -mattr=+sse2,+sse3,+ssse3 | FileCheck %s -check-prefix=CHECK -check-prefix=SSSE3
+; RUN: llc < %s -march=x86-64 -mcpu=corei7-avx | FileCheck %s -check-prefix=CHECK -check-prefix=AVX
+; RUN: llc < %s -march=x86-64 -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK -check-prefix=AVX2
+
+
+
+define <4 x float> @hadd_ps_test1(<4 x float> %A, <4 x float> %B) {
+  %vecext = extractelement <4 x float> %A, i32 0
+  %vecext1 = extractelement <4 x float> %A, i32 1
+  %add = fadd float %vecext, %vecext1
+  %vecinit = insertelement <4 x float> undef, float %add, i32 0
+  %vecext2 = extractelement <4 x float> %A, i32 2
+  %vecext3 = extractelement <4 x float> %A, i32 3
+  %add4 = fadd float %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x float> %vecinit, float %add4, i32 1
+  %vecext6 = extractelement <4 x float> %B, i32 0
+  %vecext7 = extractelement <4 x float> %B, i32 1
+  %add8 = fadd float %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x float> %vecinit5, float %add8, i32 2
+  %vecext10 = extractelement <4 x float> %B, i32 2
+  %vecext11 = extractelement <4 x float> %B, i32 3
+  %add12 = fadd float %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x float> %vecinit9, float %add12, i32 3
+  ret <4 x float> %vecinit13
+}
+; CHECK-LABEL: hadd_ps_test1
+; CHECK: haddps
+; CHECK-NEXT: ret
+
+
+define <4 x float> @hadd_ps_test2(<4 x float> %A, <4 x float> %B) {
+  %vecext = extractelement <4 x float> %A, i32 2
+  %vecext1 = extractelement <4 x float> %A, i32 3
+  %add = fadd float %vecext, %vecext1
+  %vecinit = insertelement <4 x float> undef, float %add, i32 1
+  %vecext2 = extractelement <4 x float> %A, i32 0
+  %vecext3 = extractelement <4 x float> %A, i32 1
+  %add4 = fadd float %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x float> %vecinit, float %add4, i32 0
+  %vecext6 = extractelement <4 x float> %B, i32 2
+  %vecext7 = extractelement <4 x float> %B, i32 3
+  %add8 = fadd float %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x float> %vecinit5, float %add8, i32 3
+  %vecext10 = extractelement <4 x float> %B, i32 0
+  %vecext11 = extractelement <4 x float> %B, i32 1
+  %add12 = fadd float %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x float> %vecinit9, float %add12, i32 2
+  ret <4 x float> %vecinit13
+}
+; CHECK-LABEL: hadd_ps_test2
+; CHECK: haddps
+; CHECK-NEXT: ret
+
+
+define <4 x float> @hsub_ps_test1(<4 x float> %A, <4 x float> %B) {
+  %vecext = extractelement <4 x float> %A, i32 0
+  %vecext1 = extractelement <4 x float> %A, i32 1
+  %sub = fsub float %vecext, %vecext1
+  %vecinit = insertelement <4 x float> undef, float %sub, i32 0
+  %vecext2 = extractelement <4 x float> %A, i32 2
+  %vecext3 = extractelement <4 x float> %A, i32 3
+  %sub4 = fsub float %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x float> %vecinit, float %sub4, i32 1
+  %vecext6 = extractelement <4 x float> %B, i32 0
+  %vecext7 = extractelement <4 x float> %B, i32 1
+  %sub8 = fsub float %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x float> %vecinit5, float %sub8, i32 2
+  %vecext10 = extractelement <4 x float> %B, i32 2
+  %vecext11 = extractelement <4 x float> %B, i32 3
+  %sub12 = fsub float %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x float> %vecinit9, float %sub12, i32 3
+  ret <4 x float> %vecinit13
+}
+; CHECK-LABEL: hsub_ps_test1
+; CHECK: hsubps
+; CHECK-NEXT: ret
+
+
+define <4 x float> @hsub_ps_test2(<4 x float> %A, <4 x float> %B) {
+  %vecext = extractelement <4 x float> %A, i32 2
+  %vecext1 = extractelement <4 x float> %A, i32 3
+  %sub = fsub float %vecext, %vecext1
+  %vecinit = insertelement <4 x float> undef, float %sub, i32 1
+  %vecext2 = extractelement <4 x float> %A, i32 0
+  %vecext3 = extractelement <4 x float> %A, i32 1
+  %sub4 = fsub float %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x float> %vecinit, float %sub4, i32 0
+  %vecext6 = extractelement <4 x float> %B, i32 3
+  %vecext7 = extractelement <4 x float> %B, i32 2
+  %sub8 = fsub float %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x float> %vecinit5, float %sub8, i32 3
+  %vecext10 = extractelement <4 x float> %B, i32 1
+  %vecext11 = extractelement <4 x float> %B, i32 0
+  %sub12 = fsub float %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x float> %vecinit9, float %sub12, i32 2
+  ret <4 x float> %vecinit13
+}
+; CHECK-LABEL: hsub_ps_test2
+; CHECK: hsubps
+; CHECK-NEXT: ret
+
+
+define <4 x i32> @phadd_d_test1(<4 x i32> %A, <4 x i32> %B) {
+  %vecext = extractelement <4 x i32> %A, i32 0
+  %vecext1 = extractelement <4 x i32> %A, i32 1
+  %add = add i32 %vecext, %vecext1
+  %vecinit = insertelement <4 x i32> undef, i32 %add, i32 0
+  %vecext2 = extractelement <4 x i32> %A, i32 2
+  %vecext3 = extractelement <4 x i32> %A, i32 3
+  %add4 = add i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x i32> %vecinit, i32 %add4, i32 1
+  %vecext6 = extractelement <4 x i32> %B, i32 0
+  %vecext7 = extractelement <4 x i32> %B, i32 1
+  %add8 = add i32 %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x i32> %vecinit5, i32 %add8, i32 2
+  %vecext10 = extractelement <4 x i32> %B, i32 2
+  %vecext11 = extractelement <4 x i32> %B, i32 3
+  %add12 = add i32 %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x i32> %vecinit9, i32 %add12, i32 3
+  ret <4 x i32> %vecinit13
+}
+; CHECK-LABEL: phadd_d_test1
+; SSE3-NOT: phaddd
+; SSSE3: phaddd
+; AVX: vphaddd
+; AVX2 vphaddd
+; CHECK: ret
+
+
+define <4 x i32> @phadd_d_test2(<4 x i32> %A, <4 x i32> %B) {
+  %vecext = extractelement <4 x i32> %A, i32 2
+  %vecext1 = extractelement <4 x i32> %A, i32 3
+  %add = add i32 %vecext, %vecext1
+  %vecinit = insertelement <4 x i32> undef, i32 %add, i32 1
+  %vecext2 = extractelement <4 x i32> %A, i32 0
+  %vecext3 = extractelement <4 x i32> %A, i32 1
+  %add4 = add i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x i32> %vecinit, i32 %add4, i32 0
+  %vecext6 = extractelement <4 x i32> %B, i32 2
+  %vecext7 = extractelement <4 x i32> %B, i32 3
+  %add8 = add i32 %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x i32> %vecinit5, i32 %add8, i32 3
+  %vecext10 = extractelement <4 x i32> %B, i32 0
+  %vecext11 = extractelement <4 x i32> %B, i32 1
+  %add12 = add i32 %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x i32> %vecinit9, i32 %add12, i32 2
+  ret <4 x i32> %vecinit13
+}
+; CHECK-LABEL: phadd_d_test2
+; SSE3-NOT: phaddd
+; SSSE3: phaddd
+; AVX: vphaddd
+; AVX2 vphaddd
+; CHECK: ret
+
+
+define <4 x i32> @phsub_d_test1(<4 x i32> %A, <4 x i32> %B) {
+  %vecext = extractelement <4 x i32> %A, i32 0
+  %vecext1 = extractelement <4 x i32> %A, i32 1
+  %sub = sub i32 %vecext, %vecext1
+  %vecinit = insertelement <4 x i32> undef, i32 %sub, i32 0
+  %vecext2 = extractelement <4 x i32> %A, i32 2
+  %vecext3 = extractelement <4 x i32> %A, i32 3
+  %sub4 = sub i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x i32> %vecinit, i32 %sub4, i32 1
+  %vecext6 = extractelement <4 x i32> %B, i32 0
+  %vecext7 = extractelement <4 x i32> %B, i32 1
+  %sub8 = sub i32 %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x i32> %vecinit5, i32 %sub8, i32 2
+  %vecext10 = extractelement <4 x i32> %B, i32 2
+  %vecext11 = extractelement <4 x i32> %B, i32 3
+  %sub12 = sub i32 %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x i32> %vecinit9, i32 %sub12, i32 3
+  ret <4 x i32> %vecinit13
+}
+; CHECK-LABEL: phsub_d_test1
+; SSE3-NOT: phsubd
+; SSSE3: phsubd
+; AVX: vphsubd
+; AVX2 vphsubd
+; CHECK: ret
+
+
+define <4 x i32> @phsub_d_test2(<4 x i32> %A, <4 x i32> %B) {
+  %vecext = extractelement <4 x i32> %A, i32 2
+  %vecext1 = extractelement <4 x i32> %A, i32 3
+  %sub = sub i32 %vecext, %vecext1
+  %vecinit = insertelement <4 x i32> undef, i32 %sub, i32 1
+  %vecext2 = extractelement <4 x i32> %A, i32 0
+  %vecext3 = extractelement <4 x i32> %A, i32 1
+  %sub4 = sub i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x i32> %vecinit, i32 %sub4, i32 0
+  %vecext6 = extractelement <4 x i32> %B, i32 3
+  %vecext7 = extractelement <4 x i32> %B, i32 2
+  %sub8 = sub i32 %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x i32> %vecinit5, i32 %sub8, i32 3
+  %vecext10 = extractelement <4 x i32> %B, i32 1
+  %vecext11 = extractelement <4 x i32> %B, i32 0
+  %sub12 = sub i32 %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x i32> %vecinit9, i32 %sub12, i32 2
+  ret <4 x i32> %vecinit13
+}
+; CHECK-LABEL: phsub_d_test2
+; SSE3-NOT: phsubd
+; SSSE3: phsubd
+; AVX: vphsubd
+; AVX2 vphsubd
+; CHECK: ret
+
+
+define <2 x double> @hadd_pd_test1(<2 x double> %A, <2 x double> %B) {
+  %vecext = extractelement <2 x double> %A, i32 0
+  %vecext1 = extractelement <2 x double> %A, i32 1
+  %add = fadd double %vecext, %vecext1
+  %vecinit = insertelement <2 x double> undef, double %add, i32 0
+  %vecext2 = extractelement <2 x double> %B, i32 0
+  %vecext3 = extractelement <2 x double> %B, i32 1
+  %add2 = fadd double %vecext2, %vecext3
+  %vecinit2 = insertelement <2 x double> %vecinit, double %add2, i32 1
+  ret <2 x double> %vecinit2
+}
+; CHECK-LABEL: hadd_pd_test1
+; CHECK: haddpd
+; CHECK-NEXT: ret
+
+
+define <2 x double> @hadd_pd_test2(<2 x double> %A, <2 x double> %B) {
+  %vecext = extractelement <2 x double> %A, i32 1
+  %vecext1 = extractelement <2 x double> %A, i32 0
+  %add = fadd double %vecext, %vecext1
+  %vecinit = insertelement <2 x double> undef, double %add, i32 0
+  %vecext2 = extractelement <2 x double> %B, i32 1
+  %vecext3 = extractelement <2 x double> %B, i32 0
+  %add2 = fadd double %vecext2, %vecext3
+  %vecinit2 = insertelement <2 x double> %vecinit, double %add2, i32 1
+  ret <2 x double> %vecinit2
+}
+; CHECK-LABEL: hadd_pd_test2
+; CHECK: haddpd
+; CHECK-NEXT: ret
+
+
+define <2 x double> @hsub_pd_test1(<2 x double> %A, <2 x double> %B) {
+  %vecext = extractelement <2 x double> %A, i32 0
+  %vecext1 = extractelement <2 x double> %A, i32 1
+  %sub = fsub double %vecext, %vecext1
+  %vecinit = insertelement <2 x double> undef, double %sub, i32 0
+  %vecext2 = extractelement <2 x double> %B, i32 0
+  %vecext3 = extractelement <2 x double> %B, i32 1
+  %sub2 = fsub double %vecext2, %vecext3
+  %vecinit2 = insertelement <2 x double> %vecinit, double %sub2, i32 1
+  ret <2 x double> %vecinit2
+}
+; CHECK-LABEL: hsub_pd_test1
+; CHECK: hsubpd
+; CHECK-NEXT: ret
+
+
+define <2 x double> @hsub_pd_test2(<2 x double> %A, <2 x double> %B) {
+  %vecext = extractelement <2 x double> %A, i32 1
+  %vecext1 = extractelement <2 x double> %A, i32 0
+  %sub = fsub double %vecext, %vecext1
+  %vecinit = insertelement <2 x double> undef, double %sub, i32 0
+  %vecext2 = extractelement <2 x double> %B, i32 1
+  %vecext3 = extractelement <2 x double> %B, i32 0
+  %sub2 = fsub double %vecext2, %vecext3
+  %vecinit2 = insertelement <2 x double> %vecinit, double %sub2, i32 1
+  ret <2 x double> %vecinit2
+}
+; CHECK-LABEL: hsub_pd_test2
+; CHECK: hsubpd
+; CHECK-NEXT: ret
+
+
+define <4 x double> @avx_vhadd_pd_test(<4 x double> %A, <4 x double> %B) {
+  %vecext = extractelement <4 x double> %A, i32 0
+  %vecext1 = extractelement <4 x double> %A, i32 1
+  %add = fadd double %vecext, %vecext1
+  %vecinit = insertelement <4 x double> undef, double %add, i32 0
+  %vecext2 = extractelement <4 x double> %A, i32 2
+  %vecext3 = extractelement <4 x double> %A, i32 3
+  %add4 = fadd double %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x double> %vecinit, double %add4, i32 1
+  %vecext6 = extractelement <4 x double> %B, i32 0
+  %vecext7 = extractelement <4 x double> %B, i32 1
+  %add8 = fadd double %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x double> %vecinit5, double %add8, i32 2
+  %vecext10 = extractelement <4 x double> %B, i32 2
+  %vecext11 = extractelement <4 x double> %B, i32 3
+  %add12 = fadd double %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x double> %vecinit9, double %add12, i32 3
+  ret <4 x double> %vecinit13
+}
+; CHECK-LABEL: avx_vhadd_pd_test
+; SSE3: haddpd
+; SSE3-NEXT: haddpd
+; SSSE3: haddpd
+; SSSE3: haddpd
+; AVX: vhaddpd
+; AVX2: vhaddpd
+; CHECK: ret
+
+
+define <4 x double> @avx_vhsub_pd_test(<4 x double> %A, <4 x double> %B) {
+  %vecext = extractelement <4 x double> %A, i32 0
+  %vecext1 = extractelement <4 x double> %A, i32 1
+  %sub = fsub double %vecext, %vecext1
+  %vecinit = insertelement <4 x double> undef, double %sub, i32 0
+  %vecext2 = extractelement <4 x double> %A, i32 2
+  %vecext3 = extractelement <4 x double> %A, i32 3
+  %sub4 = fsub double %vecext2, %vecext3
+  %vecinit5 = insertelement <4 x double> %vecinit, double %sub4, i32 1
+  %vecext6 = extractelement <4 x double> %B, i32 0
+  %vecext7 = extractelement <4 x double> %B, i32 1
+  %sub8 = fsub double %vecext6, %vecext7
+  %vecinit9 = insertelement <4 x double> %vecinit5, double %sub8, i32 2
+  %vecext10 = extractelement <4 x double> %B, i32 2
+  %vecext11 = extractelement <4 x double> %B, i32 3
+  %sub12 = fsub double %vecext10, %vecext11
+  %vecinit13 = insertelement <4 x double> %vecinit9, double %sub12, i32 3
+  ret <4 x double> %vecinit13
+}
+; CHECK-LABEL: avx_vhsub_pd_test
+; SSE3: hsubpd
+; SSE3-NEXT: hsubpd
+; SSSE3: hsubpd
+; SSSE3-NEXT: hsubpd
+; AVX: vhsubpd
+; AVX2: vhsubpd
+; CHECK: ret
+
+
+define <8 x i32> @avx2_vphadd_d_test(<8 x i32> %A, <8 x i32> %B) {
+  %vecext = extractelement <8 x i32> %A, i32 0
+  %vecext1 = extractelement <8 x i32> %A, i32 1
+  %add = add i32 %vecext, %vecext1
+  %vecinit = insertelement <8 x i32> undef, i32 %add, i32 0
+  %vecext2 = extractelement <8 x i32> %A, i32 2
+  %vecext3 = extractelement <8 x i32> %A, i32 3
+  %add4 = add i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <8 x i32> %vecinit, i32 %add4, i32 1
+  %vecext6 = extractelement <8 x i32> %A, i32 4
+  %vecext7 = extractelement <8 x i32> %A, i32 5
+  %add8 = add i32 %vecext6, %vecext7
+  %vecinit9 = insertelement <8 x i32> %vecinit5, i32 %add8, i32 2
+  %vecext10 = extractelement <8 x i32> %A, i32 6
+  %vecext11 = extractelement <8 x i32> %A, i32 7
+  %add12 = add i32 %vecext10, %vecext11
+  %vecinit13 = insertelement <8 x i32> %vecinit9, i32 %add12, i32 3
+  %vecext14 = extractelement <8 x i32> %B, i32 0
+  %vecext15 = extractelement <8 x i32> %B, i32 1
+  %add16 = add i32 %vecext14, %vecext15
+  %vecinit17 = insertelement <8 x i32> %vecinit13, i32 %add16, i32 4
+  %vecext18 = extractelement <8 x i32> %B, i32 2
+  %vecext19 = extractelement <8 x i32> %B, i32 3
+  %add20 = add i32 %vecext18, %vecext19
+  %vecinit21 = insertelement <8 x i32> %vecinit17, i32 %add20, i32 5
+  %vecext22 = extractelement <8 x i32> %B, i32 4
+  %vecext23 = extractelement <8 x i32> %B, i32 5
+  %add24 = add i32 %vecext22, %vecext23
+  %vecinit25 = insertelement <8 x i32> %vecinit21, i32 %add24, i32 6
+  %vecext26 = extractelement <8 x i32> %B, i32 6
+  %vecext27 = extractelement <8 x i32> %B, i32 7
+  %add28 = add i32 %vecext26, %vecext27
+  %vecinit29 = insertelement <8 x i32> %vecinit25, i32 %add28, i32 7
+  ret <8 x i32> %vecinit29
+}
+; CHECK-LABEL: avx2_vphadd_d_test
+; SSE3-NOT: phaddd
+; SSSE3: phaddd
+; SSSE3-NEXT: phaddd
+; AVX-NOT: vphaddd
+; AVX2: vphaddd
+; CHECK: ret
+


More information about the llvm-commits mailing list