[llvm] [X86] Remove LowerToHorizontalOp vectorization lowering code (PR #207406)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 3 23:42:44 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/207406
>From 54a227fb3b9ed54df4acc4e5a5449232a1a3a45d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Fri, 3 Jul 2026 12:05:09 +0100
Subject: [PATCH] [X86] Remove LowerToHorizontalOp vectorization lowering code
Thanks to improvements to SLP, InstCombine and VectorCombine - we should no longer send any of the scalar code hadd/sub patterns in the backend.
This allows to remove the old build_vector(x[0]+x[1],x[2]+x[3],y[0]+y[1],y[2]+y[3]) lowering entirely, and just rely on the combineToHorizontalAddSub folds from add(shuffle(x,y),shuffle(x,y)) patterns.
Fixes #143000
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 414 ------------------------
1 file changed, 414 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 367423117fb8d..b33b01f804c6b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -8686,176 +8686,6 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl,
return false;
}
-/// This is a helper function of LowerToHorizontalOp().
-/// This function checks that the build_vector \p N in input implements a
-/// 128-bit partial horizontal operation on a 256-bit vector, but that operation
-/// may not match the layout of an x86 256-bit horizontal instruction.
-/// In other words, if this returns true, then some extraction/insertion will
-/// be required to produce a valid horizontal instruction.
-///
-/// Parameter \p Opcode defines the kind of horizontal operation to match.
-/// For example, if \p Opcode is equal to ISD::ADD, then this function
-/// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
-/// is equal to ISD::SUB, then this function checks if this is a horizontal
-/// arithmetic sub.
-///
-/// This function only analyzes elements of \p N whose indices are
-/// in range [BaseIdx, LastIdx).
-///
-/// TODO: This function was originally used to match both real and fake partial
-/// horizontal operations, but the index-matching logic is incorrect for that.
-/// See the corrected implementation in isHopBuildVector(). Can we reduce this
-/// code because it is only used for partial h-op matching now?
-static bool isHorizontalBinOpPart(const BuildVectorSDNode *N, unsigned Opcode,
- const SDLoc &DL, SelectionDAG &DAG,
- unsigned BaseIdx, unsigned LastIdx,
- SDValue &V0, SDValue &V1) {
- EVT VT = N->getValueType(0);
- assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops");
- assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!");
- assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&
- "Invalid Vector in input!");
-
- bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD);
- bool CanFold = true;
- unsigned ExpectedVExtractIdx = BaseIdx;
- unsigned NumElts = LastIdx - BaseIdx;
- V0 = DAG.getUNDEF(VT);
- V1 = DAG.getUNDEF(VT);
-
- // Check if N implements a horizontal binop.
- for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) {
- SDValue Op = N->getOperand(i + BaseIdx);
-
- // Skip UNDEFs.
- if (Op->isUndef()) {
- // Update the expected vector extract index.
- if (i * 2 == NumElts)
- ExpectedVExtractIdx = BaseIdx;
- ExpectedVExtractIdx += 2;
- continue;
- }
-
- CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
-
- if (!CanFold)
- break;
-
- SDValue Op0 = Op.getOperand(0);
- SDValue Op1 = Op.getOperand(1);
-
- // Try to match the following pattern:
- // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
- CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
- Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
- Op0.getOperand(0) == Op1.getOperand(0) &&
- isa<ConstantSDNode>(Op0.getOperand(1)) &&
- isa<ConstantSDNode>(Op1.getOperand(1)));
- if (!CanFold)
- break;
-
- unsigned I0 = Op0.getConstantOperandVal(1);
- unsigned I1 = Op1.getConstantOperandVal(1);
-
- if (i * 2 < NumElts) {
- if (V0.isUndef()) {
- V0 = Op0.getOperand(0);
- if (V0.getValueType() != VT)
- return false;
- }
- } else {
- if (V1.isUndef()) {
- V1 = Op0.getOperand(0);
- if (V1.getValueType() != VT)
- return false;
- }
- if (i * 2 == NumElts)
- ExpectedVExtractIdx = BaseIdx;
- }
-
- SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
- if (I0 == ExpectedVExtractIdx)
- CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
- else if (IsCommutable && I1 == ExpectedVExtractIdx) {
- // Try to match the following dag sequence:
- // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
- CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
- } else
- CanFold = false;
-
- ExpectedVExtractIdx += 2;
- }
-
- return CanFold;
-}
-
-/// Emit a sequence of two 128-bit horizontal add/sub followed by
-/// a concat_vector.
-///
-/// This is a helper function of LowerToHorizontalOp().
-/// This function expects two 256-bit vectors called V0 and V1.
-/// At first, each vector is split into two separate 128-bit vectors.
-/// Then, the resulting 128-bit vectors are used to implement two
-/// horizontal binary operations.
-///
-/// The kind of horizontal binary operation is defined by \p X86Opcode.
-///
-/// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to
-/// the two new horizontal binop.
-/// When Mode is set, the first horizontal binop dag node would take as input
-/// the lower 128-bit of V0 and the upper 128-bit of V0. The second
-/// horizontal binop dag node would take as input the lower 128-bit of V1
-/// and the upper 128-bit of V1.
-/// Example:
-/// HADD V0_LO, V0_HI
-/// HADD V1_LO, V1_HI
-///
-/// Otherwise, the first horizontal binop dag node takes as input the lower
-/// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
-/// dag node takes the upper 128-bit of V0 and the upper 128-bit of V1.
-/// Example:
-/// HADD V0_LO, V1_LO
-/// HADD V0_HI, V1_HI
-///
-/// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower
-/// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to
-/// the upper 128-bits of the result.
-static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
- const SDLoc &DL, SelectionDAG &DAG,
- unsigned X86Opcode, bool Mode,
- bool isUndefLO, bool isUndefHI) {
- MVT VT = V0.getSimpleValueType();
- assert(VT.is256BitVector() && VT == V1.getSimpleValueType() &&
- "Invalid nodes in input!");
-
- unsigned NumElts = VT.getVectorNumElements();
- SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
- SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
- SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
- SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
- MVT NewVT = V0_LO.getSimpleValueType();
-
- SDValue LO = DAG.getUNDEF(NewVT);
- SDValue HI = DAG.getUNDEF(NewVT);
-
- if (Mode) {
- // Don't emit a horizontal binop if the result is expected to be UNDEF.
- if (!isUndefLO && !V0->isUndef())
- LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
- if (!isUndefHI && !V1->isUndef())
- HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
- } else {
- // Don't emit a horizontal binop if the result is expected to be UNDEF.
- if (!isUndefLO && (!V0_LO->isUndef() || !V1_LO->isUndef()))
- LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
-
- if (!isUndefHI && (!V0_HI->isUndef() || !V1_HI->isUndef()))
- HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
- }
-
- return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
-}
-
/// Returns true iff \p BV builds a vector with the result equivalent to
/// the result of ADDSUB/SUBADD operation.
/// If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1
@@ -9049,248 +8879,6 @@ static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
}
-static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG,
- unsigned &HOpcode, SDValue &V0, SDValue &V1) {
- // Initialize outputs to known values.
- MVT VT = BV->getSimpleValueType(0);
- HOpcode = ISD::DELETED_NODE;
- V0 = DAG.getUNDEF(VT);
- V1 = DAG.getUNDEF(VT);
-
- // x86 256-bit horizontal ops are defined in a non-obvious way. Each 128-bit
- // half of the result is calculated independently from the 128-bit halves of
- // the inputs, so that makes the index-checking logic below more complicated.
- unsigned NumElts = VT.getVectorNumElements();
- unsigned GenericOpcode = ISD::DELETED_NODE;
- unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1;
- unsigned NumEltsIn128Bits = NumElts / Num128BitChunks;
- unsigned NumEltsIn64Bits = NumEltsIn128Bits / 2;
- for (unsigned i = 0; i != Num128BitChunks; ++i) {
- for (unsigned j = 0; j != NumEltsIn128Bits; ++j) {
- // Ignore undef elements.
- SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
- if (Op.isUndef())
- continue;
-
- // If there's an opcode mismatch, we're done.
- if (HOpcode != ISD::DELETED_NODE && Op.getOpcode() != GenericOpcode)
- return false;
-
- // Initialize horizontal opcode.
- if (HOpcode == ISD::DELETED_NODE) {
- GenericOpcode = Op.getOpcode();
- switch (GenericOpcode) {
- // clang-format off
- case ISD::ADD: HOpcode = X86ISD::HADD; break;
- case ISD::SUB: HOpcode = X86ISD::HSUB; break;
- case ISD::FADD: HOpcode = X86ISD::FHADD; break;
- case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
- default: return false;
- // clang-format on
- }
- }
-
- SDValue Op0 = Op.getOperand(0);
- SDValue Op1 = Op.getOperand(1);
- if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
- Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
- Op0.getOperand(0) != Op1.getOperand(0) ||
- !isa<ConstantSDNode>(Op0.getOperand(1)) ||
- !isa<ConstantSDNode>(Op1.getOperand(1)) || !Op.hasOneUse())
- return false;
-
- // The source vector is chosen based on which 64-bit half of the
- // destination vector is being calculated.
- if (j < NumEltsIn64Bits) {
- if (V0.isUndef())
- V0 = Op0.getOperand(0);
- } else {
- if (V1.isUndef())
- V1 = Op0.getOperand(0);
- }
-
- SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
- if (SourceVec != Op0.getOperand(0))
- return false;
-
- // op (extract_vector_elt A, I), (extract_vector_elt A, I+1)
- unsigned ExtIndex0 = Op0.getConstantOperandVal(1);
- unsigned ExtIndex1 = Op1.getConstantOperandVal(1);
- unsigned ExpectedIndex = i * NumEltsIn128Bits +
- (j % NumEltsIn64Bits) * 2;
- if (ExpectedIndex == ExtIndex0 && ExtIndex1 == ExtIndex0 + 1)
- continue;
-
- // If this is not a commutative op, this does not match.
- if (GenericOpcode != ISD::ADD && GenericOpcode != ISD::FADD)
- return false;
-
- // Addition is commutative, so try swapping the extract indexes.
- // op (extract_vector_elt A, I+1), (extract_vector_elt A, I)
- if (ExpectedIndex == ExtIndex1 && ExtIndex0 == ExtIndex1 + 1)
- continue;
-
- // Extract indexes do not match horizontal requirement.
- return false;
- }
- }
- // We matched. Opcode and operands are returned by reference as arguments.
- return true;
-}
-
-static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
- const SDLoc &DL, SelectionDAG &DAG,
- unsigned HOpcode, SDValue V0, SDValue V1) {
- // If either input vector is not the same size as the build vector,
- // extract/insert the low bits to the correct size.
- // This is free (examples: zmm --> xmm, xmm --> ymm).
- MVT VT = BV->getSimpleValueType(0);
- unsigned Width = VT.getSizeInBits();
- if (V0.getValueSizeInBits() > Width)
- V0 = extractSubVector(V0, 0, DAG, DL, Width);
- else if (V0.getValueSizeInBits() < Width)
- V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, DL, Width);
-
- if (V1.getValueSizeInBits() > Width)
- V1 = extractSubVector(V1, 0, DAG, DL, Width);
- else if (V1.getValueSizeInBits() < Width)
- V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, DL, Width);
-
- unsigned NumElts = VT.getVectorNumElements();
- APInt DemandedElts = APInt::getAllOnes(NumElts);
- for (unsigned i = 0; i != NumElts; ++i)
- if (BV->getOperand(i).isUndef())
- DemandedElts.clearBit(i);
-
- // If we don't need the upper xmm, then perform as a xmm hop.
- unsigned HalfNumElts = NumElts / 2;
- if (VT.is256BitVector() && DemandedElts.lshr(HalfNumElts) == 0) {
- MVT HalfVT = VT.getHalfNumVectorElementsVT();
- V0 = extractSubVector(V0, 0, DAG, DL, 128);
- V1 = extractSubVector(V1, 0, DAG, DL, 128);
- SDValue Half = DAG.getNode(HOpcode, DL, HalfVT, V0, V1);
- return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, DL, 256);
- }
-
- return DAG.getNode(HOpcode, DL, VT, V0, V1);
-}
-
-/// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
-static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV, const SDLoc &DL,
- const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
- // We need at least 2 non-undef elements to make this worthwhile by default.
- unsigned NumNonUndefs =
- count_if(BV->op_values(), [](SDValue V) { return !V.isUndef(); });
- if (NumNonUndefs < 2)
- return SDValue();
-
- // There are 4 sets of horizontal math operations distinguished by type:
- // int/FP at 128-bit/256-bit. Each type was introduced with a different
- // subtarget feature. Try to match those "native" patterns first.
- MVT VT = BV->getSimpleValueType(0);
- if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) ||
- ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) ||
- ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) ||
- ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) {
- unsigned HOpcode;
- SDValue V0, V1;
- if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
- return getHopForBuildVector(BV, DL, DAG, HOpcode, V0, V1);
- }
-
- // Try harder to match 256-bit ops by using extract/concat.
- if (!Subtarget.hasAVX() || !VT.is256BitVector())
- return SDValue();
-
- // Count the number of UNDEF operands in the build_vector in input.
- unsigned NumElts = VT.getVectorNumElements();
- unsigned Half = NumElts / 2;
- unsigned NumUndefsLO = 0;
- unsigned NumUndefsHI = 0;
- for (unsigned i = 0, e = Half; i != e; ++i)
- if (BV->getOperand(i)->isUndef())
- NumUndefsLO++;
-
- for (unsigned i = Half, e = NumElts; i != e; ++i)
- if (BV->getOperand(i)->isUndef())
- NumUndefsHI++;
-
- SDValue InVec0, InVec1;
- if (VT == MVT::v8i32 || VT == MVT::v16i16) {
- SDValue InVec2, InVec3;
- unsigned X86Opcode;
- bool CanFold = true;
-
- if (isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, 0, Half, InVec0, InVec1) &&
- isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, Half, NumElts, InVec2,
- InVec3) &&
- ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
- ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
- X86Opcode = X86ISD::HADD;
- else if (isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, 0, Half, InVec0,
- InVec1) &&
- isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, Half, NumElts, InVec2,
- InVec3) &&
- ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
- ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
- X86Opcode = X86ISD::HSUB;
- else
- CanFold = false;
-
- if (CanFold) {
- // Do not try to expand this build_vector into a pair of horizontal
- // add/sub if we can emit a pair of scalar add/sub.
- if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
- return SDValue();
-
- // Convert this build_vector into a pair of horizontal binops followed by
- // a concat vector. We must adjust the outputs from the partial horizontal
- // matching calls above to account for undefined vector halves.
- SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
- SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
- assert((!V0.isUndef() || !V1.isUndef()) && "Horizontal-op of undefs?");
- bool isUndefLO = NumUndefsLO == Half;
- bool isUndefHI = NumUndefsHI == Half;
- return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO,
- isUndefHI);
- }
- }
-
- if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
- VT == MVT::v16i16) {
- unsigned X86Opcode;
- if (isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, 0, NumElts, InVec0,
- InVec1))
- X86Opcode = X86ISD::HADD;
- else if (isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, 0, NumElts, InVec0,
- InVec1))
- X86Opcode = X86ISD::HSUB;
- else if (isHorizontalBinOpPart(BV, ISD::FADD, DL, DAG, 0, NumElts, InVec0,
- InVec1))
- X86Opcode = X86ISD::FHADD;
- else if (isHorizontalBinOpPart(BV, ISD::FSUB, DL, DAG, 0, NumElts, InVec0,
- InVec1))
- X86Opcode = X86ISD::FHSUB;
- else
- return SDValue();
-
- // Don't try to expand this build_vector into a pair of horizontal add/sub
- // if we can simply emit a pair of scalar add/sub.
- if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
- return SDValue();
-
- // Convert this build_vector into two horizontal add/sub followed by
- // a concat vector.
- bool isUndefLO = NumUndefsLO == Half;
- bool isUndefHI = NumUndefsHI == Half;
- return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
- isUndefLO, isUndefHI);
- }
-
- return SDValue();
-}
-
static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG);
@@ -9948,8 +9536,6 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, dl, Subtarget, DAG))
return AddSub;
- if (SDValue HorizontalOp = LowerToHorizontalOp(BV, dl, Subtarget, DAG))
- return HorizontalOp;
if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, dl, Subtarget, DAG))
return Broadcast;
if (SDValue BitOp = lowerBuildVectorToBitOp(BV, dl, Subtarget, DAG))
More information about the llvm-commits
mailing list