[llvm] [VectorCombine] Add subvector reduction support to foldShuffleChainsToReduce (PR #199872)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 02:58:26 PDT 2026
https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/199872
>From 056256b51ed5f453b9ca37fefdd5e8e665438993 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Wed, 27 May 2026 05:12:36 +0000
Subject: [PATCH 1/7] [VectorCombine] Add subvector reduction support to
foldShuffleChainsToReduce
Extends the matcher to recognise subvector reductions where the chain
narrows through shuffles before extracting lane 0. The matcher tracks
per output lane attribution as the chain is walked and each lane carries
a per source bitmask of contributing source lanes plus a poison flag.
At the extract, lane 0's bitmasks rebuild the reduction as one or more
partial reduce intrinsics.
New test file shuffle-chain-reduction-subvector.ll with 11 tests
Fixes #197919
---
.../Transforms/Vectorize/VectorCombine.cpp | 506 +++++++++---------
.../test/Transforms/PhaseOrdering/X86/madd.ll | 8 +-
llvm/test/Transforms/PhaseOrdering/X86/sad.ll | 6 +-
.../PhaseOrdering/X86/vector-reductions.ll | 7 +-
.../load-extractelement-scalarization.ll | 33 +-
.../X86/extract-binop-inseltpoison.ll | 26 +-
.../VectorCombine/X86/extract-binop.ll | 26 +-
.../X86/shuffle-chain-reduction-subvector.ll | 219 ++++++++
.../fold-shuffle-chains-to-reduce.ll | 65 +--
9 files changed, 530 insertions(+), 366 deletions(-)
create mode 100644 llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 99e45bdc8ee21..f40ad832b6c08 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -3958,11 +3958,6 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
/// 2 3 | p p p p p p
/// 1 | p p p p p p p
///
-/// For powers of 2, there's a consistent pattern, but for other cases
-/// the parity of the current half value at each step decides the
-/// next partition half (see `ExpectedParityMask` for more logical details
-/// in generalising this).
-///
/// Ex:
/// [n = 6]
///
@@ -3970,27 +3965,6 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
/// 1 2 | p p p p
/// 1 | p p p p p
bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
- // Going bottom-up for the pattern.
- std::queue<Value *> InstWorklist;
- InstructionCost OrigCost = 0;
-
- // Common instruction operation after each shuffle op.
- std::optional<unsigned int> CommonCallOp = std::nullopt;
- std::optional<Instruction::BinaryOps> CommonBinOp = std::nullopt;
-
- // For floating-point reductions, track FMF intersection across all binops.
- FastMathFlags CommonFMF;
- bool IsFloatReduction = false;
-
- bool IsFirstCallOrBinInst = true;
- bool ShouldBeCallOrBinInst = true;
-
- // This stores the last used instructions for shuffle/common op.
- //
- // PrevVecV[0] / PrevVecV[1] store the last two simultaneous
- // instructions from either shuffle/common op.
- SmallVector<Value *, 2> PrevVecV(2, nullptr);
-
Value *VecOpEE;
if (!match(&I, m_ExtractElt(m_Value(VecOpEE), m_Zero())))
return false;
@@ -3999,265 +3973,315 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
if (!FVT)
return false;
- int64_t VecSize = FVT->getNumElements();
- if (VecSize < 2)
+ if (FVT->getNumElements() < 2)
return false;
- // Number of levels would be ~log2(n), considering we always partition
- // by half for this fold pattern.
- unsigned int NumLevels = Log2_64_Ceil(VecSize), VisitedCnt = 0;
- int64_t ShuffleMaskHalf = 1, ExpectedParityMask = 0;
+ std::optional<Instruction::BinaryOps> CommonBinOp;
+ std::optional<Intrinsic::ID> CommonCallOp;
- // This is how we generalise for all element sizes.
- // At each step, if vector size is odd, we need non-poison
- // values to cover the dominant half so we don't miss out on any element.
- //
- // This mask will help us retrieve this as we go from bottom to top:
- //
- // Mask Set -> N = N * 2 - 1
- // Mask Unset -> N = N * 2
- for (int Cur = VecSize, Mask = NumLevels - 1; Cur > 1;
- Cur = (Cur + 1) / 2, --Mask) {
- if (Cur & 1)
- ExpectedParityMask |= (1ll << Mask);
+ if (auto *BO = dyn_cast<BinaryOperator>(VecOpEE)) {
+ switch (BO->getOpcode()) {
+ case Instruction::Add:
+ case Instruction::Mul:
+ case Instruction::Or:
+ case Instruction::And:
+ case Instruction::Xor:
+ CommonBinOp = BO->getOpcode();
+ break;
+ default:
+ return false;
+ }
+ } else if (auto *MMI = dyn_cast<MinMaxIntrinsic>(VecOpEE)) {
+ CommonCallOp = MMI->getIntrinsicID();
+ } else {
+ return false;
}
- InstWorklist.push(VecOpEE);
-
- bool IsPartialReduction = false;
- bool HasLaneDuplication = false;
-
- while (!InstWorklist.empty()) {
- Value *CI = InstWorklist.front();
- InstWorklist.pop();
-
- if (auto *II = dyn_cast<IntrinsicInst>(CI)) {
- if (!ShouldBeCallOrBinInst)
- return false;
-
- if (!IsFirstCallOrBinInst && any_of(PrevVecV, equal_to(nullptr)))
- return false;
+ auto IsChainNode = [&](Value *V) {
+ if (auto *BO = dyn_cast<BinaryOperator>(V))
+ return CommonBinOp && BO->getOpcode() == *CommonBinOp;
+ if (auto *MMI = dyn_cast<MinMaxIntrinsic>(V))
+ return CommonCallOp && MMI->getIntrinsicID() == *CommonCallOp;
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(V))
+ return isa<PoisonValue>(SVI->getOperand(1));
+ return false;
+ };
- // For the first found call/bin op, the vector has to come from the
- // extract element op.
- if (II != (IsFirstCallOrBinInst ? VecOpEE : PrevVecV[0]))
- return false;
- IsFirstCallOrBinInst = false;
+ // Walk VecOpEE bottom-up. Chain nodes are single-source shuffles and
+ // matching-opcode binops/intrinsics. Anything else is a leaf source.
+ constexpr unsigned MaxChainNodes = 32;
+ SmallVector<Value *, 16> ChainPostorder;
+ SmallPtrSet<Value *, 16> Visited;
+ SmallVector<Value *, 2> Sources;
+ DenseMap<Value *, unsigned> SrcSizes;
- if (!CommonCallOp)
- CommonCallOp = II->getIntrinsicID();
- if (II->getIntrinsicID() != *CommonCallOp)
- return false;
+ struct StackEntry {
+ Value *V;
+ unsigned ChildIdx;
+ };
+ SmallVector<StackEntry, 16> Stack;
- switch (II->getIntrinsicID()) {
- case Intrinsic::umin:
- case Intrinsic::umax:
- case Intrinsic::smin:
- case Intrinsic::smax: {
- auto *Op0 = II->getOperand(0);
- auto *Op1 = II->getOperand(1);
- PrevVecV[0] = Op0;
- PrevVecV[1] = Op1;
- break;
- }
- default:
+ auto Enqueue = [&](Value *V) -> bool {
+ if (Visited.size() >= MaxChainNodes)
+ return false;
+ if (!Visited.insert(V).second)
+ return true;
+ if (!IsChainNode(V)) {
+ auto *VT = dyn_cast<FixedVectorType>(V->getType());
+ if (!VT)
return false;
- }
- ShouldBeCallOrBinInst ^= 1;
-
- IntrinsicCostAttributes ICA(
- *CommonCallOp, II->getType(),
- {PrevVecV[0]->getType(), PrevVecV[1]->getType()});
- OrigCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
-
- // We may need a swap here since it can be (a, b) or (b, a)
- // and accordingly change as we go up.
- if (!isa<ShuffleVectorInst>(PrevVecV[1]))
- std::swap(PrevVecV[0], PrevVecV[1]);
- InstWorklist.push(PrevVecV[1]);
- InstWorklist.push(PrevVecV[0]);
- } else if (auto *BinOp = dyn_cast<BinaryOperator>(CI)) {
- // Similar logic for bin ops.
+ if (SrcSizes.insert({V, VT->getNumElements()}).second)
+ Sources.push_back(V);
+ return true;
+ }
+ Stack.push_back({V, 0});
+ return true;
+ };
- if (!ShouldBeCallOrBinInst)
- return false;
+ // VecOpEE is a chain node so push directly.
+ Visited.insert(VecOpEE);
+ Stack.push_back({VecOpEE, 0});
- if (!IsFirstCallOrBinInst && any_of(PrevVecV, equal_to(nullptr)))
- return false;
+ while (!Stack.empty()) {
+ auto &Top = Stack.back();
+ Value *V = Top.V;
- if (BinOp != (IsFirstCallOrBinInst ? VecOpEE : PrevVecV[0]))
+ // Chain shuffles always have poison as op1, so only op0 matters.
+ unsigned NumOps = isa<ShuffleVectorInst>(V) ? 1 : 2;
+ if (Top.ChildIdx < NumOps) {
+ Value *Child = cast<User>(V)->getOperand(Top.ChildIdx++);
+ if (!Enqueue(Child))
return false;
- IsFirstCallOrBinInst = false;
+ } else {
+ ChainPostorder.push_back(V);
+ Stack.pop_back();
+ }
+ }
- if (!CommonBinOp)
- CommonBinOp = BinOp->getOpcode();
+ // Demote any binop/intrinsic whose operands are not themselves chain nodes.
+ // Walking past would re-derive the value from its operands, leaving the
+ // original alive if anything downstream still uses it.
+ SmallPtrSet<Value *, 16> KeptChain;
+ SmallVector<Value *, 16> NewChainPostorder;
+ NewChainPostorder.reserve(ChainPostorder.size());
+ for (Value *V : ChainPostorder) {
+ if (isa<ShuffleVectorInst>(V)) {
+ KeptChain.insert(V);
+ NewChainPostorder.push_back(V);
+ continue;
+ }
+ auto *U = cast<User>(V);
+ if (KeptChain.contains(U->getOperand(0)) ||
+ KeptChain.contains(U->getOperand(1))) {
+ KeptChain.insert(V);
+ NewChainPostorder.push_back(V);
+ continue;
+ }
+ auto *VT = cast<FixedVectorType>(V->getType());
+ if (SrcSizes.insert({V, VT->getNumElements()}).second)
+ Sources.push_back(V);
+ }
+ ChainPostorder = std::move(NewChainPostorder);
- if (BinOp->getOpcode() != *CommonBinOp)
- return false;
+ bool IsIdempotent =
+ CommonCallOp || (CommonBinOp && Instruction::isIdempotent(*CommonBinOp));
- switch (*CommonBinOp) {
- case BinaryOperator::Add:
- case BinaryOperator::Mul:
- case BinaryOperator::Or:
- case BinaryOperator::And:
- case BinaryOperator::Xor:
- case BinaryOperator::FAdd:
- case BinaryOperator::FMul: {
- auto *Op0 = BinOp->getOperand(0);
- auto *Op1 = BinOp->getOperand(1);
- PrevVecV[0] = Op0;
- PrevVecV[1] = Op1;
- break;
- }
- default:
- return false;
- }
+ // Each output lane has a bitmask of contributing source lanes and a
+ // poison flag. Shuffles permute these records. Binops union them.
+ struct LaneInfo {
+ SmallDenseMap<Value *, APInt, 2> SrcBits;
+ bool IsPoison = false;
+ };
+ DenseMap<Value *, SmallVector<LaneInfo, 16>> Attr;
+ for (Value *Src : Sources) {
+ unsigned N = SrcSizes[Src];
+ auto &A = Attr[Src];
+ A.reserve(N);
+ for (unsigned i = 0; i < N; ++i) {
+ LaneInfo LI;
+ LI.SrcBits[Src] = APInt::getOneBitSet(N, i);
+ A.push_back(std::move(LI));
+ }
+ }
- // For FP reductions, require reassoc on every binop and collect FMF.
- if (*CommonBinOp == Instruction::FAdd ||
- *CommonBinOp == Instruction::FMul) {
- if (!BinOp->hasAllowReassoc())
+ // Postorder ensures each value's operands are already in Attr.
+ for (Value *V : ChainPostorder) {
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(V)) {
+ const auto &In = Attr.at(SVI->getOperand(0));
+ auto &Out = Attr[V];
+ ArrayRef<int> Mask = SVI->getShuffleMask();
+ Out.reserve(Mask.size());
+ for (int M : Mask) {
+ if (M < 0) {
+ LaneInfo LI;
+ LI.IsPoison = true;
+ Out.push_back(std::move(LI));
+ } else if ((unsigned)M >= In.size()) {
return false;
- if (!IsFloatReduction) {
- CommonFMF = BinOp->getFastMathFlags();
- IsFloatReduction = true;
} else {
- CommonFMF &= BinOp->getFastMathFlags();
+ Out.push_back(In[M]);
}
}
-
- ShouldBeCallOrBinInst ^= 1;
-
- OrigCost +=
- TTI.getArithmeticInstrCost(*CommonBinOp, BinOp->getType(), CostKind);
-
- if (!isa<ShuffleVectorInst>(PrevVecV[1]))
- std::swap(PrevVecV[0], PrevVecV[1]);
- InstWorklist.push(PrevVecV[1]);
- InstWorklist.push(PrevVecV[0]);
- } else if (auto *SVInst = dyn_cast<ShuffleVectorInst>(CI)) {
- // We shouldn't have any null values in the previous vectors,
- // is so, there was a mismatch in pattern.
- if (ShouldBeCallOrBinInst || any_of(PrevVecV, equal_to(nullptr)))
- return false;
-
- if (SVInst != PrevVecV[1])
- return false;
-
- ArrayRef<int> CurMask;
- if (!match(SVInst, m_Shuffle(m_Specific(PrevVecV[0]), m_Poison(),
- m_Mask(CurMask))))
- return false;
-
- // Subtract the parity mask when checking the condition.
- for (int Mask = 0, MaskSize = CurMask.size(); Mask != MaskSize; ++Mask) {
- if (Mask < ShuffleMaskHalf &&
- CurMask[Mask] != ShuffleMaskHalf + Mask - (ExpectedParityMask & 1))
- return false;
- if (Mask >= ShuffleMaskHalf && CurMask[Mask] != -1)
- return false;
- }
-
- // Update mask values.
- ShuffleMaskHalf *= 2;
- ShuffleMaskHalf -= (ExpectedParityMask & 1);
- HasLaneDuplication |= (ExpectedParityMask & 1) != 0;
- ExpectedParityMask >>= 1;
-
- OrigCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
- SVInst->getType(), SVInst->getType(),
- CurMask, CostKind);
-
- VisitedCnt += 1;
- if (!ExpectedParityMask && VisitedCnt == NumLevels)
- break;
-
- ShouldBeCallOrBinInst ^= 1;
} else {
- // Check if this is a partial reduction - the chain ended because
- // the source vector is not a recognized op/shuffle.
- // Reject non-power-of-2 vectors because parity-based masks cause
- // lane duplication in the reduction tree, making the partial result
- // not a simple subvector reduction.
- if (ShouldBeCallOrBinInst && VisitedCnt >= 1 && CI == PrevVecV[0] &&
- isPowerOf2_64(VecSize)) {
- IsPartialReduction = true;
- break;
+ auto *U = cast<User>(V);
+ const auto &L = Attr.at(U->getOperand(0));
+ const auto &R = Attr.at(U->getOperand(1));
+ if (L.size() != R.size())
+ return false;
+ auto &Out = Attr[V];
+ Out.reserve(L.size());
+ for (unsigned i = 0; i < L.size(); ++i) {
+ LaneInfo OutLI;
+ OutLI.IsPoison = L[i].IsPoison || R[i].IsPoison;
+ for (auto &SBP : L[i].SrcBits)
+ OutLI.SrcBits.insert(SBP);
+ for (auto &SBP : R[i].SrcBits) {
+ auto [It, Inserted] =
+ OutLI.SrcBits.try_emplace(SBP.first, SBP.second);
+ if (!Inserted) {
+ // x op x != x for non-idempotent ops but poison lanes can be
+ // refined.
+ if (It->second.intersects(SBP.second) && !IsIdempotent &&
+ !OutLI.IsPoison)
+ return false;
+ It->second |= SBP.second;
+ }
+ }
+ Out.push_back(std::move(OutLI));
}
- return false;
}
}
- // Full reduction pattern should end with a shuffle op.
- // Partial reduction ends when the source vector is reached.
- if (ShouldBeCallOrBinInst && !IsPartialReduction)
- return false;
-
- // If the parity masks duplicated any lane, the fold only preserves semantics
- // for idempotent ops.
- if (HasLaneDuplication && CommonBinOp &&
- !Instruction::isIdempotent(*CommonBinOp))
- return false;
+ // One partial reduce per source contributing to lane 0.
+ const LaneInfo &Lane0 = Attr.at(VecOpEE)[0];
- assert(VecSize != -1 && "Expected Match for Vector Size");
-
- Value *FinalVecV = PrevVecV[0];
- if (!FinalVecV)
+ struct Partial {
+ Value *Src;
+ APInt Bits;
+ unsigned SrcSize;
+ };
+ SmallVector<Partial, 2> Partials;
+ for (Value *Src : Sources) {
+ auto It = Lane0.SrcBits.find(Src);
+ if (It == Lane0.SrcBits.end())
+ continue;
+ const APInt &Bits = It->second;
+ if (Bits.popcount() == 0)
+ continue;
+ Partials.push_back({Src, Bits, SrcSizes[Src]});
+ }
+ if (Partials.empty())
return false;
- auto *FinalVecVTy = cast<FixedVectorType>(FinalVecV->getType());
-
Intrinsic::ID ReducedOp =
(CommonCallOp ? getMinMaxReductionIntrinsicID(*CommonCallOp)
: getReductionForBinop(*CommonBinOp));
if (!ReducedOp)
return false;
+ InstructionCost OrigCost = 0;
+ for (Value *V : ChainPostorder) {
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(V)) {
+ auto *VT = cast<FixedVectorType>(SVI->getType());
+ auto *SrcVT = cast<FixedVectorType>(SVI->getOperand(0)->getType());
+ auto Kind = (VT == SrcVT) ? TargetTransformInfo::SK_PermuteSingleSrc
+ : TargetTransformInfo::SK_ExtractSubvector;
+ OrigCost +=
+ TTI.getShuffleCost(Kind, VT, SrcVT, SVI->getShuffleMask(), CostKind);
+ } else if (auto *BO = dyn_cast<BinaryOperator>(V)) {
+ OrigCost +=
+ TTI.getArithmeticInstrCost(BO->getOpcode(), BO->getType(), CostKind);
+ } else {
+ auto *II = cast<IntrinsicInst>(V);
+ IntrinsicCostAttributes ICA(
+ II->getIntrinsicID(), II->getType(),
+ {II->getOperand(0)->getType(), II->getOperand(1)->getType()});
+ OrigCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
+ }
+ }
+
+ // Each source contributes a gather+reduce, or an extractelement for a
+ // single-lane partial. Scalar ops then combine the partials.
+ Type *ElTy = FVT->getElementType();
InstructionCost NewCost = 0;
- FixedVectorType *ReduceVecTy = FinalVecVTy;
- SmallVector<int> ExtractMask;
-
- if (IsPartialReduction) {
- unsigned SubVecSize = ShuffleMaskHalf;
- ReduceVecTy = FixedVectorType::get(FVT->getElementType(), SubVecSize);
- ExtractMask.resize(SubVecSize);
- std::iota(ExtractMask.begin(), ExtractMask.end(), 0);
- NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
- ReduceVecTy, FinalVecVTy, ExtractMask,
- CostKind, 0, ReduceVecTy);
- }
-
- IntrinsicCostAttributes ICA(
- ReducedOp, ReduceVecTy->getElementType(),
- IsFloatReduction
- ? SmallVector<Type *, 2>{ReduceVecTy->getElementType(), ReduceVecTy}
- : SmallVector<Type *, 2>{ReduceVecTy},
- IsFloatReduction ? CommonFMF : FastMathFlags());
- NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
+ for (const Partial &P : Partials) {
+ auto *SrcVT = cast<FixedVectorType>(P.Src->getType());
+ unsigned Pop = P.Bits.popcount();
+ if (Pop == 1) {
+ unsigned Idx = P.Bits.countTrailingZeros();
+ NewCost += TTI.getVectorInstrCost(Instruction::ExtractElement, SrcVT,
+ CostKind, Idx);
+ continue;
+ }
+ bool IsFullSrc = Pop == P.SrcSize;
+ FixedVectorType *RVT = IsFullSrc ? SrcVT : FixedVectorType::get(ElTy, Pop);
+ if (!IsFullSrc) {
+ SmallVector<int> Mask;
+ Mask.reserve(Pop);
+ for (unsigned i = 0; i < P.SrcSize; ++i)
+ if (P.Bits[i])
+ Mask.push_back(i);
+ // A contiguous run of bits (anywhere, not just the low prefix) is an
+ // extract subvector. Pass its offset so TTI can model targets that
+ // lower upper-half extracts cheaply.
+ unsigned SubIdx = 0, SubLen;
+ auto Kind = P.Bits.isShiftedMask(SubIdx, SubLen)
+ ? TargetTransformInfo::SK_ExtractSubvector
+ : TargetTransformInfo::SK_PermuteSingleSrc;
+ NewCost +=
+ TTI.getShuffleCost(Kind, RVT, SrcVT, Mask, CostKind, SubIdx, RVT);
+ }
+ IntrinsicCostAttributes ICA(ReducedOp, RVT, {RVT});
+ NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
+ }
+ for (unsigned i = 1; i < Partials.size(); ++i) {
+ if (CommonBinOp) {
+ NewCost += TTI.getArithmeticInstrCost(*CommonBinOp, ElTy, CostKind);
+ } else {
+ IntrinsicCostAttributes ICA(*CommonCallOp, ElTy, {ElTy, ElTy});
+ NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
+ }
+ }
LLVM_DEBUG(dbgs() << "Found reduction shuffle chain: " << I << "\n OldCost : "
<< OrigCost << " vs NewCost: " << NewCost << "\n");
- if (VecOpEE->hasOneUse() ? (NewCost > OrigCost) : (NewCost >= OrigCost))
+ if (!OrigCost.isValid() || !NewCost.isValid())
return false;
- Value *ReduceInput = FinalVecV;
- if (IsPartialReduction)
- ReduceInput = Builder.CreateShuffleVector(FinalVecV, ExtractMask);
+ if (VecOpEE->hasOneUse() ? (NewCost > OrigCost) : (NewCost >= OrigCost))
+ return false;
- CallInst *ReducedResult;
- if (IsFloatReduction) {
- Value *Identity = ConstantExpr::getBinOpIdentity(
- *CommonBinOp, ReduceVecTy->getElementType(), /*AllowRHSConstant=*/false,
- CommonFMF.noSignedZeros());
- ReducedResult = Builder.CreateIntrinsic(ReducedOp, {ReduceVecTy},
- {Identity, ReduceInput});
- ReducedResult->setFastMathFlags(CommonFMF);
- } else {
- ReducedResult =
- Builder.CreateIntrinsic(ReducedOp, {ReduceVecTy}, {ReduceInput});
+ SmallVector<Value *, 2> PartialResults;
+ for (const Partial &P : Partials) {
+ unsigned Pop = P.Bits.popcount();
+ // A single-lane partial is just the source lane. No reduce needed.
+ if (Pop == 1) {
+ unsigned Idx = P.Bits.countTrailingZeros();
+ PartialResults.push_back(Builder.CreateExtractElement(P.Src, Idx));
+ continue;
+ }
+ Value *ReduceInput = P.Src;
+ if (Pop != P.SrcSize) {
+ SmallVector<int> Mask;
+ Mask.reserve(Pop);
+ for (unsigned i = 0; i < P.SrcSize; ++i)
+ if (P.Bits[i])
+ Mask.push_back(i);
+ ReduceInput = Builder.CreateShuffleVector(P.Src, Mask);
+ }
+ PartialResults.push_back(Builder.CreateIntrinsic(
+ ReducedOp, {ReduceInput->getType()}, {ReduceInput}));
}
- replaceValue(I, *ReducedResult);
+ Value *Result = PartialResults[0];
+ for (unsigned i = 1; i < PartialResults.size(); ++i) {
+ if (CommonBinOp)
+ Result = Builder.CreateBinOp(*CommonBinOp, Result, PartialResults[i]);
+ else
+ Result = Builder.CreateIntrinsic(*CommonCallOp, {ElTy},
+ {Result, PartialResults[i]});
+ }
+ replaceValue(I, *Result);
return true;
}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/madd.ll b/llvm/test/Transforms/PhaseOrdering/X86/madd.ll
index efdb5cc0f1a27..022774b8048b1 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/madd.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/madd.ll
@@ -1223,20 +1223,20 @@ define i32 @madd_quad_reduction(ptr %arg, ptr %arg1, ptr %arg2, ptr %arg3, ptr %
; CHECK-NEXT: [[TMP22:%.*]] = sext <8 x i16> [[TMP20]] to <8 x i32>
; CHECK-NEXT: [[TMP23:%.*]] = sext <8 x i16> [[TMP21]] to <8 x i32>
; CHECK-NEXT: [[TMP25:%.*]] = mul nsw <8 x i32> [[TMP23]], [[TMP22]]
-; CHECK-NEXT: [[TMP26:%.*]] = add nuw nsw <8 x i32> [[TMP25]], [[TMP19]]
+; CHECK-NEXT: [[I26:%.*]] = add nuw nsw <8 x i32> [[TMP25]], [[TMP19]]
; CHECK-NEXT: [[TMP40:%.*]] = load <8 x i16>, ptr [[ARG4]], align 1
; CHECK-NEXT: [[TMP41:%.*]] = load <8 x i16>, ptr [[ARG5]], align 1
; CHECK-NEXT: [[TMP42:%.*]] = sext <8 x i16> [[TMP40]] to <8 x i32>
; CHECK-NEXT: [[TMP43:%.*]] = sext <8 x i16> [[TMP41]] to <8 x i32>
; CHECK-NEXT: [[TMP45:%.*]] = mul nsw <8 x i32> [[TMP43]], [[TMP42]]
-; CHECK-NEXT: [[TMP56:%.*]] = add nuw nsw <8 x i32> [[TMP26]], [[TMP45]]
+; CHECK-NEXT: [[I56:%.*]] = add nuw nsw <8 x i32> [[I26]], [[TMP45]]
; CHECK-NEXT: [[TMP50:%.*]] = load <8 x i16>, ptr [[ARG6]], align 1
; CHECK-NEXT: [[TMP51:%.*]] = load <8 x i16>, ptr [[ARG7]], align 1
; CHECK-NEXT: [[TMP52:%.*]] = sext <8 x i16> [[TMP50]] to <8 x i32>
; CHECK-NEXT: [[TMP53:%.*]] = sext <8 x i16> [[TMP51]] to <8 x i32>
; CHECK-NEXT: [[TMP55:%.*]] = mul nsw <8 x i32> [[TMP53]], [[TMP52]]
-; CHECK-NEXT: [[TMP57:%.*]] = add nuw nsw <8 x i32> [[TMP56]], [[TMP55]]
-; CHECK-NEXT: [[TMP35:%.*]] = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP57]])
+; CHECK-NEXT: [[I57:%.*]] = add nuw nsw <8 x i32> [[I56]], [[TMP55]]
+; CHECK-NEXT: [[TMP35:%.*]] = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[I57]])
; CHECK-NEXT: ret i32 [[TMP35]]
;
%i = load <8 x i16>, ptr %arg, align 1
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/sad.ll b/llvm/test/Transforms/PhaseOrdering/X86/sad.ll
index a24d72282cf1b..baa32a98559ae 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/sad.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/sad.ll
@@ -484,15 +484,15 @@ define dso_local i32 @sad_unroll_nonzero_initial(ptr %arg, ptr %arg1, ptr %arg2,
; CHECK-NEXT: [[TMP6:%.*]] = zext <16 x i8> [[TMP4]] to <16 x i32>
; CHECK-NEXT: [[TMP7:%.*]] = sub nsw <16 x i32> [[TMP5]], [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP7]], i1 true)
-; CHECK-NEXT: [[TMP11:%.*]] = add nuw nsw <16 x i32> [[TMP10]], <i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[I11:%.*]] = add nuw nsw <16 x i32> [[TMP10]], <i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
; CHECK-NEXT: [[TMP12:%.*]] = load <16 x i8>, ptr [[ARG2]], align 1
; CHECK-NEXT: [[TMP13:%.*]] = load <16 x i8>, ptr [[ARG3]], align 1
; CHECK-NEXT: [[TMP14:%.*]] = zext <16 x i8> [[TMP12]] to <16 x i32>
; CHECK-NEXT: [[TMP15:%.*]] = zext <16 x i8> [[TMP13]] to <16 x i32>
; CHECK-NEXT: [[TMP16:%.*]] = sub nsw <16 x i32> [[TMP14]], [[TMP15]]
; CHECK-NEXT: [[TMP19:%.*]] = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP16]], i1 true)
-; CHECK-NEXT: [[TMP20:%.*]] = add nuw nsw <16 x i32> [[TMP11]], [[TMP19]]
-; CHECK-NEXT: [[TMP29:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP20]])
+; CHECK-NEXT: [[I20:%.*]] = add nuw nsw <16 x i32> [[I11]], [[TMP19]]
+; CHECK-NEXT: [[TMP29:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[I20]])
; CHECK-NEXT: ret i32 [[TMP29]]
;
bb:
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
index 2b38cfe7f21bd..c3464a21466de 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
@@ -24,11 +24,8 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
-; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[SHIFT1]]
-; CHECK-NEXT: [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[X210:%.*]] = tail call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP1]])
; CHECK-NEXT: ret i32 [[X210]]
;
%x0 = extractelement <4 x i32> %x, i32 0
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll b/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
index 5c035d29a7ea2..d4278c86383a1 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
@@ -540,15 +540,8 @@ define i32 @load_extract_clobber_store_between(ptr %x, ptr %y) {
define i32 @load_extract_clobber_store_between_limit(ptr %x, ptr %y, <8 x i32> %z) {
; CHECK-LABEL: @load_extract_clobber_store_between_limit(
; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <8 x i32> [[Z1:%.*]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i32> [[Z1]], [[SHIFT]]
-; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = add <8 x i32> [[TMP1]], [[SHIFT1]]
-; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[SHIFT2]]
-; CHECK-NEXT: [[SHIFT3:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[Z:%.*]] = add <8 x i32> [[TMP3]], [[SHIFT3]]
-; CHECK-NEXT: [[Z_0:%.*]] = extractelement <8 x i32> [[Z]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[Z:%.*]], <8 x i32> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT: [[Z_0:%.*]] = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> [[TMP1]])
; CHECK-NEXT: store i8 0, ptr [[Y:%.*]], align 1
; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 2
; CHECK-NEXT: [[ADD_4:%.*]] = add i32 [[Z_0]], [[R]]
@@ -572,15 +565,8 @@ define i32 @load_extract_clobber_store_between_limit(ptr %x, ptr %y, <8 x i32> %
define i32 @load_extract_clobber_store_after_limit(ptr %x, ptr %y, <8 x i32> %z) {
; LIMIT-DEFAULT-LABEL: @load_extract_clobber_store_after_limit(
-; LIMIT-DEFAULT-NEXT: [[SHIFT:%.*]] = shufflevector <8 x i32> [[Z1:%.*]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; LIMIT-DEFAULT-NEXT: [[TMP4:%.*]] = add <8 x i32> [[Z1]], [[SHIFT]]
-; LIMIT-DEFAULT-NEXT: [[SHIFT1:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; LIMIT-DEFAULT-NEXT: [[TMP2:%.*]] = add <8 x i32> [[TMP4]], [[SHIFT1]]
-; LIMIT-DEFAULT-NEXT: [[SHIFT2:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; LIMIT-DEFAULT-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[SHIFT2]]
-; LIMIT-DEFAULT-NEXT: [[SHIFT3:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; LIMIT-DEFAULT-NEXT: [[Z:%.*]] = add <8 x i32> [[TMP3]], [[SHIFT3]]
-; LIMIT-DEFAULT-NEXT: [[Z_0:%.*]] = extractelement <8 x i32> [[Z]], i32 0
+; LIMIT-DEFAULT-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[Z:%.*]], <8 x i32> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4>
+; LIMIT-DEFAULT-NEXT: [[Z_0:%.*]] = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> [[TMP2]])
; LIMIT-DEFAULT-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 2
; LIMIT-DEFAULT-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8
; LIMIT-DEFAULT-NEXT: store i8 0, ptr [[Y:%.*]], align 1
@@ -589,15 +575,8 @@ define i32 @load_extract_clobber_store_after_limit(ptr %x, ptr %y, <8 x i32> %z)
;
; LIMIT2-LABEL: @load_extract_clobber_store_after_limit(
; LIMIT2-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
-; LIMIT2-NEXT: [[SHIFT:%.*]] = shufflevector <8 x i32> [[Z1:%.*]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; LIMIT2-NEXT: [[TMP1:%.*]] = add <8 x i32> [[Z1]], [[SHIFT]]
-; LIMIT2-NEXT: [[SHIFT1:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; LIMIT2-NEXT: [[TMP2:%.*]] = add <8 x i32> [[TMP1]], [[SHIFT1]]
-; LIMIT2-NEXT: [[SHIFT2:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; LIMIT2-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[SHIFT2]]
-; LIMIT2-NEXT: [[SHIFT3:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; LIMIT2-NEXT: [[Z:%.*]] = add <8 x i32> [[TMP3]], [[SHIFT3]]
-; LIMIT2-NEXT: [[Z_0:%.*]] = extractelement <8 x i32> [[Z]], i32 0
+; LIMIT2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[Z:%.*]], <8 x i32> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4>
+; LIMIT2-NEXT: [[Z_0:%.*]] = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> [[TMP1]])
; LIMIT2-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 2
; LIMIT2-NEXT: store i8 0, ptr [[Y:%.*]], align 1
; LIMIT2-NEXT: [[ADD_4:%.*]] = add i32 [[Z_0]], [[R]]
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
index 41d77e89476ba..9791cd2c0f7a9 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
@@ -478,13 +478,7 @@ define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: @ext_ext_or_reduction_v4i32(
; CHECK-NEXT: [[Z:%.*]] = and <4 x i32> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[Z]], [[SHIFT]]
-; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[SHIFT1]]
-; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i32> [[SHIFT2]], [[TMP2]]
-; CHECK-NEXT: [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; CHECK-NEXT: [[Z0123:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[Z]])
; CHECK-NEXT: ret i32 [[Z0123]]
;
%z = and <4 x i32> %x, %y
@@ -500,11 +494,8 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
-; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
-; CHECK-NEXT: [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[X210:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP1]])
; CHECK-NEXT: ret i32 [[X210]]
;
%x0 = extractelement <4 x i32> %x, i32 0
@@ -517,13 +508,10 @@ define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: @ext_ext_partial_add_reduction_and_extra_add_v4i32(
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[Y]]
-; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
-; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[SHIFT2]], [[TMP2]]
-; CHECK-NEXT: [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[X:%.*]], i64 2
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP2]])
+; CHECK-NEXT: [[X2Y210:%.*]] = add i32 [[TMP1]], [[TMP3]]
; CHECK-NEXT: ret i32 [[X2Y210]]
;
%y0 = extractelement <4 x i32> %y, i32 0
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
index 4c1ca82b2bd06..db3ac51464e44 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
@@ -484,13 +484,7 @@ define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: @ext_ext_or_reduction_v4i32(
; CHECK-NEXT: [[Z:%.*]] = and <4 x i32> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[Z]], [[SHIFT]]
-; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[SHIFT1]]
-; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i32> [[SHIFT2]], [[TMP2]]
-; CHECK-NEXT: [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; CHECK-NEXT: [[Z0123:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[Z]])
; CHECK-NEXT: ret i32 [[Z0123]]
;
%z = and <4 x i32> %x, %y
@@ -506,11 +500,8 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
-; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
-; CHECK-NEXT: [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[X210:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP1]])
; CHECK-NEXT: ret i32 [[X210]]
;
%x0 = extractelement <4 x i32> %x, i32 0
@@ -523,13 +514,10 @@ define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: @ext_ext_partial_add_reduction_and_extra_add_v4i32(
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[Y]]
-; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
-; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[SHIFT2]], [[TMP2]]
-; CHECK-NEXT: [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[X:%.*]], i64 2
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP2]])
+; CHECK-NEXT: [[X2Y210:%.*]] = add i32 [[TMP1]], [[TMP3]]
; CHECK-NEXT: ret i32 [[X2Y210]]
;
%y0 = extractelement <4 x i32> %y, i32 0
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
new file mode 100644
index 0000000000000..f59915fce9c3b
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
@@ -0,0 +1,219 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64 -passes=vector-combine -S %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes=vector-combine -S %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes=vector-combine -S %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v4 -passes=vector-combine -S %s | FileCheck %s
+
+define i32 @test_subvector_reduce_add_v4i32(<4 x i32> %a0) {
+; CHECK-LABEL: define i32 @test_subvector_reduce_add_v4i32(
+; CHECK-SAME: <4 x i32> [[A0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[A0]])
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a0, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+ %2 = shufflevector <4 x i32> %a0, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+ %3 = add <2 x i32> %1, %2
+ %4 = shufflevector <2 x i32> %3, <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
+ %5 = add <2 x i32> %3, %4
+ %6 = extractelement <2 x i32> %5, i64 0
+ ret i32 %6
+}
+
+define i16 @test_subvector_reduce_add_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_subvector_reduce_add_v8i16(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[A0]])
+; CHECK-NEXT: ret i16 [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %3 = add <4 x i16> %1, %2
+ %4 = shufflevector <4 x i16> %3, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %5 = add <4 x i16> %3, %4
+ %6 = shufflevector <4 x i16> %5, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %7 = add <4 x i16> %5, %6
+ %8 = extractelement <4 x i16> %7, i64 0
+ ret i16 %8
+}
+
+define i8 @test_subvector_reduce_add_v16i8(<16 x i8> %a0) {
+; CHECK-LABEL: define i8 @test_subvector_reduce_add_v16i8(
+; CHECK-SAME: <16 x i8> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> [[A0]])
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+ %1 = shufflevector <16 x i8> %a0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <16 x i8> %a0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %3 = add <8 x i8> %1, %2
+ %4 = shufflevector <8 x i8> %3, <8 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+ %5 = add <8 x i8> %3, %4
+ %6 = shufflevector <8 x i8> %5, <8 x i8> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %7 = add <8 x i8> %5, %6
+ %8 = shufflevector <8 x i8> %7, <8 x i8> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %9 = add <8 x i8> %7, %8
+ %10 = extractelement <8 x i8> %9, i64 0
+ ret i8 %10
+}
+
+define i8 @test_subvector_reduce_add_v64i8(<64 x i8> %a0) {
+; CHECK-LABEL: define i8 @test_subvector_reduce_add_v64i8(
+; CHECK-SAME: <64 x i8> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[R:%.*]] = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> [[A0]])
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %1 = shufflevector <64 x i8> %a0, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+ %2 = shufflevector <64 x i8> %a0, <64 x i8> poison, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+ %3 = add <32 x i8> %1, %2
+ %4 = shufflevector <32 x i8> %3, <32 x i8> poison, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %5 = add <32 x i8> %3, %4
+ %6 = shufflevector <32 x i8> %5, <32 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %7 = add <32 x i8> %5, %6
+ %8 = shufflevector <32 x i8> %7, <32 x i8> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %9 = add <32 x i8> %7, %8
+ %10 = shufflevector <32 x i8> %9, <32 x i8> poison, <32 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %11 = add <32 x i8> %9, %10
+ %12 = shufflevector <32 x i8> %11, <32 x i8> poison, <32 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %13 = add <32 x i8> %11, %12
+ %r = extractelement <32 x i8> %13, i64 0
+ ret i8 %r
+}
+
+define i8 @test_subvector_reduce_mul_v16i8(<16 x i8> %a0) {
+; CHECK-LABEL: define i8 @test_subvector_reduce_mul_v16i8(
+; CHECK-SAME: <16 x i8> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> [[A0]])
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+ %1 = shufflevector <16 x i8> %a0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <16 x i8> %a0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %3 = mul <8 x i8> %1, %2
+ %4 = shufflevector <8 x i8> %3, <8 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+ %5 = mul <8 x i8> %3, %4
+ %6 = shufflevector <8 x i8> %5, <8 x i8> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %7 = mul <8 x i8> %5, %6
+ %8 = shufflevector <8 x i8> %7, <8 x i8> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %9 = mul <8 x i8> %7, %8
+ %10 = extractelement <8 x i8> %9, i64 0
+ ret i8 %10
+}
+
+define i16 @test_subvector_reduce_swapped_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_subvector_reduce_swapped_v8i16(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[A0]])
+; CHECK-NEXT: ret i16 [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %3 = add <4 x i16> %2, %1
+ %4 = shufflevector <4 x i16> %3, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %5 = add <4 x i16> %3, %4
+ %6 = shufflevector <4 x i16> %5, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %7 = add <4 x i16> %5, %6
+ %8 = extractelement <4 x i16> %7, i64 0
+ ret i16 %8
+}
+
+define i16 @test_subvector_reduce_deinterleave_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_subvector_reduce_deinterleave_v8i16(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[A0]])
+; CHECK-NEXT: ret i16 [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %2 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+ %3 = add <4 x i16> %1, %2
+ %4 = shufflevector <4 x i16> %3, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %5 = add <4 x i16> %3, %4
+ %6 = shufflevector <4 x i16> %5, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %7 = add <4 x i16> %5, %6
+ %8 = extractelement <4 x i16> %7, i64 0
+ ret i16 %8
+}
+
+define i32 @test_multishuffle_accumulator_v8i32(<8 x i32> %a0) {
+; CHECK-LABEL: define i32 @test_multishuffle_accumulator_v8i32(
+; CHECK-SAME: <8 x i32> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[E:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[A0]])
+; CHECK-NEXT: ret i32 [[E]]
+;
+ %s0 = shufflevector <8 x i32> %a0, <8 x i32> poison, <2 x i32> <i32 0, i32 1>
+ %s1 = shufflevector <8 x i32> %a0, <8 x i32> poison, <2 x i32> <i32 2, i32 3>
+ %s2 = shufflevector <8 x i32> %a0, <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+ %s3 = shufflevector <8 x i32> %a0, <8 x i32> poison, <2 x i32> <i32 6, i32 7>
+ %a = add <2 x i32> %s0, %s1
+ %b = add <2 x i32> %s2, %s3
+ %c = add <2 x i32> %a, %b
+ %sf = shufflevector <2 x i32> %c, <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
+ %d = add <2 x i32> %c, %sf
+ %e = extractelement <2 x i32> %d, i64 0
+ ret i32 %e
+}
+
+define i32 @test_multisource_full_plus_scalar_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: define i32 @test_multisource_full_plus_scalar_v4i32(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT: [[E:%.*]] = add i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret i32 [[E]]
+;
+ %xs1 = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %a1 = add <4 x i32> %x, %xs1
+ %xs2 = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
+ %a2 = add <4 x i32> %a1, %xs2
+ %xs3 = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
+ %a3 = add <4 x i32> %a2, %xs3
+ %ys = shufflevector <4 x i32> %y, <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
+ %a4 = add <4 x i32> %a3, %ys
+ %e = extractelement <4 x i32> %a4, i64 0
+ ret i32 %e
+}
+
+define i16 @test_neg_overlapping_masks_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_neg_overlapping_masks_v8i16(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP8]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP5]], <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i16> [[TMP7]], i64 0
+; CHECK-NEXT: ret i16 [[TMP4]]
+;
+ %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+ %3 = add <4 x i16> %1, %2
+ %4 = shufflevector <4 x i16> %3, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %5 = add <4 x i16> %3, %4
+ %6 = shufflevector <4 x i16> %5, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %7 = add <4 x i16> %5, %6
+ %8 = extractelement <4 x i16> %7, i64 0
+ ret i16 %8
+}
+
+define i16 @test_neg_different_sources_v8i16(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: define i16 @test_neg_different_sources_v8i16(
+; CHECK-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[A1]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP8]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP5]], <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i16> [[TMP7]], i64 0
+; CHECK-NEXT: ret i16 [[TMP4]]
+;
+ %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %2 = shufflevector <8 x i16> %a1, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %3 = add <4 x i16> %1, %2
+ %4 = shufflevector <4 x i16> %3, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+ %5 = add <4 x i16> %3, %4
+ %6 = shufflevector <4 x i16> %5, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %7 = add <4 x i16> %5, %6
+ %8 = extractelement <4 x i16> %7, i64 0
+ ret i16 %8
+}
diff --git a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
index 54931f59c2638..196a1cc53ab4a 100644
--- a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
+++ b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
@@ -47,6 +47,7 @@ define i16 @test_reduce_v3i16_and(<3 x i16> %a0) {
ret i16 %5
}
+; v6i16 xor chain duplicates lanes, xor is non-idempotent, so folding would miscompile.
define i16 @test_no_reduce_v6i16_xor(<6 x i16> %a0) {
; CHECK-LABEL: define i16 @test_no_reduce_v6i16_xor(
; CHECK-SAME: <6 x i16> [[A0:%.*]]) {
@@ -106,20 +107,8 @@ define i16 @test_reduce_v3i16_smax(<3 x i16> %a0) {
define i16 @test_reduce_v8i16_2(<8 x i16> %a0) {
; CHECK-LABEL: define i16 @test_reduce_v8i16_2(
; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP4:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP5]])
; CHECK-NEXT: [[TMP13:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[A0]])
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i16> [[TMP6]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i16> [[TMP9]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP9]], <8 x i16> [[TMP10]])
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i16> [[TMP11]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP16:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP11]], <8 x i16> [[TMP12]])
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i16> [[TMP16]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[A0]])
; CHECK-NEXT: [[TMP15:%.*]] = tail call i16 @llvm.umin.i16(i16 [[TMP13]], i16 [[TMP14]])
; CHECK-NEXT: ret i16 [[TMP15]]
;
@@ -144,16 +133,11 @@ define i16 @test_reduce_v8i16_2(<8 x i16> %a0) {
ret i16 %15
}
-define i16 @test_reduce_v8i16_neg1(<8 x i16> %a0) {
-; CHECK-LABEL: define i16 @test_reduce_v8i16_neg1(
+define i16 @test_reduce_v8i16_skip_lane7(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16_skip_lane7(
; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP4:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP5]])
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT: [[TMP7:%.*]] = call i16 @llvm.vector.reduce.umin.v7i16(<7 x i16> [[TMP1]])
; CHECK-NEXT: ret i16 [[TMP7]]
;
%1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -166,16 +150,15 @@ define i16 @test_reduce_v8i16_neg1(<8 x i16> %a0) {
ret i16 %7
}
-define i16 @test_reduce_v8i16_neg2(<8 x i16> %a0) {
-; CHECK-LABEL: define i16 @test_reduce_v8i16_neg2(
+define i16 @test_reduce_v8i16_mixed_ops_tail(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16_mixed_ops_tail(
; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP2:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP4:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP5]])
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP7:%.*]] = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> [[TMP5]])
; CHECK-NEXT: ret i16 [[TMP7]]
;
%1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -188,17 +171,10 @@ define i16 @test_reduce_v8i16_neg2(<8 x i16> %a0) {
ret i16 %7
}
-define i16 @test_reduce_v8i16_neg3(<8 x i16> %a0) {
-; CHECK-LABEL: define i16 @test_reduce_v8i16_neg3(
+define i16 @test_reduce_v8i16_ssa_aliased(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16_ssa_aliased(
; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP4:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP5]], <8 x i16> [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i16> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[A0]])
; CHECK-NEXT: ret i16 [[TMP8]]
;
%1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -212,16 +188,10 @@ define i16 @test_reduce_v8i16_neg3(<8 x i16> %a0) {
ret i16 %8
}
-define i16 @test_reduce_v6i16_xor_neg(<6 x i16> %a0) {
-; CHECK-LABEL: define i16 @test_reduce_v6i16_xor_neg(
+define i16 @test_reduce_v6i16_xor_poison_refines(<6 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v6i16_xor_poison_refines(
; CHECK-SAME: <6 x i16> [[A0:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <6 x i16> [[A0]], <6 x i16> poison, <6 x i32> <i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = xor <6 x i16> [[A0]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x i16> [[TMP2]], <6 x i16> poison, <6 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP4:%.*]] = xor <6 x i16> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <6 x i16> [[TMP4]], <6 x i16> poison, <6 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = xor <6 x i16> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <6 x i16> [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = call i16 @llvm.vector.reduce.xor.v6i16(<6 x i16> [[A0]])
; CHECK-NEXT: ret i16 [[TMP7]]
;
%1 = shufflevector <6 x i16> %a0, <6 x i16> poison, <6 x i32> <i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison>
@@ -284,8 +254,7 @@ define i16 @test_partial_reduce_v16i16_v4i16_umin(<16 x i16> %a0) {
ret i16 %5
}
-; Negative test: partial reduction on non-power-of-2 vectors is rejected because
-; parity-based shuffle masks cause lane duplication in the reduction tree.
+; Negative: chain duplicates lane in the result (e.g. lane 0 = a[0]+2*a[1]+a[2]). Add is non-idempotent, so folding would miscompile.
define i32 @test_no_partial_reduce_v6i32_add(<6 x i32> %a) {
; CHECK-LABEL: define i32 @test_no_partial_reduce_v6i32_add(
; CHECK-SAME: <6 x i32> [[A:%.*]]) {
>From 036ac12820f76f596b56ac098215aedf510999ab Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Thu, 28 May 2026 03:25:02 +0000
Subject: [PATCH 2/7] [VectorCombine] Pass SubTp to getShuffleCost in
foldShuffleChainsToReduce
---
llvm/lib/Transforms/Vectorize/VectorCombine.cpp | 10 ++++++----
.../AArch64/partial-reduce-crash.ll | 17 +++++++++++++++++
2 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index f40ad832b6c08..0a897bc3096fc 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -4184,10 +4184,12 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
if (auto *SVI = dyn_cast<ShuffleVectorInst>(V)) {
auto *VT = cast<FixedVectorType>(SVI->getType());
auto *SrcVT = cast<FixedVectorType>(SVI->getOperand(0)->getType());
- auto Kind = (VT == SrcVT) ? TargetTransformInfo::SK_PermuteSingleSrc
- : TargetTransformInfo::SK_ExtractSubvector;
- OrigCost +=
- TTI.getShuffleCost(Kind, VT, SrcVT, SVI->getShuffleMask(), CostKind);
+ int Index = 0;
+ auto Kind = SVI->isExtractSubvectorMask(Index)
+ ? TargetTransformInfo::SK_ExtractSubvector
+ : TargetTransformInfo::SK_PermuteSingleSrc;
+ OrigCost += TTI.getShuffleCost(Kind, VT, SrcVT, SVI->getShuffleMask(),
+ CostKind, Index, VT);
} else if (auto *BO = dyn_cast<BinaryOperator>(V)) {
OrigCost +=
TTI.getArithmeticInstrCost(BO->getOpcode(), BO->getType(), CostKind);
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/partial-reduce-crash.ll b/llvm/test/Transforms/VectorCombine/AArch64/partial-reduce-crash.ll
index 85611e1f86aab..82c241fdf3010 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/partial-reduce-crash.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/partial-reduce-crash.ll
@@ -22,3 +22,20 @@ define i32 @partial_reduce_extract_subvector_crash(<4 x i32> %vec) {
%result = extractelement <4 x i32> %add, i32 0
ret i32 %result
}
+
+; Same crash with a two-shuffle subvector split (256 -> 128).
+define i32 @subvector_split_crash(<8 x i32> %vec) {
+; CHECK-LABEL: define i32 @subvector_split_crash(
+; CHECK-SAME: <8 x i32> [[VEC:%.*]]) {
+; CHECK-NEXT: [[LO:%.*]] = shufflevector <8 x i32> [[VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[HI:%.*]] = shufflevector <8 x i32> [[VEC]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[ADD:%.*]] = add <4 x i32> [[LO]], [[HI]]
+; CHECK-NEXT: [[RESULT:%.*]] = extractelement <4 x i32> [[ADD]], i32 0
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+ %lo = shufflevector <8 x i32> %vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %hi = shufflevector <8 x i32> %vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %add = add <4 x i32> %lo, %hi
+ %result = extractelement <4 x i32> %add, i32 0
+ ret i32 %result
+}
>From 27b6764b861b433ff02ab93c6873c55f8ecc8d24 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Mon, 1 Jun 2026 22:34:21 +0000
Subject: [PATCH 3/7] [VectorCombine] Simplify foldShuffleChainsToReduce
Use TTI.getInstructionCost for the original cost loop, merge the demote
pass into the DFS, and rename the per lane masks to match DemandedElts
naming.
---
.../Transforms/Vectorize/VectorCombine.cpp | 126 +++++++-----------
1 file changed, 49 insertions(+), 77 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 0a897bc3096fc..d334485f6878b 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -4012,9 +4012,19 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
constexpr unsigned MaxChainNodes = 32;
SmallVector<Value *, 16> ChainPostorder;
SmallPtrSet<Value *, 16> Visited;
+ SmallPtrSet<Value *, 16> KeptChain;
SmallVector<Value *, 2> Sources;
DenseMap<Value *, unsigned> SrcSizes;
+ auto AddSource = [&](Value *V) -> bool {
+ auto *VT = dyn_cast<FixedVectorType>(V->getType());
+ if (!VT)
+ return false;
+ if (SrcSizes.insert({V, VT->getNumElements()}).second)
+ Sources.push_back(V);
+ return true;
+ };
+
struct StackEntry {
Value *V;
unsigned ChildIdx;
@@ -4026,14 +4036,8 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
return false;
if (!Visited.insert(V).second)
return true;
- if (!IsChainNode(V)) {
- auto *VT = dyn_cast<FixedVectorType>(V->getType());
- if (!VT)
- return false;
- if (SrcSizes.insert({V, VT->getNumElements()}).second)
- Sources.push_back(V);
- return true;
- }
+ if (!IsChainNode(V))
+ return AddSource(V);
Stack.push_back({V, 0});
return true;
};
@@ -4053,43 +4057,32 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
if (!Enqueue(Child))
return false;
} else {
- ChainPostorder.push_back(V);
+ // Demote any binop/intrinsic whose operands are not themselves chain
+ // nodes. Walking past would re-derive the value from its operands,
+ // leaving the original alive if anything downstream still uses it.
+ bool Keep = isa<ShuffleVectorInst>(V);
+ if (!Keep) {
+ auto *U = cast<User>(V);
+ Keep = KeptChain.contains(U->getOperand(0)) ||
+ KeptChain.contains(U->getOperand(1));
+ }
+ if (Keep) {
+ KeptChain.insert(V);
+ ChainPostorder.push_back(V);
+ } else if (!AddSource(V)) {
+ return false;
+ }
Stack.pop_back();
}
}
- // Demote any binop/intrinsic whose operands are not themselves chain nodes.
- // Walking past would re-derive the value from its operands, leaving the
- // original alive if anything downstream still uses it.
- SmallPtrSet<Value *, 16> KeptChain;
- SmallVector<Value *, 16> NewChainPostorder;
- NewChainPostorder.reserve(ChainPostorder.size());
- for (Value *V : ChainPostorder) {
- if (isa<ShuffleVectorInst>(V)) {
- KeptChain.insert(V);
- NewChainPostorder.push_back(V);
- continue;
- }
- auto *U = cast<User>(V);
- if (KeptChain.contains(U->getOperand(0)) ||
- KeptChain.contains(U->getOperand(1))) {
- KeptChain.insert(V);
- NewChainPostorder.push_back(V);
- continue;
- }
- auto *VT = cast<FixedVectorType>(V->getType());
- if (SrcSizes.insert({V, VT->getNumElements()}).second)
- Sources.push_back(V);
- }
- ChainPostorder = std::move(NewChainPostorder);
-
bool IsIdempotent =
CommonCallOp || (CommonBinOp && Instruction::isIdempotent(*CommonBinOp));
// Each output lane has a bitmask of contributing source lanes and a
// poison flag. Shuffles permute these records. Binops union them.
struct LaneInfo {
- SmallDenseMap<Value *, APInt, 2> SrcBits;
+ SmallDenseMap<Value *, APInt, 2> SrcElts;
bool IsPoison = false;
};
DenseMap<Value *, SmallVector<LaneInfo, 16>> Attr;
@@ -4099,7 +4092,7 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
A.reserve(N);
for (unsigned i = 0; i < N; ++i) {
LaneInfo LI;
- LI.SrcBits[Src] = APInt::getOneBitSet(N, i);
+ LI.SrcElts[Src] = APInt::getOneBitSet(N, i);
A.push_back(std::move(LI));
}
}
@@ -4133,18 +4126,16 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
for (unsigned i = 0; i < L.size(); ++i) {
LaneInfo OutLI;
OutLI.IsPoison = L[i].IsPoison || R[i].IsPoison;
- for (auto &SBP : L[i].SrcBits)
- OutLI.SrcBits.insert(SBP);
- for (auto &SBP : R[i].SrcBits) {
- auto [It, Inserted] =
- OutLI.SrcBits.try_emplace(SBP.first, SBP.second);
+ OutLI.SrcElts.insert(L[i].SrcElts.begin(), L[i].SrcElts.end());
+ for (auto &[RSrc, RMask] : R[i].SrcElts) {
+ auto [It, Inserted] = OutLI.SrcElts.try_emplace(RSrc, RMask);
if (!Inserted) {
// x op x != x for non-idempotent ops but poison lanes can be
// refined.
- if (It->second.intersects(SBP.second) && !IsIdempotent &&
+ if (It->second.intersects(RMask) && !IsIdempotent &&
!OutLI.IsPoison)
return false;
- It->second |= SBP.second;
+ It->second |= RMask;
}
}
Out.push_back(std::move(OutLI));
@@ -4157,18 +4148,18 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
struct Partial {
Value *Src;
- APInt Bits;
+ APInt Elts;
unsigned SrcSize;
};
SmallVector<Partial, 2> Partials;
for (Value *Src : Sources) {
- auto It = Lane0.SrcBits.find(Src);
- if (It == Lane0.SrcBits.end())
+ auto It = Lane0.SrcElts.find(Src);
+ if (It == Lane0.SrcElts.end())
continue;
- const APInt &Bits = It->second;
- if (Bits.popcount() == 0)
+ const APInt &Elts = It->second;
+ if (Elts.popcount() == 0)
continue;
- Partials.push_back({Src, Bits, SrcSizes[Src]});
+ Partials.push_back({Src, Elts, SrcSizes[Src]});
}
if (Partials.empty())
return false;
@@ -4180,27 +4171,8 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
return false;
InstructionCost OrigCost = 0;
- for (Value *V : ChainPostorder) {
- if (auto *SVI = dyn_cast<ShuffleVectorInst>(V)) {
- auto *VT = cast<FixedVectorType>(SVI->getType());
- auto *SrcVT = cast<FixedVectorType>(SVI->getOperand(0)->getType());
- int Index = 0;
- auto Kind = SVI->isExtractSubvectorMask(Index)
- ? TargetTransformInfo::SK_ExtractSubvector
- : TargetTransformInfo::SK_PermuteSingleSrc;
- OrigCost += TTI.getShuffleCost(Kind, VT, SrcVT, SVI->getShuffleMask(),
- CostKind, Index, VT);
- } else if (auto *BO = dyn_cast<BinaryOperator>(V)) {
- OrigCost +=
- TTI.getArithmeticInstrCost(BO->getOpcode(), BO->getType(), CostKind);
- } else {
- auto *II = cast<IntrinsicInst>(V);
- IntrinsicCostAttributes ICA(
- II->getIntrinsicID(), II->getType(),
- {II->getOperand(0)->getType(), II->getOperand(1)->getType()});
- OrigCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
- }
- }
+ for (Value *V : ChainPostorder)
+ OrigCost += TTI.getInstructionCost(cast<Instruction>(V), CostKind);
// Each source contributes a gather+reduce, or an extractelement for a
// single-lane partial. Scalar ops then combine the partials.
@@ -4208,9 +4180,9 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
InstructionCost NewCost = 0;
for (const Partial &P : Partials) {
auto *SrcVT = cast<FixedVectorType>(P.Src->getType());
- unsigned Pop = P.Bits.popcount();
+ unsigned Pop = P.Elts.popcount();
if (Pop == 1) {
- unsigned Idx = P.Bits.countTrailingZeros();
+ unsigned Idx = P.Elts.countTrailingZeros();
NewCost += TTI.getVectorInstrCost(Instruction::ExtractElement, SrcVT,
CostKind, Idx);
continue;
@@ -4221,13 +4193,13 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
SmallVector<int> Mask;
Mask.reserve(Pop);
for (unsigned i = 0; i < P.SrcSize; ++i)
- if (P.Bits[i])
+ if (P.Elts[i])
Mask.push_back(i);
// A contiguous run of bits (anywhere, not just the low prefix) is an
// extract subvector. Pass its offset so TTI can model targets that
// lower upper-half extracts cheaply.
unsigned SubIdx = 0, SubLen;
- auto Kind = P.Bits.isShiftedMask(SubIdx, SubLen)
+ auto Kind = P.Elts.isShiftedMask(SubIdx, SubLen)
? TargetTransformInfo::SK_ExtractSubvector
: TargetTransformInfo::SK_PermuteSingleSrc;
NewCost +=
@@ -4256,10 +4228,10 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
SmallVector<Value *, 2> PartialResults;
for (const Partial &P : Partials) {
- unsigned Pop = P.Bits.popcount();
+ unsigned Pop = P.Elts.popcount();
// A single-lane partial is just the source lane. No reduce needed.
if (Pop == 1) {
- unsigned Idx = P.Bits.countTrailingZeros();
+ unsigned Idx = P.Elts.countTrailingZeros();
PartialResults.push_back(Builder.CreateExtractElement(P.Src, Idx));
continue;
}
@@ -4268,7 +4240,7 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
SmallVector<int> Mask;
Mask.reserve(Pop);
for (unsigned i = 0; i < P.SrcSize; ++i)
- if (P.Bits[i])
+ if (P.Elts[i])
Mask.push_back(i);
ReduceInput = Builder.CreateShuffleVector(P.Src, Mask);
}
>From e168cc4868bd82924ea85cd5a8127d3fade5fd9c Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Wed, 10 Jun 2026 07:23:44 +0000
Subject: [PATCH 4/7] [VectorCombine] foldShuffleChainsToReduce - handle
FADD/FMUL
---
.../Transforms/Vectorize/VectorCombine.cpp | 101 ++++++++++--------
1 file changed, 58 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index d334485f6878b..c2d4e53778e4d 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -3927,43 +3927,20 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
return MadeChanges;
}
-/// For a given chain of patterns of the following form:
+/// Try to fold a chain of shuffles and ops feeding extractelement(..., 0)
+/// into llvm.vector.reduce.* over the source vector(s), by tracking which
+/// source lanes contribute to the extracted lane. For example:
///
-/// ```
-/// %1 = shufflevector <n x ty1> %0, <n x ty1> poison <n x ty2> mask
-///
-/// %2 = tail call <n x ty1> llvm.<umin/umax/smin/smax>(<n x ty1> %0, <n x
-/// ty1> %1)
-/// OR
-/// %2 = add/mul/or/and/xor <n x ty1> %0, %1
-///
-/// %3 = shufflevector <n x ty1> %2, <n x ty1> poison <n x ty2> mask
-/// ...
-/// ...
-/// %(i - 1) = tail call <n x ty1> llvm.<umin/umax/smin/smax>(<n x ty1> %(i -
-/// 3), <n x ty1> %(i - 2)
-/// OR
-/// %(i - 1) = add/mul/or/and/xor <n x ty1> %(i - 3), %(i - 2)
+/// %lo = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 0, i32 1>
+/// %hi = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 2, i32 3>
+/// %s = add <2 x i32> %lo, %hi
+/// %sh = shufflevector <2 x i32> %s, poison, <2 x i32> <i32 1, i32 poison>
+/// %r = add <2 x i32> %s, %sh
+/// %e = extractelement <2 x i32> %r, i64 0
///
-/// %(i) = extractelement <n x ty1> %(i - 1), 0
-/// ```
+/// transforms to:
///
-/// Where:
-/// `mask` follows a partition pattern:
-///
-/// Ex:
-/// [n = 8, p = poison]
-///
-/// 4 5 6 7 | p p p p
-/// 2 3 | p p p p p p
-/// 1 | p p p p p p p
-///
-/// Ex:
-/// [n = 6]
-///
-/// 3 4 5 | p p p
-/// 1 2 | p p p p
-/// 1 | p p p p p
+/// %e = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
Value *VecOpEE;
if (!match(&I, m_ExtractElt(m_Value(VecOpEE), m_Zero())))
@@ -3986,6 +3963,8 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
case Instruction::Or:
case Instruction::And:
case Instruction::Xor:
+ case Instruction::FAdd:
+ case Instruction::FMul:
CommonBinOp = BO->getOpcode();
break;
default:
@@ -3997,6 +3976,10 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
return false;
}
+ // For floating-point reductions, track FMF intersection across all binops.
+ FastMathFlags CommonFMF;
+ bool IsFloatReduction = false;
+
auto IsChainNode = [&](Value *V) {
if (auto *BO = dyn_cast<BinaryOperator>(V))
return CommonBinOp && BO->getOpcode() == *CommonBinOp;
@@ -4076,6 +4059,11 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
}
}
+ // No chain nodes (e.g. extract(binop(x, y), 0)). Rebuilding would just
+ // recreate the original extract and refold forever when extraction is free.
+ if (ChainPostorder.empty())
+ return false;
+
bool IsIdempotent =
CommonCallOp || (CommonBinOp && Instruction::isIdempotent(*CommonBinOp));
@@ -4117,6 +4105,19 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
}
} else {
auto *U = cast<User>(V);
+ // For FP reductions, require reassoc on every binop and collect FMF.
+ if (auto *BinOp = dyn_cast<BinaryOperator>(V);
+ BinOp && (*CommonBinOp == Instruction::FAdd ||
+ *CommonBinOp == Instruction::FMul)) {
+ if (!BinOp->hasAllowReassoc())
+ return false;
+ if (!IsFloatReduction) {
+ CommonFMF = BinOp->getFastMathFlags();
+ IsFloatReduction = true;
+ } else {
+ CommonFMF &= BinOp->getFastMathFlags();
+ }
+ }
const auto &L = Attr.at(U->getOperand(0));
const auto &R = Attr.at(U->getOperand(1));
if (L.size() != R.size())
@@ -4156,10 +4157,7 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
auto It = Lane0.SrcElts.find(Src);
if (It == Lane0.SrcElts.end())
continue;
- const APInt &Elts = It->second;
- if (Elts.popcount() == 0)
- continue;
- Partials.push_back({Src, Elts, SrcSizes[Src]});
+ Partials.push_back({Src, It->second, SrcSizes[Src]});
}
if (Partials.empty())
return false;
@@ -4205,7 +4203,11 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
NewCost +=
TTI.getShuffleCost(Kind, RVT, SrcVT, Mask, CostKind, SubIdx, RVT);
}
- IntrinsicCostAttributes ICA(ReducedOp, RVT, {RVT});
+ IntrinsicCostAttributes ICA(ReducedOp, ElTy,
+ IsFloatReduction
+ ? SmallVector<Type *, 2>{ElTy, RVT}
+ : SmallVector<Type *, 2>{RVT},
+ IsFloatReduction ? CommonFMF : FastMathFlags());
NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
}
for (unsigned i = 1; i < Partials.size(); ++i) {
@@ -4244,14 +4246,27 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
Mask.push_back(i);
ReduceInput = Builder.CreateShuffleVector(P.Src, Mask);
}
- PartialResults.push_back(Builder.CreateIntrinsic(
- ReducedOp, {ReduceInput->getType()}, {ReduceInput}));
+ if (IsFloatReduction) {
+ Value *Identity = ConstantExpr::getBinOpIdentity(
+ *CommonBinOp, ElTy, /*AllowRHSConstant=*/false,
+ CommonFMF.noSignedZeros());
+ auto *Rdx = Builder.CreateIntrinsic(ReducedOp, {ReduceInput->getType()},
+ {Identity, ReduceInput});
+ Rdx->setFastMathFlags(CommonFMF);
+ PartialResults.push_back(Rdx);
+ } else {
+ PartialResults.push_back(Builder.CreateIntrinsic(
+ ReducedOp, {ReduceInput->getType()}, {ReduceInput}));
+ }
}
Value *Result = PartialResults[0];
for (unsigned i = 1; i < PartialResults.size(); ++i) {
- if (CommonBinOp)
+ if (CommonBinOp) {
Result = Builder.CreateBinOp(*CommonBinOp, Result, PartialResults[i]);
- else
+ if (IsFloatReduction)
+ if (auto *RI = dyn_cast<Instruction>(Result))
+ RI->setFastMathFlags(CommonFMF);
+ } else
Result = Builder.CreateIntrinsic(*CommonCallOp, {ElTy},
{Result, PartialResults[i]});
}
>From 984eecf9b079222f4eacb06683efd68e1691b1c3 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Thu, 11 Jun 2026 12:16:21 +0000
Subject: [PATCH 5/7] [VectorCombine] Rework foldShuffleChainsToReduce to
top-down demanded elements instead of bottom-up
---
.../Transforms/Vectorize/VectorCombine.cpp | 174 ++++++++++--------
.../X86/shuffle-chain-reduction-subvector.ll | 20 +-
.../fold-shuffle-chains-to-reduce.ll | 15 +-
3 files changed, 119 insertions(+), 90 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index c2d4e53778e4d..a2b6205175676 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -3928,8 +3928,11 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
}
/// Try to fold a chain of shuffles and ops feeding extractelement(..., 0)
-/// into llvm.vector.reduce.* over the source vector(s), by tracking which
-/// source lanes contribute to the extracted lane. For example:
+/// into llvm.vector.reduce.*, by tracking which lanes contribute to the
+/// extracted lane and reducing the widest vector whose lanes each contribute
+/// once.
+///
+/// For example:
///
/// %lo = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 0, i32 1>
/// %hi = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 2, i32 3>
@@ -4067,101 +4070,126 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
bool IsIdempotent =
CommonCallOp || (CommonBinOp && Instruction::isIdempotent(*CommonBinOp));
- // Each output lane has a bitmask of contributing source lanes and a
- // poison flag. Shuffles permute these records. Binops union them.
- struct LaneInfo {
- SmallDenseMap<Value *, APInt, 2> SrcElts;
- bool IsPoison = false;
- };
- DenseMap<Value *, SmallVector<LaneInfo, 16>> Attr;
- for (Value *Src : Sources) {
- unsigned N = SrcSizes[Src];
- auto &A = Attr[Src];
- A.reserve(N);
- for (unsigned i = 0; i < N; ++i) {
- LaneInfo LI;
- LI.SrcElts[Src] = APInt::getOneBitSet(N, i);
- A.push_back(std::move(LI));
+ // For FP reductions, require reassoc on every binop and collect FMF.
+ for (Value *V : ChainPostorder) {
+ auto *BinOp = dyn_cast<BinaryOperator>(V);
+ if (!BinOp || !CommonBinOp ||
+ (*CommonBinOp != Instruction::FAdd &&
+ *CommonBinOp != Instruction::FMul))
+ continue;
+ if (!BinOp->hasAllowReassoc())
+ return false;
+ if (!IsFloatReduction) {
+ CommonFMF = BinOp->getFastMathFlags();
+ IsFloatReduction = true;
+ } else {
+ CommonFMF &= BinOp->getFastMathFlags();
}
}
- // Postorder ensures each value's operands are already in Attr.
- for (Value *V : ChainPostorder) {
+ // Top-down demanded elements. How many times does each lane of each chain
+ // value feed the extracted lane 0? Reverse postorder visits every use before
+ // its value. A binop forwards its demand to both operands and a shuffle
+ // follows its mask back to the source lane.
+ DenseMap<Value *, SmallVector<unsigned, 16>> Demand;
+ auto DemandOf = [&](Value *V) -> SmallVector<unsigned, 16> & {
+ auto &D = Demand[V];
+ if (D.empty())
+ D.assign(cast<FixedVectorType>(V->getType())->getNumElements(), 0);
+ return D;
+ };
+ DemandOf(VecOpEE)[0] = 1;
+ for (Value *V : reverse(ChainPostorder)) {
+ SmallVector<unsigned, 16> DV = Demand.lookup(V);
+ if (DV.empty())
+ continue;
if (auto *SVI = dyn_cast<ShuffleVectorInst>(V)) {
- const auto &In = Attr.at(SVI->getOperand(0));
- auto &Out = Attr[V];
ArrayRef<int> Mask = SVI->getShuffleMask();
- Out.reserve(Mask.size());
- for (int M : Mask) {
- if (M < 0) {
- LaneInfo LI;
- LI.IsPoison = true;
- Out.push_back(std::move(LI));
- } else if ((unsigned)M >= In.size()) {
- return false;
- } else {
- Out.push_back(In[M]);
- }
- }
+ auto &DS = DemandOf(SVI->getOperand(0));
+ for (unsigned i = 0, e = Mask.size(); i != e; ++i)
+ if (Mask[i] >= 0 && (unsigned)Mask[i] < DS.size())
+ DS[Mask[i]] += DV[i];
} else {
auto *U = cast<User>(V);
- // For FP reductions, require reassoc on every binop and collect FMF.
- if (auto *BinOp = dyn_cast<BinaryOperator>(V);
- BinOp && (*CommonBinOp == Instruction::FAdd ||
- *CommonBinOp == Instruction::FMul)) {
- if (!BinOp->hasAllowReassoc())
- return false;
- if (!IsFloatReduction) {
- CommonFMF = BinOp->getFastMathFlags();
- IsFloatReduction = true;
- } else {
- CommonFMF &= BinOp->getFastMathFlags();
- }
- }
- const auto &L = Attr.at(U->getOperand(0));
- const auto &R = Attr.at(U->getOperand(1));
- if (L.size() != R.size())
- return false;
- auto &Out = Attr[V];
- Out.reserve(L.size());
- for (unsigned i = 0; i < L.size(); ++i) {
- LaneInfo OutLI;
- OutLI.IsPoison = L[i].IsPoison || R[i].IsPoison;
- OutLI.SrcElts.insert(L[i].SrcElts.begin(), L[i].SrcElts.end());
- for (auto &[RSrc, RMask] : R[i].SrcElts) {
- auto [It, Inserted] = OutLI.SrcElts.try_emplace(RSrc, RMask);
- if (!Inserted) {
- // x op x != x for non-idempotent ops but poison lanes can be
- // refined.
- if (It->second.intersects(RMask) && !IsIdempotent &&
- !OutLI.IsPoison)
- return false;
- It->second |= RMask;
- }
- }
- Out.push_back(std::move(OutLI));
+ for (Value *Op : {U->getOperand(0), U->getOperand(1)}) {
+ auto &DOp = DemandOf(Op);
+ for (unsigned i = 0, e = DV.size(); i != e && i < DOp.size(); ++i)
+ DOp[i] += DV[i];
}
}
}
- // One partial reduce per source contributing to lane 0.
- const LaneInfo &Lane0 = Attr.at(VecOpEE)[0];
+ // A vector is cleanly reducible if no demanded lane is counted more than once
+ // (idempotent ops tolerate duplicates).
+ auto Clean = [&](ArrayRef<unsigned> D) {
+ bool Any = false;
+ for (unsigned C : D) {
+ if (!C)
+ continue;
+ Any = true;
+ if (C > 1 && !IsIdempotent)
+ return false;
+ }
+ return Any;
+ };
+ auto LaneMask = [](ArrayRef<unsigned> D, unsigned N) {
+ APInt E(N, 0);
+ for (unsigned i = 0, e = D.size(); i != e; ++i)
+ if (D[i])
+ E.setBit(i);
+ return E;
+ };
struct Partial {
Value *Src;
APInt Elts;
unsigned SrcSize;
};
+ // Reduce the leaf sources when they are cleanly demanded, otherwise the
+ // deepest intermediate whose lanes all feed the result.
SmallVector<Partial, 2> Partials;
+ bool SourcesClean = true;
for (Value *Src : Sources) {
- auto It = Lane0.SrcElts.find(Src);
- if (It == Lane0.SrcElts.end())
+ SmallVector<unsigned, 16> D = Demand.lookup(Src);
+ if (!Clean(D)) {
+ if (any_of(D, [](unsigned C) { return C; }))
+ SourcesClean = false;
+ continue;
+ }
+ Partials.push_back({Src, LaneMask(D, SrcSizes[Src]), SrcSizes[Src]});
+ }
+ // The deepest intermediate whose lanes all feed the result, each exactly once
+ // for non-idempotent ops.
+ Value *FullInter = nullptr;
+ for (Value *V : ChainPostorder) {
+ if (!isa<BinaryOperator>(V))
continue;
- Partials.push_back({Src, It->second, SrcSizes[Src]});
+ SmallVector<unsigned, 16> D = Demand.lookup(V);
+ if (!D.empty() && all_of(D, [&](unsigned C) {
+ return C >= 1 && (IsIdempotent || C == 1);
+ })) {
+ FullInter = V;
+ break;
+ }
+ }
+
+ // Multi-source chains also take the intermediate, since one reduce replaces
+ // the per-source reduces.
+ if (FullInter && (!SourcesClean || Partials.size() != 1)) {
+ unsigned N = cast<FixedVectorType>(FullInter->getType())->getNumElements();
+ Partials.clear();
+ Partials.push_back({FullInter, APInt::getAllOnes(N), N});
+ } else if (!SourcesClean) {
+ return false;
}
if (Partials.empty())
return false;
+ // A real reduction combines >1 lane of some source. Multiple single-lane
+ // would loop.
+ if (none_of(Partials, [](const Partial &P) { return P.Elts.popcount() > 1; }))
+ return false;
+
Intrinsic::ID ReducedOp =
(CommonCallOp ? getMinMaxReductionIntrinsicID(*CommonCallOp)
: getReductionForBinop(*CommonBinOp));
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
index f59915fce9c3b..599ba8e30bb47 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
@@ -170,17 +170,13 @@ define i32 @test_multisource_full_plus_scalar_v4i32(<4 x i32> %x, <4 x i32> %y)
ret i32 %e
}
-define i16 @test_neg_overlapping_masks_v8i16(<8 x i16> %a0) {
-; CHECK-LABEL: define i16 @test_neg_overlapping_masks_v8i16(
+define i16 @test_overlapping_masks_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_overlapping_masks_v8i16(
; CHECK-SAME: <8 x i16> [[A0:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i16> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP8]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP5]], <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i16> [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i16> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP3]])
; CHECK-NEXT: ret i16 [[TMP4]]
;
%1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -194,17 +190,13 @@ define i16 @test_neg_overlapping_masks_v8i16(<8 x i16> %a0) {
ret i16 %8
}
-define i16 @test_neg_different_sources_v8i16(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: define i16 @test_neg_different_sources_v8i16(
+define i16 @test_different_sources_v8i16(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: define i16 @test_different_sources_v8i16(
; CHECK-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[A1]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i16> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP8]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP5]], <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i16> [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i16> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP3]])
; CHECK-NEXT: ret i16 [[TMP4]]
;
%1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
index 196a1cc53ab4a..0cad902db0fa4 100644
--- a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
+++ b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
@@ -188,10 +188,19 @@ define i16 @test_reduce_v8i16_ssa_aliased(<8 x i16> %a0) {
ret i16 %8
}
-define i16 @test_reduce_v6i16_xor_poison_refines(<6 x i16> %a0) {
-; CHECK-LABEL: define i16 @test_reduce_v6i16_xor_poison_refines(
+; Lane 0 is poison: the shuffle padding flows through the xor chain. Folding to
+; reduce.xor(%a0) would be a valid refinement, but a poison-producing chain isn't
+; a real reduction pattern, so it is left alone.
+define i16 @test_no_reduce_v6i16_xor_poison(<6 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_no_reduce_v6i16_xor_poison(
; CHECK-SAME: <6 x i16> [[A0:%.*]]) {
-; CHECK-NEXT: [[TMP7:%.*]] = call i16 @llvm.vector.reduce.xor.v6i16(<6 x i16> [[A0]])
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <6 x i16> [[A0]], <6 x i16> poison, <6 x i32> <i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = xor <6 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x i16> [[TMP2]], <6 x i16> poison, <6 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = xor <6 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <6 x i16> [[TMP4]], <6 x i16> poison, <6 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP6:%.*]] = xor <6 x i16> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <6 x i16> [[TMP6]], i64 0
; CHECK-NEXT: ret i16 [[TMP7]]
;
%1 = shufflevector <6 x i16> %a0, <6 x i16> poison, <6 x i32> <i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison>
>From 00403bcc96601e4d37dab8f72f9977fed5df9f80 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Thu, 11 Jun 2026 14:10:28 +0000
Subject: [PATCH 6/7] [VectorCombine] Simplify subvector FP path in
foldShuffleChainsToReduce
---
.../lib/Transforms/Vectorize/VectorCombine.cpp | 18 +++---------------
.../X86/shuffle-chain-reduction-subvector.ll | 15 +++++++++++++++
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index a2b6205175676..ff02b64309d32 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -3960,19 +3960,9 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
std::optional<Intrinsic::ID> CommonCallOp;
if (auto *BO = dyn_cast<BinaryOperator>(VecOpEE)) {
- switch (BO->getOpcode()) {
- case Instruction::Add:
- case Instruction::Mul:
- case Instruction::Or:
- case Instruction::And:
- case Instruction::Xor:
- case Instruction::FAdd:
- case Instruction::FMul:
- CommonBinOp = BO->getOpcode();
- break;
- default:
+ if (!getReductionForBinop(BO->getOpcode()))
return false;
- }
+ CommonBinOp = BO->getOpcode();
} else if (auto *MMI = dyn_cast<MinMaxIntrinsic>(VecOpEE)) {
CommonCallOp = MMI->getIntrinsicID();
} else {
@@ -4073,9 +4063,7 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
// For FP reductions, require reassoc on every binop and collect FMF.
for (Value *V : ChainPostorder) {
auto *BinOp = dyn_cast<BinaryOperator>(V);
- if (!BinOp || !CommonBinOp ||
- (*CommonBinOp != Instruction::FAdd &&
- *CommonBinOp != Instruction::FMul))
+ if (!BinOp || !BinOp->getType()->isFPOrFPVectorTy())
continue;
if (!BinOp->hasAllowReassoc())
return false;
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
index 599ba8e30bb47..471cbc0b5198e 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-subvector.ll
@@ -131,6 +131,21 @@ define i16 @test_subvector_reduce_deinterleave_v8i16(<8 x i16> %a0) {
ret i16 %8
}
+define float @test_subvector_reduce_fadd_v4f32(<4 x float> %a0) {
+; CHECK-LABEL: define float @test_subvector_reduce_fadd_v4f32(
+; CHECK-SAME: <4 x float> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[A0]])
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = shufflevector <4 x float> %a0, <4 x float> poison, <2 x i32> <i32 0, i32 1>
+ %2 = shufflevector <4 x float> %a0, <4 x float> poison, <2 x i32> <i32 2, i32 3>
+ %3 = fadd reassoc <2 x float> %1, %2
+ %4 = shufflevector <2 x float> %3, <2 x float> poison, <2 x i32> <i32 1, i32 poison>
+ %5 = fadd reassoc <2 x float> %3, %4
+ %6 = extractelement <2 x float> %5, i64 0
+ ret float %6
+}
+
define i32 @test_multishuffle_accumulator_v8i32(<8 x i32> %a0) {
; CHECK-LABEL: define i32 @test_multishuffle_accumulator_v8i32(
; CHECK-SAME: <8 x i32> [[A0:%.*]]) #[[ATTR0]] {
>From d310375573a7df5515347ad04196ef6f432955d0 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Fri, 12 Jun 2026 09:57:53 +0000
Subject: [PATCH 7/7] [VectorCombine] foldShuffleChainsToReduce reject
intermediates that don't cover the chain
Require every path from the extract to stay inside the chain until it
reaches the intermediate.
Miscompilation found by llvm-hackme.
---
.../Transforms/Vectorize/VectorCombine.cpp | 43 +++++++++++++++----
.../fold-shuffle-chains-to-reduce.ll | 23 ++++++++++
2 files changed, 57 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index ff02b64309d32..cce6a37713636 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -4146,19 +4146,44 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
}
Partials.push_back({Src, LaneMask(D, SrcSizes[Src]), SrcSizes[Src]});
}
- // The deepest intermediate whose lanes all feed the result, each exactly once
- // for non-idempotent ops.
+ // Reducing V replaces the entire chain, so every contribution to the result
+ // must flow through V. Reject if anything above V reads outside the chain.
+ auto CoversChain = [&](Value *V) {
+ SmallVector<Value *, 8> Worklist(1, VecOpEE);
+ SmallPtrSet<Value *, 8> Seen;
+ Seen.insert(VecOpEE);
+ while (!Worklist.empty()) {
+ auto *U = cast<Instruction>(Worklist.pop_back_val());
+ unsigned NumOps = isa<ShuffleVectorInst>(U) ? 1 : 2;
+ for (unsigned I = 0; I != NumOps; ++I) {
+ Value *Op = U->getOperand(I);
+ if (Op == V || !Seen.insert(Op).second)
+ continue;
+ if (!KeptChain.contains(Op))
+ return false;
+ Worklist.push_back(Op);
+ }
+ }
+ return true;
+ };
+
+ // Every lane feeds the result exactly once unless the op is idempotent.
+ auto EveryLaneOnce = [&](ArrayRef<unsigned> D) {
+ return !D.empty() && all_of(D, [&](unsigned C) {
+ return C >= 1 && (IsIdempotent || C == 1);
+ });
+ };
+
+ // Find the deepest intermediate where every lane feeds the result exactly
+ // once (idempotent ops tolerate duplicates) and that covers the whole chain.
Value *FullInter = nullptr;
for (Value *V : ChainPostorder) {
if (!isa<BinaryOperator>(V))
continue;
- SmallVector<unsigned, 16> D = Demand.lookup(V);
- if (!D.empty() && all_of(D, [&](unsigned C) {
- return C >= 1 && (IsIdempotent || C == 1);
- })) {
- FullInter = V;
- break;
- }
+ if (!EveryLaneOnce(Demand.lookup(V)) || !CoversChain(V))
+ continue;
+ FullInter = V;
+ break;
}
// Multi-source chains also take the intermediate, since one reduce replaces
diff --git a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
index 0cad902db0fa4..eb3cfd19ea348 100644
--- a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
+++ b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
@@ -415,3 +415,26 @@ define float @test_reduce_v4f32_fadd_fmf_intersect(<4 x float> %a0) {
%5 = extractelement <4 x float> %4, i64 0
ret float %5
}
+
+; Reconvergent chain (lane 0 = 2*(a[0]+a[1]+a[2])). An inner sub-sum is cleanly demanded but does not cover the full reduction so the fold must reduce the covering intermediate.
+define i32 @test_reduce_reconvergent_intermediate_v4i32(<4 x i32> %a) {
+; CHECK-LABEL: define i32 @test_reduce_reconvergent_intermediate_v4i32(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) {
+; CHECK-NEXT: [[SH1:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[SH2:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> poison, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: [[SH3:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[B1:%.*]] = add <2 x i32> [[SH1]], [[SH2]]
+; CHECK-NEXT: [[B2:%.*]] = add <2 x i32> [[SH3]], [[B1]]
+; CHECK-NEXT: [[E:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[B2]])
+; CHECK-NEXT: ret i32 [[E]]
+;
+ %sh1 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+ %sh2 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 1, i32 2>
+ %sh3 = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %b1 = add <2 x i32> %sh1, %sh2
+ %b2 = add <2 x i32> %sh3, %b1
+ %sh4 = shufflevector <2 x i32> %b2, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+ %c = add <2 x i32> %b2, %sh4
+ %e = extractelement <2 x i32> %c, i64 0
+ ret i32 %e
+}
More information about the llvm-commits
mailing list