[llvm] [VPlan] Introduce ComputeReductionResult VPInstruction opcode. (PR #70253)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 28 08:54:38 PST 2023
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/70253
>From 829bad89b8fe9ac4adcae12c8d4b914095a37247 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 16 Sep 2023 20:09:00 +0100
Subject: [PATCH 1/4] [VPlan] Introduce ComputeReductionResult VPInstruction
opcode.
This patch introduces a new ComputeReductionResult opcode to compute the
final reduction result in the middle block. The code from fixReduction
has been moved to ComputeReductionResult, after some earlier cleanup
changes to model parts of fixReduction explicitly elsewhere as needed.
The recipe may be broken down further in the future.
Note that the ComputeReductionResult at the moment also creates the phi
node to merge the reduction result from the trip count check and the
middle block, to be used as resume value for the scalar remainder loop.
Once we have a VPValue for the reduction result, this can also be
modeled explicitly and moved out of the recipe.
---
.../Vectorize/LoopVectorizationPlanner.h | 11 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 274 ++++++------------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 136 +++++++++
...-order-recurrence-sink-replicate-region.ll | 3 +-
.../LoopVectorize/vplan-printing.ll | 7 +-
7 files changed, 237 insertions(+), 198 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 577ce8000de27b..a4ead0523a2fb7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -347,10 +347,13 @@ class LoopVectorizationPlanner {
/// vectorization re-using plans for both the main and epilogue vector loops.
/// It should be removed once the re-use issue has been fixed.
/// \p ExpandedSCEVs is passed during execution of the plan for epilogue loop
- /// to re-use expansion results generated during main plan execution. Returns
- /// a mapping of SCEVs to their expanded IR values. Note that this is a
- /// temporary workaround needed due to the current epilogue handling.
- DenseMap<const SCEV *, Value *>
+ /// to re-use expansion results generated during main plan execution.
+ ///
+ /// Returns a mapping of SCEVs to their expanded IR values and a mapping for
+ /// the reduction resume values. Note that this is a temporary workaround
+ /// needed due to the current epilogue handling.
+ std::pair<DenseMap<const SCEV *, Value *>,
+ DenseMap<const RecurrenceDescriptor *, Value *>>
executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
InnerLoopVectorizer &LB, DominatorTree *DT,
bool IsEpilogueVectorization,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f82e161fb846d1..7b7bc826c4beab 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -584,10 +584,6 @@ class InnerLoopVectorizer {
/// able to vectorize with strict in-order reductions for the given RdxDesc.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc);
- // Returns the resume value (bc.merge.rdx) for a reduction as
- // generated by fixReduction.
- PHINode *getReductionResumeValue(const RecurrenceDescriptor &RdxDesc);
-
/// Create a new phi node for the induction variable \p OrigPhi to resume
/// iteration count in the scalar epilogue, from where the vectorized loop
/// left off. \p Step is the SCEV-expanded induction step to use. In cases
@@ -1166,14 +1162,6 @@ void InnerLoopVectorizer::collectPoisonGeneratingRecipes(
}
}
-PHINode *InnerLoopVectorizer::getReductionResumeValue(
- const RecurrenceDescriptor &RdxDesc) {
- auto It = ReductionResumeValues.find(&RdxDesc);
- assert(It != ReductionResumeValues.end() &&
- "Expected to find a resume value for the reduction.");
- return It->second;
-}
-
namespace llvm {
// Loop vectorization cost-model hints how the scalar epilogue loop should be
@@ -3513,11 +3501,6 @@ void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
VPBasicBlock *Header =
State.Plan->getVectorLoopRegion()->getEntryBasicBlock();
- for (VPRecipeBase &R : Header->phis()) {
- if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R))
- fixReduction(ReductionPhi, State);
- }
-
for (VPRecipeBase &R : Header->phis()) {
if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
fixFixedOrderRecurrence(FOR, State);
@@ -3643,169 +3626,6 @@ void InnerLoopVectorizer::fixFixedOrderRecurrence(
Phi->setName("scalar.recur");
}
-void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
- VPTransformState &State) {
- PHINode *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
- // Get it's reduction variable descriptor.
- assert(Legal->isReductionVariable(OrigPhi) &&
- "Unable to find the reduction variable");
- const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-
- RecurKind RK = RdxDesc.getRecurrenceKind();
- TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
- Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
- if (auto *I = dyn_cast<Instruction>(&*ReductionStartValue))
- State.setDebugLocFrom(I->getDebugLoc());
-
- VPValue *LoopExitInstDef = PhiR->getBackedgeValue();
-
- // Before each round, move the insertion point right between
- // the PHIs and the values we are going to write.
- // This allows us to write both PHINodes and the extractelement
- // instructions.
- Builder.SetInsertPoint(LoopMiddleBlock,
- LoopMiddleBlock->getFirstInsertionPt());
-
- State.setDebugLocFrom(LoopExitInst->getDebugLoc());
-
- Type *PhiTy = OrigPhi->getType();
- // If tail is folded by masking, the vector value to leave the loop should be
- // a Select choosing between the vectorized LoopExitInst and vectorized Phi,
- // instead of the former. For an inloop reduction the reduction will already
- // be predicated, and does not need to be handled here.
- if (Cost->foldTailByMasking() && !PhiR->isInLoop()) {
- VPValue *Def = nullptr;
- for (VPUser *U : LoopExitInstDef->users()) {
- auto *S = dyn_cast<VPInstruction>(U);
- if (S && S->getOpcode() == Instruction::Select) {
- Def = S;
- break;
- }
- }
- if (Def)
- LoopExitInstDef = Def;
- }
-
- VectorParts RdxParts(UF);
- for (unsigned Part = 0; Part < UF; ++Part)
- RdxParts[Part] = State.get(LoopExitInstDef, Part);
-
- // If the vector reduction can be performed in a smaller type, we truncate
- // then extend the loop exit value to enable InstCombine to evaluate the
- // entire expression in the smaller type.
- if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
- Builder.SetInsertPoint(LoopMiddleBlock,
- LoopMiddleBlock->getFirstInsertionPt());
- Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
- for (unsigned Part = 0; Part < UF; ++Part) {
- RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
- }
- }
-
- // Reduce all of the unrolled parts into a single vector.
- Value *ReducedPartRdx = RdxParts[0];
- unsigned Op = RecurrenceDescriptor::getOpcode(RK);
-
- // The middle block terminator has already been assigned a DebugLoc here (the
- // OrigLoop's single latch terminator). We want the whole middle block to
- // appear to execute on this line because: (a) it is all compiler generated,
- // (b) these instructions are always executed after evaluating the latch
- // conditional branch, and (c) other passes may add new predecessors which
- // terminate on this line. This is the easiest way to ensure we don't
- // accidentally cause an extra step back into the loop while debugging.
- State.setDebugLocFrom(LoopMiddleBlock->getTerminator()->getDebugLoc());
- if (PhiR->isOrdered())
- ReducedPartRdx = RdxParts[UF - 1];
- else {
- // Floating-point operations should have some FMF to enable the reduction.
- IRBuilderBase::FastMathFlagGuard FMFG(Builder);
- Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
- for (unsigned Part = 1; Part < UF; ++Part) {
- Value *RdxPart = RdxParts[Part];
- if (Op != Instruction::ICmp && Op != Instruction::FCmp)
- ReducedPartRdx = Builder.CreateBinOp(
- (Instruction::BinaryOps)Op, RdxPart, ReducedPartRdx, "bin.rdx");
- else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK))
- ReducedPartRdx = createAnyOfOp(Builder, ReductionStartValue, RK,
- ReducedPartRdx, RdxPart);
- else
- ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
- }
- }
-
- // Create the reduction after the loop. Note that inloop reductions create the
- // target reduction in the loop using a Reduction recipe.
- if (VF.isVector() && !PhiR->isInLoop()) {
- ReducedPartRdx =
- createTargetReduction(Builder, RdxDesc, ReducedPartRdx, OrigPhi);
- // If the reduction can be performed in a smaller type, we need to extend
- // the reduction to the wider type before we branch to the original loop.
- if (PhiTy != RdxDesc.getRecurrenceType())
- ReducedPartRdx = RdxDesc.isSigned()
- ? Builder.CreateSExt(ReducedPartRdx, PhiTy)
- : Builder.CreateZExt(ReducedPartRdx, PhiTy);
- }
-
- PHINode *ResumePhi =
- dyn_cast<PHINode>(PhiR->getStartValue()->getUnderlyingValue());
-
- // Create a phi node that merges control-flow from the backedge-taken check
- // block and the middle block.
- PHINode *BCBlockPhi = PHINode::Create(PhiTy, 2, "bc.merge.rdx",
- LoopScalarPreHeader->getTerminator());
-
- // If we are fixing reductions in the epilogue loop then we should already
- // have created a bc.merge.rdx Phi after the main vector body. Ensure that
- // we carry over the incoming values correctly.
- for (auto *Incoming : predecessors(LoopScalarPreHeader)) {
- if (Incoming == LoopMiddleBlock)
- BCBlockPhi->addIncoming(ReducedPartRdx, Incoming);
- else if (ResumePhi && llvm::is_contained(ResumePhi->blocks(), Incoming))
- BCBlockPhi->addIncoming(ResumePhi->getIncomingValueForBlock(Incoming),
- Incoming);
- else
- BCBlockPhi->addIncoming(ReductionStartValue, Incoming);
- }
-
- // Set the resume value for this reduction
- ReductionResumeValues.insert({&RdxDesc, BCBlockPhi});
-
- // If there were stores of the reduction value to a uniform memory address
- // inside the loop, create the final store here.
- if (StoreInst *SI = RdxDesc.IntermediateStore) {
- StoreInst *NewSI =
- Builder.CreateAlignedStore(ReducedPartRdx, SI->getPointerOperand(),
- SI->getAlign());
- propagateMetadata(NewSI, SI);
-
- // If the reduction value is used in other places,
- // then let the code below create PHI's for that.
- }
-
- // Now, we need to fix the users of the reduction variable
- // inside and outside of the scalar remainder loop.
-
- // We know that the loop is in LCSSA form. We need to update the PHI nodes
- // in the exit blocks. See comment on analogous loop in
- // fixFixedOrderRecurrence for a more complete explaination of the logic.
- if (!Cost->requiresScalarEpilogue(VF.isVector()))
- for (PHINode &LCSSAPhi : LoopExitBlock->phis())
- if (llvm::is_contained(LCSSAPhi.incoming_values(), LoopExitInst)) {
- LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock);
- State.Plan->removeLiveOut(&LCSSAPhi);
- }
-
- // Fix the scalar loop reduction variable with the incoming reduction sum
- // from the vector body and from the backedge value.
- int IncomingEdgeBlockIdx =
- OrigPhi->getBasicBlockIndex(OrigLoop->getLoopLatch());
- assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index");
- // Pick the other block.
- int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
- OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
- OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
-}
-
void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
// The basic block and loop containing the predicated instruction.
auto *PredBB = PredInst->getParent();
@@ -7585,7 +7405,9 @@ static void AddRuntimeUnrollDisableMetaData(Loop *L) {
}
}
-SCEV2ValueTy LoopVectorizationPlanner::executePlan(
+std::pair<DenseMap<const SCEV *, Value *>,
+ DenseMap<const RecurrenceDescriptor *, Value *>>
+LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueVectorization,
const DenseMap<const SCEV *, Value *> *ExpandedSCEVs) {
@@ -7614,6 +7436,7 @@ SCEV2ValueTy LoopVectorizationPlanner::executePlan(
State.Builder.SetInsertPoint(OrigLoop->getLoopPreheader()->getTerminator());
BestVPlan.getPreheader()->execute(&State);
}
+ State.CFG.VPBB2IRBB[BestVPlan.getPreheader()] = OrigLoop->getLoopPreheader();
if (!ILV.getTripCount())
ILV.setTripCount(State.get(BestVPlan.getTripCount(), {0, 0}));
else
@@ -7664,6 +7487,46 @@ SCEV2ValueTy LoopVectorizationPlanner::executePlan(
BestVPlan.execute(&State);
+ DenseMap<const RecurrenceDescriptor *, Value *> ReductionResumeValues;
+ BasicBlock *OrigScalarPH = OrigLoop->getLoopPreheader();
+ // Check if ResResult is a CompueReductionResult instruction, and if it is
+ // return the merge phi node for it.
+ auto GetMergePhiForReduction = [&State,
+ OrigScalarPH](VPInstruction *RedResult)
+ -> std::pair<const RecurrenceDescriptor *, PHINode *> {
+ if (!RedResult ||
+ RedResult->getOpcode() != VPInstruction::ComputeReductionResult)
+ return {nullptr, nullptr};
+
+ Value *FinalValue =
+ State.get(RedResult, VPIteration(State.UF - 1,
+ VPLane::getLastLaneForVF(State.VF)));
+ auto *PhiR = cast<VPReductionPHIRecipe>(RedResult->getOperand(0));
+ assert(FinalValue->hasOneUse() || PhiR->isInLoop() ||
+ PhiR->getRecurrenceDescriptor().IntermediateStore &&
+ "unexpected final value");
+
+ // Find the merge phi, which must be located in the preheader of the
+ // original scalar loop.
+ PHINode *MergePhi = nullptr;
+ for (Value *U : FinalValue->users()) {
+ auto *P = dyn_cast<PHINode>(U);
+ if (P && OrigScalarPH == P->getParent()) {
+ MergePhi = P;
+ break;
+ }
+ }
+ assert(MergePhi && "must have merge phi");
+ return {&PhiR->getRecurrenceDescriptor(), MergePhi};
+ };
+ for (VPRecipeBase &R : *cast<VPBasicBlock>(
+ BestVPlan.getVectorLoopRegion()->getSingleSuccessor())) {
+ const auto &[RedRdx, MergePhi] =
+ GetMergePhiForReduction(dyn_cast<VPInstruction>(&R));
+ if (!RedRdx)
+ continue;
+ ReductionResumeValues[RedRdx] = MergePhi;
+ }
// Keep all loop hints from the original loop on the vector loop (we'll
// replace the vectorizer-specific hints below).
MDNode *OrigLoopID = OrigLoop->getLoopID();
@@ -7697,7 +7560,7 @@ SCEV2ValueTy LoopVectorizationPlanner::executePlan(
ILV.printDebugTracesAtEnd();
- return State.ExpandedSCEVs;
+ return {State.ExpandedSCEVs, ReductionResumeValues};
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -8973,7 +8836,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
void LoopVectorizationPlanner::adjustRecipesForReductions(
VPBasicBlock *LatchVPBB, VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder,
ElementCount MinVF) {
- VPBasicBlock *Header = Plan->getVectorLoopRegion()->getEntryBasicBlock();
+ VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
+ VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
// Gather all VPReductionPHIRecipe and sort them so that Intermediate stores
// sank outside of the loop would keep the same order as they had in the
// original loop.
@@ -9016,7 +8880,25 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
SmallVector<VPReductionPHIRecipe *> InLoopReductionPhis;
for (VPRecipeBase &R : Header->phis()) {
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
+ if (!PhiR)
+ continue;
+
+ // TODO: At the moment ComputeReductionResult also creates the bc.merge.rdx
+ // phi nodes, hence it needs to be created unconditionally here, until the
+ // reduction resume value handling is also modeled in VPlan.
+ auto *FinalReductionResult =
+ new VPInstruction(VPInstruction::ComputeReductionResult,
+ {PhiR, PhiR->getBackedgeValue()});
+ cast<VPBasicBlock>(VectorLoopRegion->getSingleSuccessor())
+ ->appendRecipe(FinalReductionResult);
+ for (VPUser *U : to_vector(PhiR->getBackedgeValue()->users())) {
+ auto *LiveOut = dyn_cast<VPLiveOut>(U);
+ if (!LiveOut)
+ continue;
+ LiveOut->setOperand(0, FinalReductionResult);
+ }
+
+ if (!PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
continue;
InLoopReductionPhis.push_back(PhiR);
}
@@ -9051,6 +8933,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
for (VPRecipeBase *CurrentLink : Worklist.getArrayRef().drop_front()) {
VPValue *PreviousLinkV = PreviousLink->getVPSingleValue();
+ if (CurrentLink->getParent()->getParent() != VectorLoopRegion)
+ continue;
Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
// Index of the first operand which holds a non-mask vector operand.
@@ -9152,8 +9036,17 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
: new VPInstruction(Instruction::Select, {Cond, Red, PhiR});
Result->insertBefore(&*Builder.getInsertPoint());
Red->replaceUsesWithIf(
- Result->getVPSingleValue(),
- [](VPUser &U, unsigned) { return isa<VPLiveOut>(&U); });
+ Result->getVPSingleValue(), [VectorLoopRegion](VPUser &U, unsigned) {
+ auto *UR = cast<VPRecipeBase>(&U);
+ if (UR->getParent()->getParent() != VectorLoopRegion) {
+ assert(cast<VPInstruction>(UR)->getOpcode() ==
+ VPInstruction::ComputeReductionResult &&
+ "user outside of the vector region must be "
+ "ComputeReductionResult");
+ return true;
+ }
+ return false;
+ });
if (PreferPredicatedReductionSelect ||
TTI.preferPredicatedReductionSelect(
PhiR->getRecurrenceDescriptor().getOpcode(), PhiTy,
@@ -10185,8 +10078,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EPI, &LVL, &CM, BFI, PSI, Checks);
VPlan &BestMainPlan = LVP.getBestPlanFor(EPI.MainLoopVF);
- auto ExpandedSCEVs = LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF,
- BestMainPlan, MainILV, DT, true);
+ const auto &[ExpandedSCEVs, ReductionResumeValues] = LVP.executePlan(
+ EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, DT, true);
++LoopsVectorized;
// Second pass vectorizes the epilogue and adjusts the control flow
@@ -10227,8 +10120,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
Value *ResumeV = nullptr;
// TODO: Move setting of resume values to prepareToExecute.
if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
- ResumeV = MainILV.getReductionResumeValue(
- ReductionPhi->getRecurrenceDescriptor());
+ ResumeV = ReductionResumeValues
+ .find(&ReductionPhi->getRecurrenceDescriptor())
+ ->second;
} else {
// Create induction resume values for both widened pointer and
// integer/fp inductions and update the start value of the induction
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 263d9938d1f0f4..2a33dd400f2d98 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -456,6 +456,7 @@ void VPBasicBlock::execute(VPTransformState *State) {
// The Exit block of a loop is always set to be successor 0 of the Exiting
// block.
cast<BranchInst>(ExitingBB->getTerminator())->setSuccessor(0, NewBB);
+ State->Builder.SetInsertPoint(NewBB->getFirstNonPHI());
} else if (PrevVPBB && /* A */
!((SingleHPred = getSingleHierarchicalPredecessor()) &&
SingleHPred->getExitingBasicBlock() == PrevVPBB &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 94cb7688981361..548f0d832c9f95 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1061,7 +1061,8 @@ class VPInstruction : public VPRecipeWithIRFlags, public VPValue {
// Increment the canonical IV separately for each unrolled part.
CanonicalIVIncrementForPart,
BranchOnCount,
- BranchOnCond
+ BranchOnCond,
+ ComputeReductionResult,
};
private:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c23428e2ba34e8..e39cb8776d987a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -28,6 +28,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <cassert>
@@ -397,6 +398,138 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
return CondBr;
}
+ case VPInstruction::ComputeReductionResult: {
+ if (Part != 0)
+ return State.get(
+ this, VPIteration(State.UF - 1, VPLane::getLastLaneForVF(State.VF)));
+
+ auto *PhiR = dyn_cast<VPReductionPHIRecipe>(getOperand(0));
+ auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
+ // Get it's reduction variable descriptor.
+ const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+
+ BasicBlock *LoopMiddleBlock = State.CFG.VPBB2IRBB[getParent()];
+ Builder.SetInsertPoint(LoopMiddleBlock,
+ LoopMiddleBlock->getFirstInsertionPt());
+
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
+ Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
+ if (auto *I = dyn_cast<Instruction>(&*ReductionStartValue))
+ State.setDebugLocFrom(I->getDebugLoc());
+
+ VPValue *LoopExitInstDef = getOperand(1);
+ // This is the vector-clone of the value that leaves the loop.
+ // State.setDebugLocFrom(LoopExitInst->getDebugLoc());
+
+ Type *PhiTy = OrigPhi->getType();
+ VectorParts RdxParts(State.UF);
+ for (unsigned Part = 0; Part < State.UF; ++Part)
+ RdxParts[Part] = State.get(LoopExitInstDef, Part);
+
+ // If the vector reduction can be performed in a smaller type, we truncate
+ // then extend the loop exit value to enable InstCombine to evaluate the
+ // entire expression in the smaller type.
+ if (State.VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
+ Builder.SetInsertPoint(LoopMiddleBlock,
+ LoopMiddleBlock->getFirstInsertionPt());
+ Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
+ for (unsigned Part = 0; Part < State.UF; ++Part)
+ RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
+ }
+ // Reduce all of the unrolled parts into a single vector.
+ Value *ReducedPartRdx = RdxParts[0];
+ unsigned Op = RecurrenceDescriptor::getOpcode(RK);
+
+ // The middle block terminator has already been assigned a DebugLoc here
+ // (the OrigLoop's single latch terminator). We want the whole middle block
+ // to appear to execute on this line because: (a) it is all compiler
+ // generated, (b) these instructions are always executed after evaluating
+ // the latch conditional branch, and (c) other passes may add new
+ // predecessors which terminate on this line. This is the easiest way to
+ // ensure we don't accidentally cause an extra step back into the loop while
+ // debugging.
+ State.setDebugLocFrom(LoopMiddleBlock->getTerminator()->getDebugLoc());
+ if (PhiR->isOrdered()) {
+ ReducedPartRdx = RdxParts[State.UF - 1];
+ } else {
+ // Floating-point operations should have some FMF to enable the reduction.
+ IRBuilderBase::FastMathFlagGuard FMFG(Builder);
+ Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
+ for (unsigned Part = 1; Part < State.UF; ++Part) {
+ Value *RdxPart = RdxParts[Part];
+ if (Op != Instruction::ICmp && Op != Instruction::FCmp)
+ ReducedPartRdx = Builder.CreateBinOp(
+ (Instruction::BinaryOps)Op, RdxPart, ReducedPartRdx, "bin.rdx");
+ else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK))
+ ReducedPartRdx = createAnyOfOp(Builder, ReductionStartValue, RK,
+ ReducedPartRdx, RdxPart);
+ else
+ ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
+ }
+ }
+
+ // Create the reduction after the loop. Note that inloop reductions create
+ // the target reduction in the loop using a Reduction recipe.
+ if (State.VF.isVector() && !PhiR->isInLoop()) {
+ ReducedPartRdx =
+ createTargetReduction(Builder, RdxDesc, ReducedPartRdx, OrigPhi);
+ // If the reduction can be performed in a smaller type, we need to extend
+ // the reduction to the wider type before we branch to the original loop.
+ if (PhiTy != RdxDesc.getRecurrenceType())
+ ReducedPartRdx = RdxDesc.isSigned()
+ ? Builder.CreateSExt(ReducedPartRdx, PhiTy)
+ : Builder.CreateZExt(ReducedPartRdx, PhiTy);
+ }
+
+ auto *ResumePhi =
+ dyn_cast<PHINode>(PhiR->getStartValue()->getUnderlyingValue());
+
+ auto *OrigLoop = State.LI->getLoopFor(OrigPhi->getParent());
+ // TODO: bc.merge.rdx should not be created here, instead it should be
+ // modeled in VPlan.
+ BasicBlock *LoopScalarPreHeader = OrigLoop->getLoopPreheader();
+ // Create a phi node that merges control-flow from the backedge-taken check
+ // block and the middle block.
+ auto *BCBlockPhi = PHINode::Create(PhiTy, 2, "bc.merge.rdx",
+ LoopScalarPreHeader->getTerminator());
+
+ // If we are fixing reductions in the epilogue loop then we should already
+ // have created a bc.merge.rdx Phi after the main vector body. Ensure that
+ // we carry over the incoming values correctly.
+ for (auto *Incoming : predecessors(LoopScalarPreHeader)) {
+ if (Incoming == LoopMiddleBlock)
+ BCBlockPhi->addIncoming(ReducedPartRdx, Incoming);
+ else if (ResumePhi && is_contained(ResumePhi->blocks(), Incoming))
+ BCBlockPhi->addIncoming(ResumePhi->getIncomingValueForBlock(Incoming),
+ Incoming);
+ else
+ BCBlockPhi->addIncoming(ReductionStartValue, Incoming);
+ }
+
+ // If there were stores of the reduction value to a uniform memory address
+ // inside the loop, create the final store here.
+ if (StoreInst *SI = RdxDesc.IntermediateStore) {
+ auto *NewSI = Builder.CreateAlignedStore(
+ ReducedPartRdx, SI->getPointerOperand(), SI->getAlign());
+ propagateMetadata(NewSI, SI);
+ }
+
+ // TODO: This fixup should instead be modeled in VPlan.
+ // Fix the scalar loop reduction variable with the incoming reduction sum
+ // from the vector body and from the backedge value.
+ int IncomingEdgeBlockIdx =
+ OrigPhi->getBasicBlockIndex(OrigLoop->getLoopLatch());
+ assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index");
+ // Pick the other block.
+ int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
+ OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
+ OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
+
+ State.set(this, ReducedPartRdx,
+ VPIteration(State.UF - 1, VPLane::getLastLaneForVF(State.VF)));
+ return ReducedPartRdx;
+ }
default:
llvm_unreachable("Unsupported opcode for instruction");
}
@@ -473,6 +606,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
case VPInstruction::BranchOnCount:
O << "branch-on-count";
break;
+ case VPInstruction::ComputeReductionResult:
+ O << "compute-reduction-result";
+ break;
default:
O << Instruction::getOpcodeName(getOpcode());
}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 060d28f030f72e..d022fe3ba3d150 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -215,9 +215,10 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: Live-out i32 %res = vp<[[SEL]]>
+; CHECK-NEXT: Live-out i32 %res = vp<[[RED_RES]]>
; CHECK-NEXT: }
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 13f444eaaf76a5..f758d2bed69021 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -132,9 +132,10 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%red>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: Live-out float %red.next.lcssa = ir<%red.next>
+; CHECK-NEXT: Live-out float %red.next.lcssa = vp<[[RED_RES]]>
; CHECK-NEXT: }
;
entry:
@@ -179,6 +180,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result ir<%red>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
@@ -376,9 +378,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%sum.07>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: Live-out float %muladd.lcssa = ir<%muladd>
+; CHECK-NEXT: Live-out float %muladd.lcssa = vp<[[RED_RES]]>
; CHECK-NEXT:}
entry:
>From d6be41c25056f8f0474fbf4b3791fe225fd3582e Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 4 Dec 2023 16:11:03 +0000
Subject: [PATCH 2/4] fixup! use find_if
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7b7bc826c4beab..2724a54adc9cb2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7508,16 +7508,12 @@ LoopVectorizationPlanner::executePlan(
// Find the merge phi, which must be located in the preheader of the
// original scalar loop.
- PHINode *MergePhi = nullptr;
- for (Value *U : FinalValue->users()) {
+ auto It = find_if(FinalValue->users(), [OrigScalarPH](Value *U) {
auto *P = dyn_cast<PHINode>(U);
- if (P && OrigScalarPH == P->getParent()) {
- MergePhi = P;
- break;
- }
- }
- assert(MergePhi && "must have merge phi");
- return {&PhiR->getRecurrenceDescriptor(), MergePhi};
+ return P && OrigScalarPH == P->getParent();
+ });
+ assert(It != FinalValue->users().end() && "must have merge phi");
+ return {&PhiR->getRecurrenceDescriptor(), cast<PHINode>(*It)};
};
for (VPRecipeBase &R : *cast<VPBasicBlock>(
BestVPlan.getVectorLoopRegion()->getSingleSuccessor())) {
>From 95f532e5d965f5607b76778aeb88975a3b7e8e08 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 20 Dec 2023 20:02:06 +0000
Subject: [PATCH 3/4] !fixup address @ayalz latest comments, thanks!
---
.../Transforms/Vectorize/LoopVectorize.cpp | 136 +++++++-----------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +-
2 files changed, 56 insertions(+), 88 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2724a54adc9cb2..959c3e0650e48b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -622,9 +622,6 @@ class InnerLoopVectorizer {
BasicBlock *MiddleBlock, BasicBlock *VectorHeader,
VPlan &Plan, VPTransformState &State);
- /// Handle all cross-iteration phis in the header.
- void fixCrossIterationPHIs(VPTransformState &State);
-
/// Create the exit value of first order recurrences in the middle block and
/// update their users.
void fixFixedOrderRecurrence(VPFirstOrderRecurrencePHIRecipe *PhiR,
@@ -3422,8 +3419,15 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
// At this point every instruction in the original loop is widened to a
// vector form. Now we need to fix the recurrences in the loop. These PHI
// nodes are currently empty because we did not want to introduce cycles.
- // This is the second stage of vectorizing recurrences.
- fixCrossIterationPHIs(State);
+ // This is the second stage of vectorizing recurrences. Note that fixing
+ // reduction phis are already modeled in VPlan.
+ // TODO: Also model fixing fixed-order recurrence phis in VPlan.
+ VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion();
+ VPBasicBlock *Header = VectorRegion->getEntryBasicBlock();
+ for (VPRecipeBase &R : Header->phis()) {
+ if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
+ fixFixedOrderRecurrence(FOR, State);
+ }
// Forget the original basic block.
PSE.getSE()->forgetLoop(OrigLoop);
@@ -3438,7 +3442,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
for (PHINode &PN : Exit->phis())
PSE.getSE()->forgetLcssaPhiWithNewPredecessor(OrigLoop, &PN);
- VPBasicBlock *LatchVPBB = Plan.getVectorLoopRegion()->getExitingBasicBlock();
+ VPBasicBlock *LatchVPBB = VectorRegion->getExitingBasicBlock();
Loop *VectorLoop = LI->getLoopFor(State.CFG.VPBB2IRBB[LatchVPBB]);
if (Cost->requiresScalarEpilogue(VF.isVector())) {
// No edge from the middle block to the unique exit block has been inserted
@@ -3491,22 +3495,6 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
VF.getKnownMinValue() * UF);
}
-void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
- // In order to support recurrences we need to be able to vectorize Phi nodes.
- // Phi nodes have cycles, so we need to vectorize them in two stages. This is
- // stage #2: We now need to fix the recurrences by adding incoming edges to
- // the currently empty PHI nodes. At this point every instruction in the
- // original loop is widened to a vector form so we can use them to construct
- // the incoming edges.
- VPBasicBlock *Header =
- State.Plan->getVectorLoopRegion()->getEntryBasicBlock();
-
- for (VPRecipeBase &R : Header->phis()) {
- if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
- fixFixedOrderRecurrence(FOR, State);
- }
-}
-
void InnerLoopVectorizer::fixFixedOrderRecurrence(
VPFirstOrderRecurrencePHIRecipe *PhiR, VPTransformState &State) {
// This is the second phase of vectorizing first-order recurrences. An
@@ -7405,6 +7393,33 @@ static void AddRuntimeUnrollDisableMetaData(Loop *L) {
}
}
+// Check if RedResult is a CompueReductionResult instruction, and if it is
+// add it to \p ReductionResumeValues with the merge phi node for it.
+static void getMergePhiForReduction(
+ VPInstruction *RedResult,
+ DenseMap<const RecurrenceDescriptor *, Value *> &ReductionResumeValues,
+ VPTransformState &State, BasicBlock *OrigScalarPH) {
+ if (!RedResult ||
+ RedResult->getOpcode() != VPInstruction::ComputeReductionResult)
+ return;
+
+ Value *FinalValue = State.get(
+ RedResult, VPIteration(State.UF - 1, VPLane::getLastLaneForVF(State.VF)));
+ auto *PhiR = cast<VPReductionPHIRecipe>(RedResult->getOperand(0));
+ const RecurrenceDescriptor &RedRdx = PhiR->getRecurrenceDescriptor();
+ assert(FinalValue->hasOneUse() || PhiR->isInLoop() ||
+ RedRdx.IntermediateStore && "unexpected final value");
+
+ // Find the merge phi, which must be located in the preheader of the
+ // original scalar loop.
+ auto It = find_if(FinalValue->users(), [OrigScalarPH](Value *U) {
+ auto *P = dyn_cast<PHINode>(U);
+ return P && OrigScalarPH == P->getParent();
+ });
+ assert(It != FinalValue->users().end() && "must have merge phi");
+ ReductionResumeValues[&RedRdx] = cast<PHINode>(*It);
+}
+
std::pair<DenseMap<const SCEV *, Value *>,
DenseMap<const RecurrenceDescriptor *, Value *>>
LoopVectorizationPlanner::executePlan(
@@ -7436,7 +7451,6 @@ LoopVectorizationPlanner::executePlan(
State.Builder.SetInsertPoint(OrigLoop->getLoopPreheader()->getTerminator());
BestVPlan.getPreheader()->execute(&State);
}
- State.CFG.VPBB2IRBB[BestVPlan.getPreheader()] = OrigLoop->getLoopPreheader();
if (!ILV.getTripCount())
ILV.setTripCount(State.get(BestVPlan.getTripCount(), {0, 0}));
else
@@ -7487,42 +7501,17 @@ LoopVectorizationPlanner::executePlan(
BestVPlan.execute(&State);
+ // 2.5 Collect reduction resume values.
DenseMap<const RecurrenceDescriptor *, Value *> ReductionResumeValues;
BasicBlock *OrigScalarPH = OrigLoop->getLoopPreheader();
- // Check if ResResult is a CompueReductionResult instruction, and if it is
- // return the merge phi node for it.
- auto GetMergePhiForReduction = [&State,
- OrigScalarPH](VPInstruction *RedResult)
- -> std::pair<const RecurrenceDescriptor *, PHINode *> {
- if (!RedResult ||
- RedResult->getOpcode() != VPInstruction::ComputeReductionResult)
- return {nullptr, nullptr};
-
- Value *FinalValue =
- State.get(RedResult, VPIteration(State.UF - 1,
- VPLane::getLastLaneForVF(State.VF)));
- auto *PhiR = cast<VPReductionPHIRecipe>(RedResult->getOperand(0));
- assert(FinalValue->hasOneUse() || PhiR->isInLoop() ||
- PhiR->getRecurrenceDescriptor().IntermediateStore &&
- "unexpected final value");
-
- // Find the merge phi, which must be located in the preheader of the
- // original scalar loop.
- auto It = find_if(FinalValue->users(), [OrigScalarPH](Value *U) {
- auto *P = dyn_cast<PHINode>(U);
- return P && OrigScalarPH == P->getParent();
- });
- assert(It != FinalValue->users().end() && "must have merge phi");
- return {&PhiR->getRecurrenceDescriptor(), cast<PHINode>(*It)};
- };
- for (VPRecipeBase &R : *cast<VPBasicBlock>(
- BestVPlan.getVectorLoopRegion()->getSingleSuccessor())) {
- const auto &[RedRdx, MergePhi] =
- GetMergePhiForReduction(dyn_cast<VPInstruction>(&R));
- if (!RedRdx)
- continue;
- ReductionResumeValues[RedRdx] = MergePhi;
+ auto *ExitBlock =
+ cast<VPBasicBlock>(BestVPlan.getVectorLoopRegion()->getSingleSuccessor());
+ for (VPRecipeBase &R : *ExitBlock) {
+ getMergePhiForReduction(dyn_cast<VPInstruction>(&R), ReductionResumeValues,
+ State, OrigScalarPH);
}
+
+ // 2.6. Maintain Loop Hints
// Keep all loop hints from the original loop on the vector loop (we'll
// replace the vectorizer-specific hints below).
MDNode *OrigLoopID = OrigLoop->getLoopID();
@@ -8873,7 +8862,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
for (VPRecipeBase *R : ReductionPHIList)
R->moveBefore(*Header, Header->getFirstNonPhi());
- SmallVector<VPReductionPHIRecipe *> InLoopReductionPhis;
for (VPRecipeBase &R : Header->phis()) {
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
if (!PhiR)
@@ -8887,19 +8875,13 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
{PhiR, PhiR->getBackedgeValue()});
cast<VPBasicBlock>(VectorLoopRegion->getSingleSuccessor())
->appendRecipe(FinalReductionResult);
- for (VPUser *U : to_vector(PhiR->getBackedgeValue()->users())) {
- auto *LiveOut = dyn_cast<VPLiveOut>(U);
- if (!LiveOut)
- continue;
- LiveOut->setOperand(0, FinalReductionResult);
- }
+ PhiR->getBackedgeValue()->replaceUsesWithIf(
+ FinalReductionResult,
+ [](VPUser &User, unsigned) { return isa<VPLiveOut>(&User); });
if (!PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
continue;
- InLoopReductionPhis.push_back(PhiR);
- }
- for (VPReductionPHIRecipe *PhiR : InLoopReductionPhis) {
const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
RecurKind Kind = RdxDesc.getRecurrenceKind();
assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
@@ -8912,7 +8894,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
VPRecipeBase *Cur = Worklist[I];
for (VPUser *U : Cur->getVPSingleValue()->users()) {
auto *UserRecipe = dyn_cast<VPRecipeBase>(U);
- if (!UserRecipe)
+ if (!UserRecipe || UserRecipe == FinalReductionResult)
continue;
assert(UserRecipe->getNumDefinedValues() == 1 &&
"recipes must define exactly one result value");
@@ -8929,8 +8911,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
for (VPRecipeBase *CurrentLink : Worklist.getArrayRef().drop_front()) {
VPValue *PreviousLinkV = PreviousLink->getVPSingleValue();
- if (CurrentLink->getParent()->getParent() != VectorLoopRegion)
- continue;
Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
// Index of the first operand which holds a non-mask vector operand.
@@ -9031,18 +9011,12 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
? new VPInstruction(Instruction::Select, {Cond, Red, PhiR}, FMFs)
: new VPInstruction(Instruction::Select, {Cond, Red, PhiR});
Result->insertBefore(&*Builder.getInsertPoint());
- Red->replaceUsesWithIf(
- Result->getVPSingleValue(), [VectorLoopRegion](VPUser &U, unsigned) {
- auto *UR = cast<VPRecipeBase>(&U);
- if (UR->getParent()->getParent() != VectorLoopRegion) {
- assert(cast<VPInstruction>(UR)->getOpcode() ==
- VPInstruction::ComputeReductionResult &&
- "user outside of the vector region must be "
- "ComputeReductionResult");
- return true;
- }
- return false;
- });
+ Red->replaceUsesWithIf(Result->getVPSingleValue(),
+ [](VPUser &U, unsigned) {
+ return isa<VPInstruction>(&U) &&
+ cast<VPInstruction>(&U)->getOpcode() ==
+ VPInstruction::ComputeReductionResult;
+ });
if (PreferPredicatedReductionSelect ||
TTI.preferPredicatedReductionSelect(
PhiR->getRecurrenceDescriptor().getOpcode(), PhiTy,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e39cb8776d987a..102f9bfeb113e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -408,10 +408,6 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
// Get it's reduction variable descriptor.
const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
- BasicBlock *LoopMiddleBlock = State.CFG.VPBB2IRBB[getParent()];
- Builder.SetInsertPoint(LoopMiddleBlock,
- LoopMiddleBlock->getFirstInsertionPt());
-
RecurKind RK = RdxDesc.getRecurrenceKind();
TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
@@ -419,9 +415,6 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
State.setDebugLocFrom(I->getDebugLoc());
VPValue *LoopExitInstDef = getOperand(1);
- // This is the vector-clone of the value that leaves the loop.
- // State.setDebugLocFrom(LoopExitInst->getDebugLoc());
-
Type *PhiTy = OrigPhi->getType();
VectorParts RdxParts(State.UF);
for (unsigned Part = 0; Part < State.UF; ++Part)
@@ -430,6 +423,7 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
// If the vector reduction can be performed in a smaller type, we truncate
// then extend the loop exit value to enable InstCombine to evaluate the
// entire expression in the smaller type.
+ BasicBlock *LoopMiddleBlock = State.CFG.VPBB2IRBB[getParent()];
if (State.VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
Builder.SetInsertPoint(LoopMiddleBlock,
LoopMiddleBlock->getFirstInsertionPt());
>From a0e66a4cafd0d5c9af2db51c27f74ad569bbfc14 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 28 Dec 2023 16:53:31 +0000
Subject: [PATCH 4/4] !fixup Address latest comments, thanks!
---
.../Transforms/Vectorize/LoopVectorize.cpp | 10 +++++-----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 2 ++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 20 +++++++++----------
.../AArch64/scalable-strict-fadd.ll | 2 +-
.../LoopVectorize/AArch64/strict-fadd.ll | 2 +-
.../AArch64/sve-interleaved-accesses.ll | 2 +-
.../ARM/tail-fold-multiple-icmps.ll | 2 +-
.../epilog-vectorization-reductions.ll | 4 ++--
.../LoopVectorize/interleaved-accesses.ll | 2 +-
.../reduction-with-invariant-store.ll | 16 +++++++--------
11 files changed, 32 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 959c3e0650e48b..d929f7f1ccddac 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7395,7 +7395,7 @@ static void AddRuntimeUnrollDisableMetaData(Loop *L) {
// Check if RedResult is a CompueReductionResult instruction, and if it is
// add it to \p ReductionResumeValues with the merge phi node for it.
-static void getMergePhiForReduction(
+static void collectMergePhiForReduction(
VPInstruction *RedResult,
DenseMap<const RecurrenceDescriptor *, Value *> &ReductionResumeValues,
VPTransformState &State, BasicBlock *OrigScalarPH) {
@@ -7403,8 +7403,8 @@ static void getMergePhiForReduction(
RedResult->getOpcode() != VPInstruction::ComputeReductionResult)
return;
- Value *FinalValue = State.get(
- RedResult, VPIteration(State.UF - 1, VPLane::getLastLaneForVF(State.VF)));
+ Value *FinalValue =
+ State.get(RedResult, VPIteration(State.UF - 1, VPLane::getFirstLane()));
auto *PhiR = cast<VPReductionPHIRecipe>(RedResult->getOperand(0));
const RecurrenceDescriptor &RedRdx = PhiR->getRecurrenceDescriptor();
assert(FinalValue->hasOneUse() || PhiR->isInLoop() ||
@@ -7507,8 +7507,8 @@ LoopVectorizationPlanner::executePlan(
auto *ExitBlock =
cast<VPBasicBlock>(BestVPlan.getVectorLoopRegion()->getSingleSuccessor());
for (VPRecipeBase &R : *ExitBlock) {
- getMergePhiForReduction(dyn_cast<VPInstruction>(&R), ReductionResumeValues,
- State, OrigScalarPH);
+ collectMergePhiForReduction(dyn_cast<VPInstruction>(&R),
+ ReductionResumeValues, State, OrigScalarPH);
}
// 2.6. Maintain Loop Hints
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2a33dd400f2d98..48d05ecdef9065 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -446,6 +446,7 @@ void VPBasicBlock::execute(VPTransformState *State) {
// ExitBB can be re-used for the exit block of the Plan.
NewBB = State->CFG.ExitBB;
State->CFG.PrevBB = NewBB;
+ State->Builder.SetInsertPoint(NewBB->getFirstNonPHI());
// Update the branch instruction in the predecessor to branch to ExitBB.
VPBlockBase *PredVPB = getSingleHierarchicalPredecessor();
@@ -456,7 +457,6 @@ void VPBasicBlock::execute(VPTransformState *State) {
// The Exit block of a loop is always set to be successor 0 of the Exiting
// block.
cast<BranchInst>(ExitingBB->getTerminator())->setSuccessor(0, NewBB);
- State->Builder.SetInsertPoint(NewBB->getFirstNonPHI());
} else if (PrevVPBB && /* A */
!((SingleHPred = getSingleHierarchicalPredecessor()) &&
SingleHPred->getExitingBasicBlock() == PrevVPBB &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 548f0d832c9f95..fe63735e71ca2c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3103,6 +3103,8 @@ inline bool isUniformAfterVectorization(VPValue *VPV) {
return Rep->isUniform();
if (auto *GEP = dyn_cast<VPWidenGEPRecipe>(Def))
return all_of(GEP->operands(), isUniformAfterVectorization);
+ if (auto *VPI = dyn_cast<VPInstruction>(Def))
+ return VPI->getOpcode() == VPInstruction::ComputeReductionResult;
return false;
}
} // end namespace vputils
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 102f9bfeb113e3..a84f088c6323e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -400,33 +400,32 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
}
case VPInstruction::ComputeReductionResult: {
if (Part != 0)
- return State.get(
- this, VPIteration(State.UF - 1, VPLane::getLastLaneForVF(State.VF)));
+ return State.get(this, 0);
+ // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
+ // and will be removed by breaking up the recipe further.
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(getOperand(0));
auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
- // Get it's reduction variable descriptor.
+ // Get its reduction variable descriptor.
const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
RecurKind RK = RdxDesc.getRecurrenceKind();
TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
- Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
if (auto *I = dyn_cast<Instruction>(&*ReductionStartValue))
State.setDebugLocFrom(I->getDebugLoc());
- VPValue *LoopExitInstDef = getOperand(1);
+ VPValue *LoopExitingDef = getOperand(1);
Type *PhiTy = OrigPhi->getType();
VectorParts RdxParts(State.UF);
for (unsigned Part = 0; Part < State.UF; ++Part)
- RdxParts[Part] = State.get(LoopExitInstDef, Part);
+ RdxParts[Part] = State.get(LoopExitingDef, Part);
// If the vector reduction can be performed in a smaller type, we truncate
// then extend the loop exit value to enable InstCombine to evaluate the
// entire expression in the smaller type.
+ // TODO: Handle this in truncateToMinBW.
BasicBlock *LoopMiddleBlock = State.CFG.VPBB2IRBB[getParent()];
if (State.VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
- Builder.SetInsertPoint(LoopMiddleBlock,
- LoopMiddleBlock->getFirstInsertionPt());
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
for (unsigned Part = 0; Part < State.UF; ++Part)
RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
@@ -479,7 +478,7 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
auto *ResumePhi =
dyn_cast<PHINode>(PhiR->getStartValue()->getUnderlyingValue());
- auto *OrigLoop = State.LI->getLoopFor(OrigPhi->getParent());
+ const Loop *OrigLoop = State.LI->getLoopFor(OrigPhi->getParent());
// TODO: bc.merge.rdx should not be created here, instead it should be
// modeled in VPlan.
BasicBlock *LoopScalarPreHeader = OrigLoop->getLoopPreheader();
@@ -518,10 +517,9 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
// Pick the other block.
int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
+ Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
- State.set(this, ReducedPartRdx,
- VPIteration(State.UF - 1, VPLane::getLastLaneForVF(State.VF)));
return ReducedPartRdx;
}
default:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index cdc50c57b947fb..9f389188141940 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -612,8 +612,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
; CHECK-UNORDERED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-UNORDERED-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-UNORDERED: middle.block:
-; CHECK-UNORDERED-NEXT: [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP14]])
; CHECK-UNORDERED-NEXT: [[TMP20:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP15]])
+; CHECK-UNORDERED-NEXT: [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP14]])
; CHECK-UNORDERED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; CHECK-UNORDERED-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK-UNORDERED: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll
index 33b5273217cdf7..7c1247e9ebc8fa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll
@@ -285,8 +285,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <4 x float> %[[STRIDED2:.*]], %[[VEC_PHI2]]
; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd
; CHECK-UNORDERED: middle.block
-; CHECK-UNORDERED: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD1]])
; CHECK-UNORDERED: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD2]])
+; CHECK-UNORDERED: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD1]])
; CHECK-UNORDERED: for.body
; CHECK-UNORDERED: %[[LOAD1:.*]] = load float, ptr
; CHECK-UNORDERED: %[[FADD1:.*]] = fadd float %[[LOAD1]], {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index f600a0d5877d33..fa8e35f4697e91 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -821,8 +821,8 @@ define void @int_float_struct(%struct.IntFloat* nocapture readonly %p) #0 {
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP4]])
; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP5]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP4]])
; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll
index 038725db279bb8..f58d864e1e1477 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll
@@ -26,8 +26,8 @@ define arm_aapcs_vfpcc i32 @minmaxval4(ptr nocapture readonly %x, ptr nocapture
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index 03903d80cfd6ec..4df5332a47d4cc 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -313,8 +313,8 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) {
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP3]])
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP3]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
@@ -344,8 +344,8 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) {
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC5]]
; CHECK-NEXT: br i1 [[TMP15]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: vec.epilog.middle.block:
-; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP13]])
; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP14]])
+; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP13]])
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index 187eefbe9b5956..3b9edb11c5da2d 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -837,8 +837,8 @@ define void @int_float_struct(ptr nocapture readonly %A) #0 {
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll b/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll
index 2a2d55fb75c53c..5584aa969367ac 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll
@@ -562,10 +562,10 @@ exit: ; preds = %for.body
define void @reduc_add_mul_store_same_ptr(ptr %dst, ptr readonly %src) {
; CHECK-LABEL: define void @reduc_add_mul_store_same_ptr
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1:%.*]])
-; CHECK-NEXT: store i32 [[TMP2]], ptr %dst, align 4
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP3:%.*]])
; CHECK-NEXT: store i32 [[TMP4]], ptr %dst, align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1:%.*]])
+; CHECK-NEXT: store i32 [[TMP2]], ptr %dst, align 4
;
entry:
br label %for.body
@@ -591,10 +591,10 @@ exit:
define void @reduc_mul_add_store_same_ptr(ptr %dst, ptr readonly %src) {
; CHECK-LABEL: define void @reduc_mul_add_store_same_ptr
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1:%.*]])
-; CHECK-NEXT: store i32 [[TMP2]], ptr %dst, align 4
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3:%.*]])
; CHECK-NEXT: store i32 [[TMP4]], ptr %dst, align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1:%.*]])
+; CHECK-NEXT: store i32 [[TMP2]], ptr %dst, align 4
;
entry:
br label %for.body
@@ -621,10 +621,10 @@ exit:
define void @reduc_add_mul_store_different_ptr(ptr %dst1, ptr %dst2, ptr readonly %src) {
; CHECK-LABEL: define void @reduc_add_mul_store_different_ptr
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1:%.*]])
-; CHECK-NEXT: store i32 [[TMP2]], ptr %dst1, align 4
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP3:%.*]])
; CHECK-NEXT: store i32 [[TMP4]], ptr %dst2, align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1:%.*]])
+; CHECK-NEXT: store i32 [[TMP2]], ptr %dst1, align 4
;
entry:
br label %for.body
@@ -650,10 +650,10 @@ exit:
define void @reduc_mul_add_store_different_ptr(ptr %dst1, ptr %dst2, ptr readonly %src) {
; CHECK-LABEL: define void @reduc_mul_add_store_different_ptr
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1:%.*]])
-; CHECK-NEXT: store i32 [[TMP2]], ptr %dst1, align 4
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3:%.*]])
; CHECK-NEXT: store i32 [[TMP4]], ptr %dst2, align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1:%.*]])
+; CHECK-NEXT: store i32 [[TMP2]], ptr %dst1, align 4
;
entry:
br label %for.body
More information about the llvm-commits
mailing list