[PATCH] D100113: [LV] Move reduction PHI node fixup to VPlan::execute (NFC).
Florian Hahn via Phabricator via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 31 12:31:03 PDT 2021
fhahn updated this revision to Diff 363313.
fhahn added a comment.
Rebased after recent changes, should be much simpler now :)
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D100113/new/
https://reviews.llvm.org/D100113
Files:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
Index: llvm/lib/Transforms/Vectorize/VPlan.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -867,6 +867,21 @@
if (!EnableVPlanNativePath)
updateDominatorTree(State->DT, VectorPreHeaderBB, VectorLatchBB,
L->getExitBlock());
+
+ // Fixup reduction PHI nodes in the vectorized loop header.
+ for (VPRecipeBase &R : Header->phis()) {
+ auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
+ if (!PhiR)
+ continue;
+
+ unsigned LastPartForNewPhi = PhiR->isOrdered() ? 1 : State->UF;
+ for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
+ Value *VecRdxPhi = State->get(PhiR->getVPSingleValue(), Part);
+ Value *Val = State->get(PhiR->getBackedgeValue(),
+ PhiR->isOrdered() ? State->UF - 1 : Part);
+ cast<PHINode>(VecRdxPhi)->addIncoming(Val, VectorLatchBB);
+ }
+ }
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -594,8 +594,7 @@
/// update their users.
void fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR, VPTransformState &State);
- /// Fix a reduction cross-iteration phi. This is the second phase of
- /// vectorizing this phi node.
+ /// Create code for the loop exit value of the reduction.
void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State);
/// Clear NSW/NUW flags from reduction instructions if necessary.
@@ -4303,22 +4302,6 @@
// Wrap flags are in general invalid after vectorization, clear them.
clearReductionWrapFlags(RdxDesc, State);
- // Fix the vector-loop phi.
-
- // Reductions do not have to start at zero. They can start with
- // any loop invariant values.
- BasicBlock *VectorLoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
-
- unsigned LastPartForNewPhi = PhiR->isOrdered() ? 1 : UF;
- for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
- Value *VecRdxPhi = State.get(PhiR->getVPSingleValue(), Part);
- Value *Val = State.get(PhiR->getBackedgeValue(), Part);
- if (PhiR->isOrdered())
- Val = State.get(PhiR->getBackedgeValue(), UF - 1);
-
- cast<PHINode>(VecRdxPhi)->addIncoming(Val, VectorLoopLatch);
- }
-
// Before each round, move the insertion point right between
// the PHIs and the values we are going to write.
// This allows us to write both PHINodes and the extractelement
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D100113.363313.patch
Type: text/x-patch
Size: 2661 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20210731/6654cdc4/attachment.bin>
More information about the llvm-commits
mailing list