================
@@ -7405,124 +7391,27 @@ BasicBlock *EpilogueVectorizerMainLoop::emitIterationCountCheck(
// EpilogueVectorizerEpilogueLoop
//===--------------------------------------------------------------------===//
-/// This function is partially responsible for generating the control flow
-/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
+/// This function creates a new scalar preheader, using the previous one as
+/// entry block to the epilogue VPlan. The minimum iteration check is already
+/// created in VPlan.
BasicBlock *EpilogueVectorizerEpilogueLoop::createVectorizedLoopSkeleton() {
+ BasicBlock *OriginalScalarPH = OrigLoop->getLoopPreheader();
BasicBlock *ScalarPH = createScalarPreheader("vec.epilog.");
- BasicBlock *VectorPH = ScalarPH->getSinglePredecessor();
- // Now, compare the remaining count and if there aren't enough iterations to
- // execute the vectorized epilogue skip to the scalar part.
- VectorPH->setName("vec.epilog.ph");
- BasicBlock *VecEpilogueIterationCountCheck =
- SplitBlock(VectorPH, VectorPH->begin(), DT, LI, nullptr,
- "vec.epilog.iter.check", true);
- VectorPHVPBB = replaceVPBBWithIRVPBB(VectorPHVPBB, VectorPH);
-
- emitMinimumVectorEpilogueIterCountCheck(VectorPH, ScalarPH,
- VecEpilogueIterationCountCheck);
- AdditionalBypassBlock = VecEpilogueIterationCountCheck;
-
- // Adjust the control flow taking the state info from the main loop
- // vectorization into account.
- assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&
- "expected this to be saved from the previous pass.");
- EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith(
- VecEpilogueIterationCountCheck, VectorPH);
-
- EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
- VecEpilogueIterationCountCheck, ScalarPH);
-
- // Adjust the terminators of runtime check blocks and phis using them.
- BasicBlock *SCEVCheckBlock = RTChecks.getSCEVChecks().second;
- BasicBlock *MemCheckBlock = RTChecks.getMemRuntimeChecks().second;
- if (SCEVCheckBlock)
- SCEVCheckBlock->getTerminator()->replaceUsesOfWith(
- VecEpilogueIterationCountCheck, ScalarPH);
- if (MemCheckBlock)
- MemCheckBlock->getTerminator()->replaceUsesOfWith(
- VecEpilogueIterationCountCheck, ScalarPH);
-
- DT->changeImmediateDominator(ScalarPH, EPI.EpilogueIterationCountCheck);
-
- // The vec.epilog.iter.check block may contain Phi nodes from inductions or
- // reductions which merge control-flow from the latch block and the middle
- // block. Update the incoming values here and move the Phi into the preheader.
- SmallVector<PHINode *, 4> PhisInBlock(
- llvm::make_pointer_range(VecEpilogueIterationCountCheck->phis()));
-
- for (PHINode *Phi : PhisInBlock) {
- Phi->moveBefore(VectorPH->getFirstNonPHIIt());
- Phi->replaceIncomingBlockWith(
- VecEpilogueIterationCountCheck->getSinglePredecessor(),
- VecEpilogueIterationCountCheck);
-
- // If the phi doesn't have an incoming value from the
- // EpilogueIterationCountCheck, we are done. Otherwise remove the incoming
- // value and also those from other check blocks. This is needed for
- // reduction phis only.
- if (none_of(Phi->blocks(), [&](BasicBlock *IncB) {
- return EPI.EpilogueIterationCountCheck == IncB;
- }))
+ OriginalScalarPH->setName("vec.epilog.iter.check");
+ VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(OriginalScalarPH);
+ VPBasicBlock *OldEntry = Plan.getEntry();
+ for (auto &R : make_early_inc_range(
+ make_range(OldEntry->getFirstNonPhi(), OldEntry->end()))) {
+ if (isa<VPIRInstruction>(&R))
----------------
ayalz wrote:
Worth noting why phi's and VPIRInstructions are excluded when moving recipes from OldEntry to NewEntry? VPIRInstructions are unmovable by definition and phi's are expected to be VPIRPhi's (i.e., also VPIRInstructions)?
https://github.com/llvm/llvm-project/pull/157545