[llvm] d7414ff - [LoopFusion][NFC] Share fusion tail between guarded and unguarded paths (#205492)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 24 00:43:00 PDT 2026
Author: Madhur Amilkanthwar
Date: 2026-06-24T07:42:54Z
New Revision: d7414ff58979052ee6f9979c96e54b565870ca18
URL: https://github.com/llvm/llvm-project/commit/d7414ff58979052ee6f9979c96e54b565870ca18
DIFF: https://github.com/llvm/llvm-project/commit/d7414ff58979052ee6f9979c96e54b565870ca18.diff
LOG: [LoopFusion][NFC] Share fusion tail between guarded and unguarded paths (#205492)
`performFusion()` and `fuseGuardedLoops()` carried two
character-for-character identical tails: header-PHI migration plus latch
rewiring, and the SCEV-forget / block-merge / latch-merge finalization.
Extract them into `rewireFusedHeaderPHIsAndLatches()` and
`finalizeFusedLoop()` and call both from each path.
Added:
Modified:
llvm/lib/Transforms/Scalar/LoopFuse.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 1b83c971c01bf..012a54de173e7 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1354,6 +1354,121 @@ struct LoopFuser {
}
}
+ /// Move FC1's header PHIs into FC0's header, insert the loop-carried PHIs
+ /// needed to keep SSA valid when FC0 exits without taking its back-edge, and
+ /// rewire both latches to form the fused loop. Latch dominator-tree updates
+ /// are appended to \p TreeUpdates for the caller to apply.
+ void rewireFusedHeaderPHIsAndLatches(
+ const FusionCandidate &FC0, const FusionCandidate &FC1,
+ const SmallVectorImpl<PHINode *> &OriginalFC0PHIs,
+ SmallVectorImpl<DominatorTree::UpdateType> &TreeUpdates) {
+ // Moves the phi nodes from the second to the first loops header block.
+ while (PHINode *PHI = dyn_cast<PHINode>(&FC1.Header->front())) {
+ if (SE.isSCEVable(PHI->getType()))
+ SE.forgetValue(PHI);
+ if (PHI->hasNUsesOrMore(1))
+ PHI->moveBefore(FC0.Header->getFirstInsertionPt());
+ else
+ PHI->eraseFromParent();
+ }
+
+ // Introduce new phi nodes in the second loop header to ensure
+ // exiting the first and jumping to the header of the second does not break
+ // the SSA property of the phis originally in the first loop. See also the
+ // comment above.
+ BasicBlock::iterator L1HeaderIP = FC1.Header->begin();
+ for (PHINode *LCPHI : OriginalFC0PHIs) {
+ int L1LatchBBIdx = LCPHI->getBasicBlockIndex(FC1.Latch);
+ assert(L1LatchBBIdx >= 0 &&
+ "Expected loop carried value to be rewired at this point!");
+
+ Value *LCV = LCPHI->getIncomingValue(L1LatchBBIdx);
+
+ PHINode *L1HeaderPHI =
+ PHINode::Create(LCV->getType(), 2, LCPHI->getName() + ".afterFC0");
+ L1HeaderPHI->insertBefore(L1HeaderIP);
+ L1HeaderPHI->addIncoming(LCV, FC0.Latch);
+ L1HeaderPHI->addIncoming(PoisonValue::get(LCV->getType()),
+ FC0.ExitingBlock);
+
+ LCPHI->setIncomingValue(L1LatchBBIdx, L1HeaderPHI);
+ }
+
+ // Replace latch terminator destinations.
+ FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
+ FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
+
+ // Modify the latch branch of FC0 to be unconditional as both successors of
+ // the branch are the same.
+ simplifyLatchBranch(FC0);
+
+ // If FC0.Latch and FC0.ExitingBlock are the same then we have already
+ // performed the updates above.
+ if (FC0.Latch != FC0.ExitingBlock)
+ TreeUpdates.emplace_back(DominatorTree::UpdateType(
+ DominatorTree::Insert, FC0.Latch, FC1.Header));
+
+ TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
+ FC0.Latch, FC0.Header));
+ TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Insert,
+ FC1.Latch, FC0.Header));
+ TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
+ FC1.Latch, FC1.Header));
+ }
+
+ /// Forget cached SCEV state for both loops, move all of FC1's blocks and
+ /// child loops into FC0, erase the now-empty FC1, and merge the latches.
+ /// Returns the fused loop (FC0.L).
+ Loop *finalizeFusedLoop(const FusionCandidate &FC0,
+ const FusionCandidate &FC1) {
+ // Is there a way to keep SE up-to-date so we don't need to forget the loops
+ // and rebuild the information in subsequent passes of fusion?
+ // Note: Need to forget the loops before merging the loop latches, as
+ // mergeLatch may remove the only block in FC1.
+ SE.forgetLoop(FC1.L);
+ SE.forgetLoop(FC0.L);
+
+ // Merge the loops.
+ SmallVector<BasicBlock *, 8> Blocks(FC1.L->blocks());
+ for (BasicBlock *BB : Blocks) {
+ FC0.L->addBlockEntry(BB);
+ FC1.L->removeBlockFromLoop(BB);
+ if (LI.getLoopFor(BB) != FC1.L)
+ continue;
+ LI.changeLoopFor(BB, FC0.L);
+ }
+ while (!FC1.L->isInnermost()) {
+ const auto &ChildLoopIt = FC1.L->begin();
+ Loop *ChildLoop = *ChildLoopIt;
+ FC1.L->removeChildLoop(ChildLoopIt);
+ FC0.L->addChildLoop(ChildLoop);
+ }
+
+ // Delete the now empty loop L1.
+ LI.erase(FC1.L);
+
+ // Forget block dispositions as well, so that there are no dangling
+ // pointers to erased/free'ed blocks. It should be done after mergeLatch()
+ // since merging the latches may affect the dispositions.
+ SE.forgetBlockAndLoopDispositions();
+
+ // Move instructions from FC0.Latch to FC1.Latch.
+ // Note: mergeLatch requires an updated DT.
+ mergeLatch(FC0, FC1);
+
+#ifndef NDEBUG
+ assert(!verifyFunction(*FC0.Header->getParent(), &errs()));
+ assert(DT.verify(DominatorTree::VerificationLevel::Fast));
+ assert(PDT.verify());
+ LI.verify(DT);
+ SE.verify();
+#endif
+
+ LLVM_DEBUG(dbgs() << "Fusion done:\n");
+
+ return FC0.L;
+ }
+
/// Fuse two fusion candidates, creating a new fused loop.
///
/// This method contains the mechanics of fusing two loops, represented by \p
@@ -1472,58 +1587,7 @@ struct LoopFuser {
TreeUpdates.emplace_back(DominatorTree::UpdateType(
DominatorTree::Delete, FC1.Preheader, FC1.Header));
- // Moves the phi nodes from the second to the first loops header block.
- while (PHINode *PHI = dyn_cast<PHINode>(&FC1.Header->front())) {
- if (SE.isSCEVable(PHI->getType()))
- SE.forgetValue(PHI);
- if (PHI->hasNUsesOrMore(1))
- PHI->moveBefore(FC0.Header->getFirstInsertionPt());
- else
- PHI->eraseFromParent();
- }
-
- // Introduce new phi nodes in the second loop header to ensure
- // exiting the first and jumping to the header of the second does not break
- // the SSA property of the phis originally in the first loop. See also the
- // comment above.
- BasicBlock::iterator L1HeaderIP = FC1.Header->begin();
- for (PHINode *LCPHI : OriginalFC0PHIs) {
- int L1LatchBBIdx = LCPHI->getBasicBlockIndex(FC1.Latch);
- assert(L1LatchBBIdx >= 0 &&
- "Expected loop carried value to be rewired at this point!");
-
- Value *LCV = LCPHI->getIncomingValue(L1LatchBBIdx);
-
- PHINode *L1HeaderPHI =
- PHINode::Create(LCV->getType(), 2, LCPHI->getName() + ".afterFC0");
- L1HeaderPHI->insertBefore(L1HeaderIP);
- L1HeaderPHI->addIncoming(LCV, FC0.Latch);
- L1HeaderPHI->addIncoming(PoisonValue::get(LCV->getType()),
- FC0.ExitingBlock);
-
- LCPHI->setIncomingValue(L1LatchBBIdx, L1HeaderPHI);
- }
-
- // Replace latch terminator destinations.
- FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
- FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
-
- // Modify the latch branch of FC0 to be unconditional as both successors of
- // the branch are the same.
- simplifyLatchBranch(FC0);
-
- // If FC0.Latch and FC0.ExitingBlock are the same then we have already
- // performed the updates above.
- if (FC0.Latch != FC0.ExitingBlock)
- TreeUpdates.emplace_back(DominatorTree::UpdateType(
- DominatorTree::Insert, FC0.Latch, FC1.Header));
-
- TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
- FC0.Latch, FC0.Header));
- TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Insert,
- FC1.Latch, FC0.Header));
- TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
- FC1.Latch, FC1.Header));
+ rewireFusedHeaderPHIsAndLatches(FC0, FC1, OriginalFC0PHIs, TreeUpdates);
// Update DT/PDT
DTU.applyUpdates(TreeUpdates);
@@ -1537,52 +1601,7 @@ struct LoopFuser {
DTU.flush();
- // Is there a way to keep SE up-to-date so we don't need to forget the loops
- // and rebuild the information in subsequent passes of fusion?
- // Note: Need to forget the loops before merging the loop latches, as
- // mergeLatch may remove the only block in FC1.
- SE.forgetLoop(FC1.L);
- SE.forgetLoop(FC0.L);
-
- // Merge the loops.
- SmallVector<BasicBlock *, 8> Blocks(FC1.L->blocks());
- for (BasicBlock *BB : Blocks) {
- FC0.L->addBlockEntry(BB);
- FC1.L->removeBlockFromLoop(BB);
- if (LI.getLoopFor(BB) != FC1.L)
- continue;
- LI.changeLoopFor(BB, FC0.L);
- }
- while (!FC1.L->isInnermost()) {
- const auto &ChildLoopIt = FC1.L->begin();
- Loop *ChildLoop = *ChildLoopIt;
- FC1.L->removeChildLoop(ChildLoopIt);
- FC0.L->addChildLoop(ChildLoop);
- }
-
- // Delete the now empty loop L1.
- LI.erase(FC1.L);
-
- // Forget block dispositions as well, so that there are no dangling
- // pointers to erased/free'ed blocks. It should be done after mergeLatch()
- // since merging the latches may affect the dispositions.
- SE.forgetBlockAndLoopDispositions();
-
- // Move instructions from FC0.Latch to FC1.Latch.
- // Note: mergeLatch requires an updated DT.
- mergeLatch(FC0, FC1);
-
-#ifndef NDEBUG
- assert(!verifyFunction(*FC0.Header->getParent(), &errs()));
- assert(DT.verify(DominatorTree::VerificationLevel::Fast));
- assert(PDT.verify());
- LI.verify(DT);
- SE.verify();
-#endif
-
- LLVM_DEBUG(dbgs() << "Fusion done:\n");
-
- return FC0.L;
+ return finalizeFusedLoop(FC0, FC1);
}
/// Report details on loop fusion opportunities.
@@ -1758,60 +1777,7 @@ struct LoopFuser {
TreeUpdates.emplace_back(DominatorTree::UpdateType(
DominatorTree::Delete, FC1.Preheader, FC1.Header));
- // Moves the phi nodes from the second to the first loops header block.
- while (PHINode *PHI = dyn_cast<PHINode>(&FC1.Header->front())) {
- if (SE.isSCEVable(PHI->getType()))
- SE.forgetValue(PHI);
- if (PHI->hasNUsesOrMore(1))
- PHI->moveBefore(FC0.Header->getFirstInsertionPt());
- else
- PHI->eraseFromParent();
- }
-
- // Introduce new phi nodes in the second loop header to ensure
- // exiting the first and jumping to the header of the second does not break
- // the SSA property of the phis originally in the first loop. See also the
- // comment above.
- BasicBlock::iterator L1HeaderIP = FC1.Header->begin();
- for (PHINode *LCPHI : OriginalFC0PHIs) {
- int L1LatchBBIdx = LCPHI->getBasicBlockIndex(FC1.Latch);
- assert(L1LatchBBIdx >= 0 &&
- "Expected loop carried value to be rewired at this point!");
-
- Value *LCV = LCPHI->getIncomingValue(L1LatchBBIdx);
-
- PHINode *L1HeaderPHI =
- PHINode::Create(LCV->getType(), 2, LCPHI->getName() + ".afterFC0");
- L1HeaderPHI->insertBefore(L1HeaderIP);
- L1HeaderPHI->addIncoming(LCV, FC0.Latch);
- L1HeaderPHI->addIncoming(PoisonValue::get(LCV->getType()),
- FC0.ExitingBlock);
-
- LCPHI->setIncomingValue(L1LatchBBIdx, L1HeaderPHI);
- }
-
- // Update the latches
-
- // Replace latch terminator destinations.
- FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
- FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
-
- // Modify the latch branch of FC0 to be unconditional as both successors of
- // the branch are the same.
- simplifyLatchBranch(FC0);
-
- // If FC0.Latch and FC0.ExitingBlock are the same then we have already
- // performed the updates above.
- if (FC0.Latch != FC0.ExitingBlock)
- TreeUpdates.emplace_back(DominatorTree::UpdateType(
- DominatorTree::Insert, FC0.Latch, FC1.Header));
-
- TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
- FC0.Latch, FC0.Header));
- TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Insert,
- FC1.Latch, FC0.Header));
- TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
- FC1.Latch, FC1.Header));
+ rewireFusedHeaderPHIsAndLatches(FC0, FC1, OriginalFC0PHIs, TreeUpdates);
// All done
// Apply the updates to the Dominator Tree and cleanup.
@@ -1834,52 +1800,7 @@ struct LoopFuser {
DTU.deleteBB(FC0.ExitBlock);
DTU.flush();
- // Is there a way to keep SE up-to-date so we don't need to forget the loops
- // and rebuild the information in subsequent passes of fusion?
- // Note: Need to forget the loops before merging the loop latches, as
- // mergeLatch may remove the only block in FC1.
- SE.forgetLoop(FC1.L);
- SE.forgetLoop(FC0.L);
-
- // Merge the loops.
- SmallVector<BasicBlock *, 8> Blocks(FC1.L->blocks());
- for (BasicBlock *BB : Blocks) {
- FC0.L->addBlockEntry(BB);
- FC1.L->removeBlockFromLoop(BB);
- if (LI.getLoopFor(BB) != FC1.L)
- continue;
- LI.changeLoopFor(BB, FC0.L);
- }
- while (!FC1.L->isInnermost()) {
- const auto &ChildLoopIt = FC1.L->begin();
- Loop *ChildLoop = *ChildLoopIt;
- FC1.L->removeChildLoop(ChildLoopIt);
- FC0.L->addChildLoop(ChildLoop);
- }
-
- // Delete the now empty loop L1.
- LI.erase(FC1.L);
-
- // Forget block dispositions as well, so that there are no dangling
- // pointers to erased/free'ed blocks. It should be done after mergeLatch()
- // since merging the latches may affect the dispositions.
- SE.forgetBlockAndLoopDispositions();
-
- // Move instructions from FC0.Latch to FC1.Latch.
- // Note: mergeLatch requires an updated DT.
- mergeLatch(FC0, FC1);
-
-#ifndef NDEBUG
- assert(!verifyFunction(*FC0.Header->getParent(), &errs()));
- assert(DT.verify(DominatorTree::VerificationLevel::Fast));
- assert(PDT.verify());
- LI.verify(DT);
- SE.verify();
-#endif
-
- LLVM_DEBUG(dbgs() << "Fusion done:\n");
-
- return FC0.L;
+ return finalizeFusedLoop(FC0, FC1);
}
};
} // namespace
More information about the llvm-commits
mailing list