[llvm] d7414ff - [LoopFusion][NFC] Share fusion tail between guarded and unguarded paths (#205492)

via llvm-commits llvm-commits at lists.llvm.org
Wed Jun 24 00:43:00 PDT 2026


Author: Madhur Amilkanthwar
Date: 2026-06-24T07:42:54Z
New Revision: d7414ff58979052ee6f9979c96e54b565870ca18

URL: https://github.com/llvm/llvm-project/commit/d7414ff58979052ee6f9979c96e54b565870ca18
DIFF: https://github.com/llvm/llvm-project/commit/d7414ff58979052ee6f9979c96e54b565870ca18.diff

LOG: [LoopFusion][NFC] Share fusion tail between guarded and unguarded paths (#205492)

`performFusion()` and `fuseGuardedLoops()` carried two
character-for-character identical tails: header-PHI migration plus latch
rewiring, and the SCEV-forget / block-merge / latch-merge finalization.
Extract them into `rewireFusedHeaderPHIsAndLatches()` and
`finalizeFusedLoop()` and call both from each path.

Added: 
    

Modified: 
    llvm/lib/Transforms/Scalar/LoopFuse.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 1b83c971c01bf..012a54de173e7 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1354,6 +1354,121 @@ struct LoopFuser {
     }
   }
 
+  /// Move FC1's header PHIs into FC0's header, insert the loop-carried PHIs
+  /// needed to keep SSA valid when FC0 exits without taking its back-edge, and
+  /// rewire both latches to form the fused loop. Latch dominator-tree updates
+  /// are appended to \p TreeUpdates for the caller to apply.
+  void rewireFusedHeaderPHIsAndLatches(
+      const FusionCandidate &FC0, const FusionCandidate &FC1,
+      const SmallVectorImpl<PHINode *> &OriginalFC0PHIs,
+      SmallVectorImpl<DominatorTree::UpdateType> &TreeUpdates) {
+    // Moves the phi nodes from the second to the first loops header block.
+    while (PHINode *PHI = dyn_cast<PHINode>(&FC1.Header->front())) {
+      if (SE.isSCEVable(PHI->getType()))
+        SE.forgetValue(PHI);
+      if (PHI->hasNUsesOrMore(1))
+        PHI->moveBefore(FC0.Header->getFirstInsertionPt());
+      else
+        PHI->eraseFromParent();
+    }
+
+    // Introduce new phi nodes in the second loop header to ensure
+    // exiting the first and jumping to the header of the second does not break
+    // the SSA property of the phis originally in the first loop. See also the
+    // comment above.
+    BasicBlock::iterator L1HeaderIP = FC1.Header->begin();
+    for (PHINode *LCPHI : OriginalFC0PHIs) {
+      int L1LatchBBIdx = LCPHI->getBasicBlockIndex(FC1.Latch);
+      assert(L1LatchBBIdx >= 0 &&
+             "Expected loop carried value to be rewired at this point!");
+
+      Value *LCV = LCPHI->getIncomingValue(L1LatchBBIdx);
+
+      PHINode *L1HeaderPHI =
+          PHINode::Create(LCV->getType(), 2, LCPHI->getName() + ".afterFC0");
+      L1HeaderPHI->insertBefore(L1HeaderIP);
+      L1HeaderPHI->addIncoming(LCV, FC0.Latch);
+      L1HeaderPHI->addIncoming(PoisonValue::get(LCV->getType()),
+                               FC0.ExitingBlock);
+
+      LCPHI->setIncomingValue(L1LatchBBIdx, L1HeaderPHI);
+    }
+
+    // Replace latch terminator destinations.
+    FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
+    FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
+
+    // Modify the latch branch of FC0 to be unconditional as both successors of
+    // the branch are the same.
+    simplifyLatchBranch(FC0);
+
+    // If FC0.Latch and FC0.ExitingBlock are the same then we have already
+    // performed the updates above.
+    if (FC0.Latch != FC0.ExitingBlock)
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Insert, FC0.Latch, FC1.Header));
+
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
+                                                       FC0.Latch, FC0.Header));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Insert,
+                                                       FC1.Latch, FC0.Header));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
+                                                       FC1.Latch, FC1.Header));
+  }
+
+  /// Forget cached SCEV state for both loops, move all of FC1's blocks and
+  /// child loops into FC0, erase the now-empty FC1, and merge the latches.
+  /// Returns the fused loop (FC0.L).
+  Loop *finalizeFusedLoop(const FusionCandidate &FC0,
+                          const FusionCandidate &FC1) {
+    // Is there a way to keep SE up-to-date so we don't need to forget the loops
+    // and rebuild the information in subsequent passes of fusion?
+    // Note: Need to forget the loops before merging the loop latches, as
+    // mergeLatch may remove the only block in FC1.
+    SE.forgetLoop(FC1.L);
+    SE.forgetLoop(FC0.L);
+
+    // Merge the loops.
+    SmallVector<BasicBlock *, 8> Blocks(FC1.L->blocks());
+    for (BasicBlock *BB : Blocks) {
+      FC0.L->addBlockEntry(BB);
+      FC1.L->removeBlockFromLoop(BB);
+      if (LI.getLoopFor(BB) != FC1.L)
+        continue;
+      LI.changeLoopFor(BB, FC0.L);
+    }
+    while (!FC1.L->isInnermost()) {
+      const auto &ChildLoopIt = FC1.L->begin();
+      Loop *ChildLoop = *ChildLoopIt;
+      FC1.L->removeChildLoop(ChildLoopIt);
+      FC0.L->addChildLoop(ChildLoop);
+    }
+
+    // Delete the now empty loop L1.
+    LI.erase(FC1.L);
+
+    // Forget block dispositions as well, so that there are no dangling
+    // pointers to erased/free'ed blocks. It should be done after mergeLatch()
+    // since merging the latches may affect the dispositions.
+    SE.forgetBlockAndLoopDispositions();
+
+    // Move instructions from FC0.Latch to FC1.Latch.
+    // Note: mergeLatch requires an updated DT.
+    mergeLatch(FC0, FC1);
+
+#ifndef NDEBUG
+    assert(!verifyFunction(*FC0.Header->getParent(), &errs()));
+    assert(DT.verify(DominatorTree::VerificationLevel::Fast));
+    assert(PDT.verify());
+    LI.verify(DT);
+    SE.verify();
+#endif
+
+    LLVM_DEBUG(dbgs() << "Fusion done:\n");
+
+    return FC0.L;
+  }
+
   /// Fuse two fusion candidates, creating a new fused loop.
   ///
   /// This method contains the mechanics of fusing two loops, represented by \p
@@ -1472,58 +1587,7 @@ struct LoopFuser {
     TreeUpdates.emplace_back(DominatorTree::UpdateType(
         DominatorTree::Delete, FC1.Preheader, FC1.Header));
 
-    // Moves the phi nodes from the second to the first loops header block.
-    while (PHINode *PHI = dyn_cast<PHINode>(&FC1.Header->front())) {
-      if (SE.isSCEVable(PHI->getType()))
-        SE.forgetValue(PHI);
-      if (PHI->hasNUsesOrMore(1))
-        PHI->moveBefore(FC0.Header->getFirstInsertionPt());
-      else
-        PHI->eraseFromParent();
-    }
-
-    // Introduce new phi nodes in the second loop header to ensure
-    // exiting the first and jumping to the header of the second does not break
-    // the SSA property of the phis originally in the first loop. See also the
-    // comment above.
-    BasicBlock::iterator L1HeaderIP = FC1.Header->begin();
-    for (PHINode *LCPHI : OriginalFC0PHIs) {
-      int L1LatchBBIdx = LCPHI->getBasicBlockIndex(FC1.Latch);
-      assert(L1LatchBBIdx >= 0 &&
-             "Expected loop carried value to be rewired at this point!");
-
-      Value *LCV = LCPHI->getIncomingValue(L1LatchBBIdx);
-
-      PHINode *L1HeaderPHI =
-          PHINode::Create(LCV->getType(), 2, LCPHI->getName() + ".afterFC0");
-      L1HeaderPHI->insertBefore(L1HeaderIP);
-      L1HeaderPHI->addIncoming(LCV, FC0.Latch);
-      L1HeaderPHI->addIncoming(PoisonValue::get(LCV->getType()),
-                               FC0.ExitingBlock);
-
-      LCPHI->setIncomingValue(L1LatchBBIdx, L1HeaderPHI);
-    }
-
-    // Replace latch terminator destinations.
-    FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
-    FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
-
-    // Modify the latch branch of FC0 to be unconditional as both successors of
-    // the branch are the same.
-    simplifyLatchBranch(FC0);
-
-    // If FC0.Latch and FC0.ExitingBlock are the same then we have already
-    // performed the updates above.
-    if (FC0.Latch != FC0.ExitingBlock)
-      TreeUpdates.emplace_back(DominatorTree::UpdateType(
-          DominatorTree::Insert, FC0.Latch, FC1.Header));
-
-    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
-                                                       FC0.Latch, FC0.Header));
-    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Insert,
-                                                       FC1.Latch, FC0.Header));
-    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
-                                                       FC1.Latch, FC1.Header));
+    rewireFusedHeaderPHIsAndLatches(FC0, FC1, OriginalFC0PHIs, TreeUpdates);
 
     // Update DT/PDT
     DTU.applyUpdates(TreeUpdates);
@@ -1537,52 +1601,7 @@ struct LoopFuser {
 
     DTU.flush();
 
-    // Is there a way to keep SE up-to-date so we don't need to forget the loops
-    // and rebuild the information in subsequent passes of fusion?
-    // Note: Need to forget the loops before merging the loop latches, as
-    // mergeLatch may remove the only block in FC1.
-    SE.forgetLoop(FC1.L);
-    SE.forgetLoop(FC0.L);
-
-    // Merge the loops.
-    SmallVector<BasicBlock *, 8> Blocks(FC1.L->blocks());
-    for (BasicBlock *BB : Blocks) {
-      FC0.L->addBlockEntry(BB);
-      FC1.L->removeBlockFromLoop(BB);
-      if (LI.getLoopFor(BB) != FC1.L)
-        continue;
-      LI.changeLoopFor(BB, FC0.L);
-    }
-    while (!FC1.L->isInnermost()) {
-      const auto &ChildLoopIt = FC1.L->begin();
-      Loop *ChildLoop = *ChildLoopIt;
-      FC1.L->removeChildLoop(ChildLoopIt);
-      FC0.L->addChildLoop(ChildLoop);
-    }
-
-    // Delete the now empty loop L1.
-    LI.erase(FC1.L);
-
-    // Forget block dispositions as well, so that there are no dangling
-    // pointers to erased/free'ed blocks. It should be done after mergeLatch()
-    // since merging the latches may affect the dispositions.
-    SE.forgetBlockAndLoopDispositions();
-
-    // Move instructions from FC0.Latch to FC1.Latch.
-    // Note: mergeLatch requires an updated DT.
-    mergeLatch(FC0, FC1);
-
-#ifndef NDEBUG
-    assert(!verifyFunction(*FC0.Header->getParent(), &errs()));
-    assert(DT.verify(DominatorTree::VerificationLevel::Fast));
-    assert(PDT.verify());
-    LI.verify(DT);
-    SE.verify();
-#endif
-
-    LLVM_DEBUG(dbgs() << "Fusion done:\n");
-
-    return FC0.L;
+    return finalizeFusedLoop(FC0, FC1);
   }
 
   /// Report details on loop fusion opportunities.
@@ -1758,60 +1777,7 @@ struct LoopFuser {
     TreeUpdates.emplace_back(DominatorTree::UpdateType(
         DominatorTree::Delete, FC1.Preheader, FC1.Header));
 
-    // Moves the phi nodes from the second to the first loops header block.
-    while (PHINode *PHI = dyn_cast<PHINode>(&FC1.Header->front())) {
-      if (SE.isSCEVable(PHI->getType()))
-        SE.forgetValue(PHI);
-      if (PHI->hasNUsesOrMore(1))
-        PHI->moveBefore(FC0.Header->getFirstInsertionPt());
-      else
-        PHI->eraseFromParent();
-    }
-
-    // Introduce new phi nodes in the second loop header to ensure
-    // exiting the first and jumping to the header of the second does not break
-    // the SSA property of the phis originally in the first loop. See also the
-    // comment above.
-    BasicBlock::iterator L1HeaderIP = FC1.Header->begin();
-    for (PHINode *LCPHI : OriginalFC0PHIs) {
-      int L1LatchBBIdx = LCPHI->getBasicBlockIndex(FC1.Latch);
-      assert(L1LatchBBIdx >= 0 &&
-             "Expected loop carried value to be rewired at this point!");
-
-      Value *LCV = LCPHI->getIncomingValue(L1LatchBBIdx);
-
-      PHINode *L1HeaderPHI =
-          PHINode::Create(LCV->getType(), 2, LCPHI->getName() + ".afterFC0");
-      L1HeaderPHI->insertBefore(L1HeaderIP);
-      L1HeaderPHI->addIncoming(LCV, FC0.Latch);
-      L1HeaderPHI->addIncoming(PoisonValue::get(LCV->getType()),
-                               FC0.ExitingBlock);
-
-      LCPHI->setIncomingValue(L1LatchBBIdx, L1HeaderPHI);
-    }
-
-    // Update the latches
-
-    // Replace latch terminator destinations.
-    FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
-    FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
-
-    // Modify the latch branch of FC0 to be unconditional as both successors of
-    // the branch are the same.
-    simplifyLatchBranch(FC0);
-
-    // If FC0.Latch and FC0.ExitingBlock are the same then we have already
-    // performed the updates above.
-    if (FC0.Latch != FC0.ExitingBlock)
-      TreeUpdates.emplace_back(DominatorTree::UpdateType(
-          DominatorTree::Insert, FC0.Latch, FC1.Header));
-
-    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
-                                                       FC0.Latch, FC0.Header));
-    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Insert,
-                                                       FC1.Latch, FC0.Header));
-    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
-                                                       FC1.Latch, FC1.Header));
+    rewireFusedHeaderPHIsAndLatches(FC0, FC1, OriginalFC0PHIs, TreeUpdates);
 
     // All done
     // Apply the updates to the Dominator Tree and cleanup.
@@ -1834,52 +1800,7 @@ struct LoopFuser {
     DTU.deleteBB(FC0.ExitBlock);
     DTU.flush();
 
-    // Is there a way to keep SE up-to-date so we don't need to forget the loops
-    // and rebuild the information in subsequent passes of fusion?
-    // Note: Need to forget the loops before merging the loop latches, as
-    // mergeLatch may remove the only block in FC1.
-    SE.forgetLoop(FC1.L);
-    SE.forgetLoop(FC0.L);
-
-    // Merge the loops.
-    SmallVector<BasicBlock *, 8> Blocks(FC1.L->blocks());
-    for (BasicBlock *BB : Blocks) {
-      FC0.L->addBlockEntry(BB);
-      FC1.L->removeBlockFromLoop(BB);
-      if (LI.getLoopFor(BB) != FC1.L)
-        continue;
-      LI.changeLoopFor(BB, FC0.L);
-    }
-    while (!FC1.L->isInnermost()) {
-      const auto &ChildLoopIt = FC1.L->begin();
-      Loop *ChildLoop = *ChildLoopIt;
-      FC1.L->removeChildLoop(ChildLoopIt);
-      FC0.L->addChildLoop(ChildLoop);
-    }
-
-    // Delete the now empty loop L1.
-    LI.erase(FC1.L);
-
-    // Forget block dispositions as well, so that there are no dangling
-    // pointers to erased/free'ed blocks. It should be done after mergeLatch()
-    // since merging the latches may affect the dispositions.
-    SE.forgetBlockAndLoopDispositions();
-
-    // Move instructions from FC0.Latch to FC1.Latch.
-    // Note: mergeLatch requires an updated DT.
-    mergeLatch(FC0, FC1);
-
-#ifndef NDEBUG
-    assert(!verifyFunction(*FC0.Header->getParent(), &errs()));
-    assert(DT.verify(DominatorTree::VerificationLevel::Fast));
-    assert(PDT.verify());
-    LI.verify(DT);
-    SE.verify();
-#endif
-
-    LLVM_DEBUG(dbgs() << "Fusion done:\n");
-
-    return FC0.L;
+    return finalizeFusedLoop(FC0, FC1);
   }
 };
 } // namespace


        


More information about the llvm-commits mailing list