[llvm] [VPlan] Rewrite sinkScalarOperands (PR #151696)

Ramkumar Ramachandra via llvm-commits llvm-commits at lists.llvm.org
Wed Sep 3 03:56:36 PDT 2025


================
@@ -122,53 +122,64 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
 }
 
 static bool sinkScalarOperands(VPlan &Plan) {
-  auto Iter = vp_depth_first_deep(Plan.getEntry());
+  bool ScalarVFOnly = Plan.hasScalarVFOnly();
   bool Changed = false;
   // First, collect the operands of all recipes in replicate blocks as seeds for
   // sinking.
   SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
-  for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Iter)) {
+  for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(
+           vp_depth_first_deep(Plan.getEntry()))) {
     VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
     if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
       continue;
-    VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(EntryVPBB->getSuccessors()[0]);
+    VPBasicBlock *VPBB =
+        dyn_cast<VPBasicBlock>(EntryVPBB->getSuccessors().front());
     if (!VPBB || VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
       continue;
     for (auto &Recipe : *VPBB) {
-      for (VPValue *Op : Recipe.operands())
-        if (auto *Def =
-                dyn_cast_or_null<VPSingleDefRecipe>(Op->getDefiningRecipe()))
-          WorkList.insert(std::make_pair(VPBB, Def));
+      for (VPValue *Op : Recipe.operands()) {
+        auto *Def =
+            dyn_cast_or_null<VPSingleDefRecipe>(Op->getDefiningRecipe());
+        if (!Def)
+          continue;
+
+        // We only know how to duplicate VPReplicateRecipes and
+        // VPScalarIVStepsRecipes for now.
+        if (!isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(Def))
+          continue;
+
+        // TODO: Relax checks in the future, e.g. we could also sink reads if
+        // their memory location is not modified in the vector loop.
+        if (Def->getParent() == VPBB || Def->mayHaveSideEffects() ||
+            Def->mayReadOrWriteMemory())
+          continue;
+
+        if (auto *RepR = dyn_cast<VPReplicateRecipe>(Op))
+          if (!ScalarVFOnly && RepR->isSingleScalar())
+            continue;
+
+        WorkList.insert(std::make_pair(VPBB, Def));
+      }
     }
   }
 
-  bool ScalarVFOnly = Plan.hasScalarVFOnly();
   // Try to sink each replicate or scalar IV steps recipe in the worklist.
-  for (unsigned I = 0; I != WorkList.size(); ++I) {
+  for (const auto &Item : WorkList) {
     VPBasicBlock *SinkTo;
     VPSingleDefRecipe *SinkCandidate;
-    std::tie(SinkTo, SinkCandidate) = WorkList[I];
-    if (SinkCandidate->getParent() == SinkTo ||
-        SinkCandidate->mayHaveSideEffects() ||
-        SinkCandidate->mayReadOrWriteMemory())
-      continue;
-    if (auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
-      if (!ScalarVFOnly && RepR->isSingleScalar())
-        continue;
-    } else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
-      continue;
+    std::tie(SinkTo, SinkCandidate) = Item;
 
-    bool NeedsDuplicating = false;
     // All recipe users of the sink candidate must be in the same block SinkTo
     // or all users outside of SinkTo must be uniform-after-vectorization (
     // i.e., only first lane is used) . In the latter case, we need to duplicate
     // SinkCandidate.
-    auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
-                            SinkCandidate](VPUser *U) {
+    bool NeedsDuplicating = false;
+    auto CanSinkWithUser = [SinkTo, SinkCandidate,
+                            &NeedsDuplicating](VPUser *U) {
       auto *UI = cast<VPRecipeBase>(U);
       if (UI->getParent() == SinkTo)
         return true;
-      NeedsDuplicating = UI->onlyFirstLaneUsed(SinkCandidate);
+      NeedsDuplicating |= UI->onlyFirstLaneUsed(SinkCandidate);
----------------
artagnon wrote:

Ah, it wasn't a bug after all.

https://github.com/llvm/llvm-project/pull/151696


More information about the llvm-commits mailing list