[llvm] [VPlan] Introduce explicit ExtractFromEnd recipes for live-outs. (PR #100658)

[Florian Hahn via llvm-commits]

================
@@ -8563,25 +8573,37 @@ static void addUsersInExitBlock(
     VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue, Plan);
     // Exit values for inductions are computed and updated outside of VPlan and
     // independent of induction recipes.
+    // Exit values for first-order recurrences are added separately in
+    // addLiveOutsForFirstOrderRecurrences.
     // TODO: Compute induction exit values in VPlan, use VPLiveOuts to update
     // live-outs.
     if ((isa<VPWidenIntOrFpInductionRecipe>(V) &&
          !cast<VPWidenIntOrFpInductionRecipe>(V)->getTruncInst()) ||
-        isa<VPWidenPointerInductionRecipe>(V) ||
+        isa<VPWidenPointerInductionRecipe, VPFirstOrderRecurrencePHIRecipe>(
+            V) ||
         (isa<Instruction>(IncomingValue) &&
          any_of(IncomingValue->users(), [&Inductions](User *U) {
            auto *P = dyn_cast<PHINode>(U);
            return P && Inductions.contains(P);
          })))
       continue;
-    Plan.addLiveOut(&ExitPhi, V);
+
+    VPValue *Ext = B.createNaryOp(
+        VPInstruction::ExtractFromEnd,
+        {V, Plan.getOrAddLiveIn(ConstantInt::get(
+                IntegerType::get(ExitBB->getContext(), 32), 1))});
+    Plan.addLiveOut(&ExitPhi, Ext);
   }
 }
 
-/// Feed a resume value for every FOR from the vector loop to the scalar loop,
-/// if middle block branches to scalar preheader, by introducing ExtractFromEnd
-/// and ResumePhi recipes in each, respectively, and a VPLiveOut which uses the
-/// latter and corresponds to the scalar header.
+/// Handle live-outs for first order reductions, both in the scalar preheader
+/// and the original exit block:
+/// 1. Feed a resume value for every FOR from the vector loop to the scalar
+///    loop, if middle block branches to scalar preheader, by introducing
+///    ExtractFromEnd and ResumePhi recipes in each, respectively, and a
+///    VPLiveOut which uses the latter and corresponds to the scalar header.
+/// 2. Feed the penultimate value of recurrences to their LCSSA phi users in
+///     the original exit block using a VPLiveOut.
----------------
fhahn wrote:

Fixed, thanks!

https://github.com/llvm/llvm-project/pull/100658