[llvm] [LV][EVL] Support fixed-order recurrence idiom with EVL tail folding. (PR #124093)

[Alexey Bataev via llvm-commits]

================
@@ -1635,18 +1635,62 @@ void VPlanTransforms::addActiveLaneMask(
     HeaderMask->replaceAllUsesWith(LaneMask);
 }
 
+/// Adjust the way the resume value is obtained when using tail folding by EVL.
+/// Expanding ExtractFromEnd since the penultimate EVL could not equals to
+/// VFxUF. Expand
+///   %resume = ExtractFromEnd %vec, 1
+/// to
+///   %last.active.idx = sub %EVL, 1
+///   %resume = extractelement %vec, %last.active.idx
+static void adjustResumePhisForEVL(VPlan &Plan, VPValue &EVL) {
+  LLVMContext &Ctx = Plan.getCanonicalIV()->getScalarType()->getContext();
+  using namespace VPlanPatternMatch;
+  for (VPRecipeBase &R : *cast<VPBasicBlock>(Plan.getScalarPreheader())) {
+    VPValue *FromMiddleBlock;
+    if (!match(&R, m_VPInstruction<VPInstruction::ResumePhi>(
+                       m_VPValue(FromMiddleBlock), m_VPValue())))
+      continue;
+
+    VPValue *ExtractFrom;
+    if (match(FromMiddleBlock, m_VPInstruction<VPInstruction::ExtractFromEnd>(
+                                   m_VPValue(ExtractFrom), m_SpecificInt(1)))) {
+      // Skip if all elements are uniform.
+      if (vputils::isUniformAfterVectorization(ExtractFrom))
+        continue;
+      auto *ExtractR = cast<VPInstruction>(FromMiddleBlock);
+      VPBuilder Builder(ExtractR);
+      VPValue *OneVPV =
+          Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(Ctx), 1));
+      VPValue *LastActiveIdx =
+          Builder.createNaryOp(Instruction::Sub, {&EVL, OneVPV},
+                               ExtractR->getDebugLoc(), "last.active.idx");
+      VPValue *NewExtract = Builder.createNaryOp(
+          Instruction::ExtractElement, {ExtractFrom, LastActiveIdx},
+          ExtractR->getDebugLoc(), ExtractR->getName());
+      ExtractR->replaceAllUsesWith(NewExtract);
+      ExtractR->eraseFromParent();
+    }
+    assert((!dyn_cast<VPInstruction>(FromMiddleBlock) ||
+            cast<VPInstruction>(FromMiddleBlock)->getOpcode() !=
+                VPInstruction::ExtractFromEnd) &&
+           "Only extract the last lane for resumed values");
+  }
+}
+
 /// Try to convert \p CurRecipe to a corresponding EVL-based recipe. Returns
 /// nullptr if no EVL-based recipe could be created.
 /// \p HeaderMask  Header Mask.
 /// \p CurRecipe   Recipe to be transform.
 /// \p TypeInfo    VPlan-based type analysis.
 /// \p AllOneMask  The vector mask parameter of vector-predication intrinsics.
 /// \p EVL         The explicit vector length parameter of vector-predication
-/// intrinsics.
+///                intrinsics.
+/// \p PrevEVL     The explicit vector length of the previous iteration.
 static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
                                      VPRecipeBase &CurRecipe,
                                      VPTypeAnalysis &TypeInfo,
-                                     VPValue &AllOneMask, VPValue &EVL) {
+                                     VPValue &AllOneMask, VPValue &EVL,
+                                     VPValue *PrevEVL) {
----------------
alexey-bataev wrote:

```suggestion
                                     VPValue &PrevEVL) {
```
?

https://github.com/llvm/llvm-project/pull/124093