[llvm] [LV, VP]VP intrinsics support for the Loop Vectorizer + adding new tail-folding mode using EVL. (PR #76172)

[Alexey Bataev via llvm-commits]

================
@@ -348,6 +348,44 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
     Value *Zero = ConstantInt::get(ScalarTC->getType(), 0);
     return Builder.CreateSelect(Cmp, Sub, Zero);
   }
+  case VPInstruction::ExplicitVectorLength: {
+    // Compute EVL
+    auto GetSetVL = [=](VPTransformState &State, Value *EVL) {
+      assert(EVL->getType()->isIntegerTy() &&
+             "Requested vector length should be an integer.");
+
+      // TODO: Add support for MaxSafeDist for correct loop emission.
+      Value *VFArg = State.Builder.getInt32(State.VF.getKnownMinValue());
+
+      Value *GVL = State.Builder.CreateIntrinsic(
+          State.Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length,
+          {EVL, VFArg, State.Builder.getTrue()});
+      return GVL;
+    };
+    // TODO: Restructure this code with an explicit remainder loop, vsetvli can
+    // be outside of the main loop.
+    assert(State.UF == 1 &&
+           "No unrolling expected for predicated vectorization.");
+    // Compute VTC - IV as the EVL(requested vector length).
+    Value *Index = State.get(getOperand(0), 0);
+    Value *TripCount = State.get(getOperand(1), VPIteration(0, 0));
+    Value *EVL = State.Builder.CreateSub(TripCount, Index);
+    Value *SetVL = GetSetVL(State, EVL);
+    State.EVL = this;
+    return SetVL;
+  }
+  case VPInstruction::ExplicitVectorLengthIVIncrement: {
+    assert(State.UF == 1 && Part == 0 &&
+           "Expected unroll factor 1 for VP vectorization.");
+    Value *Phi = State.get(getOperand(0), 0);
+    Value *EVL = State.get(getOperand(1), 0);
+    assert(EVL->getType()->getScalarSizeInBits() <=
+               Phi->getType()->getScalarSizeInBits() &&
+           "EVL type must be smaller than Phi type.");
+    EVL = Builder.CreateIntCast(EVL, Phi->getType(), /*isSigned=*/false);
----------------
alexey-bataev wrote:

Still does not work:
opt: lib/Transforms/Vectorize/VPlan.cpp:290: llvm::Value *llvm::VPTransformState::get(llvm::VPValue *, unsigned int): Assertion `(isa<VPWidenIntOrFpInductionRecipe>(Def->getDefiningRecipe()) || isa<VPScalarIVStepsRecipe>(Def->getDefiningRecipe()) || isa<VPExpandSCEVRecipe>(Def->getDefiningRecipe())) && "unexpected recipe found to be invariant"' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.


https://github.com/llvm/llvm-project/pull/76172