[llvm] 60ed236 - [LV][EVL]Explicitly model AVL as sub, original TC, EVL_PHI.
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 25 05:58:33 PDT 2024
Author: Alexey Bataev
Date: 2024-09-25T08:58:29-04:00
New Revision: 60ed2361c0917b4f8d54cb85935cfbf8904aa51d
URL: https://github.com/llvm/llvm-project/commit/60ed2361c0917b4f8d54cb85935cfbf8904aa51d
DIFF: https://github.com/llvm/llvm-project/commit/60ed2361c0917b4f8d54cb85935cfbf8904aa51d.diff
LOG: [LV][EVL]Explicitly model AVL as sub, original TC, EVL_PHI.
Patch explicitly models AVL as sub original TC, EVL_PHI instead of
having it in EXPLICIT-VECTOR-LENGTH VPInstruction. Required for correct
safe dependence distance suport.
Reviewers: fhahn, ayalz
Reviewed By: ayalz
Pull Request: https://github.com/llvm/llvm-project/pull/108869
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f33293e65010f9..3f5b73d2d43c38 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -477,27 +477,19 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateSelect(Cmp, Sub, Zero);
}
case VPInstruction::ExplicitVectorLength: {
- // Compute EVL
- auto GetEVL = [=](VPTransformState &State, Value *AVL) {
- assert(AVL->getType()->isIntegerTy() &&
- "Requested vector length should be an integer.");
-
- // TODO: Add support for MaxSafeDist for correct loop emission.
- assert(State.VF.isScalable() && "Expected scalable vector factor.");
- Value *VFArg = State.Builder.getInt32(State.VF.getKnownMinValue());
-
- Value *EVL = State.Builder.CreateIntrinsic(
- State.Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length,
- {AVL, VFArg, State.Builder.getTrue()});
- return EVL;
- };
// TODO: Restructure this code with an explicit remainder loop, vsetvli can
// be outside of the main loop.
- // Compute VTC - IV as the AVL (requested vector length).
- Value *Index = State.get(getOperand(0), VPIteration(0, 0));
- Value *TripCount = State.get(getOperand(1), VPIteration(0, 0));
- Value *AVL = State.Builder.CreateSub(TripCount, Index);
- Value *EVL = GetEVL(State, AVL);
+ Value *AVL = State.get(getOperand(0), VPIteration(0, 0));
+ // Compute EVL
+ assert(AVL->getType()->isIntegerTy() &&
+ "Requested vector length should be an integer.");
+
+ assert(State.VF.isScalable() && "Expected scalable vector factor.");
+ Value *VFArg = State.Builder.getInt32(State.VF.getKnownMinValue());
+
+ Value *EVL = State.Builder.CreateIntrinsic(
+ State.Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length,
+ {AVL, VFArg, State.Builder.getTrue()});
return EVL;
}
case VPInstruction::CanonicalIVIncrementForPart: {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3b37a1ec9560ee..6872cc535a10bf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1423,7 +1423,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
/// ...
/// %EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ %StartV, %vector.ph ],
/// [ %NextEVLIV, %vector.body ]
-/// %VPEVL = EXPLICIT-VECTOR-LENGTH %EVLPhi, original TC
+/// %AVL = sub original TC, %EVLPhi
+/// %VPEVL = EXPLICIT-VECTOR-LENGTH %AVL
/// ...
/// %NextEVLIV = add IVSize (cast i32 %VPEVVL to IVSize), %EVLPhi
/// ...
@@ -1453,9 +1454,15 @@ bool VPlanTransforms::tryAddExplicitVectorLength(VPlan &Plan) {
// Create the ExplicitVectorLengthPhi recipe in the main loop.
auto *EVLPhi = new VPEVLBasedIVPHIRecipe(StartV, DebugLoc());
EVLPhi->insertAfter(CanonicalIVPHI);
- auto *VPEVL = new VPInstruction(VPInstruction::ExplicitVectorLength,
- {EVLPhi, Plan.getTripCount()});
- VPEVL->insertBefore(*Header, Header->getFirstNonPhi());
+ // TODO: Add support for MaxSafeDist for correct loop emission.
+ // Compute original TC - IV as the AVL (application vector length).
+ auto *AVL = new VPInstruction(
+ Instruction::Sub, {Plan.getTripCount(), EVLPhi},
+ DebugLoc(), "avl");
+ AVL->insertBefore(*Header, Header->getFirstNonPhi());
+ auto *VPEVL =
+ new VPInstruction(VPInstruction::ExplicitVectorLength, AVL, DebugLoc());
+ VPEVL->insertAfter(AVL);
auto *CanonicalIVIncrement =
cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index 11405a1c911584..90c209cf3f5186 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -39,7 +39,8 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
; IF-EVL-INLOOP-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%[0-9]+]]>
; IF-EVL-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi ir<%start>, ir<[[RDX_NEXT:%.+]]>
-; IF-EVL-INLOOP-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[EVL_PHI]]>, ir<%n>
+; IF-EVL-INLOOP-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%n>, vp<[[EVL_PHI]]>
+; IF-EVL-INLOOP-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; IF-EVL-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
; IF-EVL-INLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
index 6dfe5b608199b3..c14a8bce8f48d8 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -23,7 +23,8 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: vector.body:
; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%[0-9]+]]>
-; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[EVL_PHI]]>, ir<%N>
+; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
+; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
More information about the llvm-commits
mailing list