[llvm] [VPlan] Implement VPReductionRecipe::computeCost(). NFC (PR #107790)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 8 20:05:11 PDT 2024
https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/107790
>From 49aead52e78fb2b9521908f70c4bef81372c4a0b Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Wed, 4 Sep 2024 20:52:14 -0700
Subject: [PATCH 1/4] [VPlan] Implment VPReductionRecipe::computeCost(). NFC
Implementation of `computeCost()` function for `VPReductionRecipe`.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 24 +++++++++++++++++++
2 files changed, 28 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8c5246d613c13d..05d0da691354c0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2399,6 +2399,10 @@ class VPReductionRecipe : public VPSingleDefRecipe {
/// Generate the reduction in the loop
void execute(VPTransformState &State) override;
+ /// Return the cost of VPReductionRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9b1294f2c42822..ca04d6c9c74b2e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2012,6 +2012,30 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
State.set(this, NewRed, /*IsScalar*/ true);
}
+InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ RecurKind RdxKind = RdxDesc.getRecurrenceKind();
+ Type *ElementTy = RdxDesc.getRecurrenceType();
+ auto *VectorTy = dyn_cast<VectorType>(ToVectorTy(ElementTy, VF));
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ unsigned Opcode = RdxDesc.getOpcode();
+
+ if (VectorTy == nullptr)
+ return InstructionCost::getInvalid();
+
+ // Cost = Reduction cost + BinOp cost
+ InstructionCost Cost =
+ Ctx.TTI.getArithmeticInstrCost(Opcode, ElementTy, CostKind);
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind)) {
+ Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
+ return Cost + Ctx.TTI.getMinMaxReductionCost(
+ Id, VectorTy, RdxDesc.getFastMathFlags(), CostKind);
+ }
+
+ return Cost + Ctx.TTI.getArithmeticReductionCost(
+ Opcode, VectorTy, RdxDesc.getFastMathFlags(), CostKind);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPReductionRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
>From 86140e0ea1296d41cd16c95ba7ed43bce0ae2761 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 24 Sep 2024 23:31:24 -0700
Subject: [PATCH 2/4] Address comments.
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ca04d6c9c74b2e..6bd353b32bd120 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2016,13 +2016,10 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
RecurKind RdxKind = RdxDesc.getRecurrenceKind();
Type *ElementTy = RdxDesc.getRecurrenceType();
- auto *VectorTy = dyn_cast<VectorType>(ToVectorTy(ElementTy, VF));
+ auto *VectorTy = cast<VectorType>(ToVectorTy(ElementTy, VF));
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
unsigned Opcode = RdxDesc.getOpcode();
- if (VectorTy == nullptr)
- return InstructionCost::getInvalid();
-
// Cost = Reduction cost + BinOp cost
InstructionCost Cost =
Ctx.TTI.getArithmeticInstrCost(Opcode, ElementTy, CostKind);
>From f15462230caeab6750536dd55208dc9135bfb38c Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Fri, 4 Oct 2024 03:19:36 -0700
Subject: [PATCH 3/4] Address comments.
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 6bd353b32bd120..cc0231ce766128 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2015,7 +2015,14 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
RecurKind RdxKind = RdxDesc.getRecurrenceKind();
- Type *ElementTy = RdxDesc.getRecurrenceType();
+ // TODO: Support any-of reduction and in-loop reduction
+ assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) &&
+ "Not support any-of reduction in VPlan-based cost model currently.");
+
+ Type *ElementTy = Ctx.Types.inferScalarType(this->getVPSingleValue());
+ assert(ElementTy->getTypeID() == RdxDesc.getRecurrenceType()->getTypeID() &&
+ "Infered type and recurrence type mismatch.");
+
auto *VectorTy = cast<VectorType>(ToVectorTy(ElementTy, VF));
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
unsigned Opcode = RdxDesc.getOpcode();
>From 20e90e70d0dd2bc502b0936f420a70b6c83de754 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 8 Oct 2024 20:03:47 -0700
Subject: [PATCH 4/4] Address comments and add `isInLoopReduction()`.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +++++++
llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 22 ++++++++++++-------
3 files changed, 26 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 001c8987667df8..866568cb9b2a8a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7103,6 +7103,14 @@ bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
SkipCostComputation.contains(UI);
}
+bool VPCostContext::isInLoopReduction(const Instruction *UI, ElementCount VF,
+ Type *VectorTy) const {
+ return CM
+ .getReductionPatternCost(const_cast<Instruction *>(UI), VF, VectorTy,
+ TTI::TCK_RecipThroughput)
+ .has_value();
+}
+
InstructionCost
LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
VPCostContext &CostCtx) const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 05d0da691354c0..41ae3a184091a4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -718,6 +718,10 @@ struct VPCostContext {
/// Return true if the cost for \p UI shouldn't be computed, e.g. because it
/// has already been pre-computed.
bool skipCostComputation(Instruction *UI, bool IsVector) const;
+
+ /// Return true if the \p UI is part of the in-loop reduction.
+ bool isInLoopReduction(const Instruction *UI, ElementCount VF,
+ Type *VectorTy) const;
};
/// VPRecipeBase is a base class modeling a sequence of one or more output IR
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cc0231ce766128..8217b8cda5118b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2015,18 +2015,24 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
RecurKind RdxKind = RdxDesc.getRecurrenceKind();
- // TODO: Support any-of reduction and in-loop reduction
- assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) &&
- "Not support any-of reduction in VPlan-based cost model currently.");
-
- Type *ElementTy = Ctx.Types.inferScalarType(this->getVPSingleValue());
- assert(ElementTy->getTypeID() == RdxDesc.getRecurrenceType()->getTypeID() &&
- "Infered type and recurrence type mismatch.");
-
+ Type *ElementTy = Ctx.Types.inferScalarType(this);
auto *VectorTy = cast<VectorType>(ToVectorTy(ElementTy, VF));
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
unsigned Opcode = RdxDesc.getOpcode();
+ // TODO: Support any-of reduction and in-loop reductions.
+ assert(
+ (!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) ||
+ ForceTargetInstructionCost.getNumOccurrences() > 0) &&
+ "Any-of reduction not implemented in VPlan-based cost model currently.");
+ assert(
+ (!Ctx.isInLoopReduction(getUnderlyingInstr(), VF, VectorTy) ||
+ ForceTargetInstructionCost.getNumOccurrences() > 0) &&
+ "In-loop reduction not implemented in VPlan-based cost model currently.");
+
+ assert(ElementTy->getTypeID() == RdxDesc.getRecurrenceType()->getTypeID() &&
+ "Infered type and recurrence type mismatch.");
+
// Cost = Reduction cost + BinOp cost
InstructionCost Cost =
Ctx.TTI.getArithmeticInstrCost(Opcode, ElementTy, CostKind);
More information about the llvm-commits
mailing list