[llvm] [VPlan][NFC] Add cost for `VPWidenMemoryRecipe`. (PR #105614)

Elvis Wang via llvm-commits llvm-commits at lists.llvm.org
Sun Sep 1 17:08:30 PDT 2024


https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/105614

>From d72e70caf3aee9abfad626ca84b1f5b0406a7d6d Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Wed, 21 Aug 2024 20:39:31 -0700
Subject: [PATCH 1/4] [VPlan] Add cost for `VPWidenMemoryRecipe`

In this patch, we add the `computeCost()` function for `VPWidenMemoryRecipe`.
---
 llvm/lib/Transforms/Vectorize/VPlan.h         |  7 +++++
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 31 +++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 36a1aa08654d5b..b3edf7c4b51685 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2535,6 +2535,13 @@ class VPWidenMemoryRecipe : public VPRecipeBase {
     llvm_unreachable("VPWidenMemoryRecipe should not be instantiated.");
   }
 
+  /// Get element Type
+  Type *getElementType() const { return getLoadStoreType(&Ingredient); }
+
+  /// Return the cost of this VPWidenMemoryRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
   Instruction &getIngredient() const { return Ingredient; }
 };
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 53b28a692059f6..17125bb88b958f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2079,6 +2079,37 @@ void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
+InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
+                                                 VPCostContext &Ctx) const {
+  Instruction *I = getInstructionForCost(this);
+  Type *Ty = ToVectorTy(getElementType(), VF);
+  const Align Alignment = getLoadStoreAlignment(const_cast<Instruction *>(I));
+  const Value *Ptr = getLoadStorePointerOperand(I);
+  unsigned AS = getLoadStoreAddressSpace(const_cast<Instruction *>(I));
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+
+  if (Consecutive) {
+    InstructionCost Cost = 0;
+    if (IsMasked) {
+      Cost += Ctx.TTI.getMaskedMemoryOpCost(I->getOpcode(), Ty, Alignment, AS,
+                                            CostKind);
+    } else {
+      TTI::OperandValueInfo OpInfo = Ctx.TTI.getOperandInfo(I->getOperand(0));
+      Cost += Ctx.TTI.getMemoryOpCost(I->getOpcode(), Ty, Alignment, AS,
+                                      CostKind, OpInfo, I);
+    }
+    if (Reverse)
+      Cost += Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
+                                     cast<VectorType>(Ty), std::nullopt,
+                                     CostKind, 0);
+
+    return Cost;
+  }
+  return Ctx.TTI.getAddressComputationCost(Ty) +
+         Ctx.TTI.getGatherScatterOpCost(I->getOpcode(), Ty, Ptr, IsMasked,
+                                        Alignment, CostKind, I);
+}
+
 void VPWidenLoadRecipe::execute(VPTransformState &State) {
   auto *LI = cast<LoadInst>(&Ingredient);
 

>From 53972d570b474e84f7e60414e039ce0c6350d580 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 26 Aug 2024 06:10:41 -0700
Subject: [PATCH 2/4] Address comments

---
 llvm/lib/Transforms/Vectorize/VPlan.h         |  3 -
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 55 +++++++++++--------
 2 files changed, 31 insertions(+), 27 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b3edf7c4b51685..743552ea1306f0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2535,9 +2535,6 @@ class VPWidenMemoryRecipe : public VPRecipeBase {
     llvm_unreachable("VPWidenMemoryRecipe should not be instantiated.");
   }
 
-  /// Get element Type
-  Type *getElementType() const { return getLoadStoreType(&Ingredient); }
-
   /// Return the cost of this VPWidenMemoryRecipe.
   InstructionCost computeCost(ElementCount VF,
                               VPCostContext &Ctx) const override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 17125bb88b958f..2d9db9db1765d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2081,33 +2081,40 @@ void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent,
 
 InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
                                                  VPCostContext &Ctx) const {
-  Instruction *I = getInstructionForCost(this);
-  Type *Ty = ToVectorTy(getElementType(), VF);
-  const Align Alignment = getLoadStoreAlignment(const_cast<Instruction *>(I));
-  const Value *Ptr = getLoadStorePointerOperand(I);
-  unsigned AS = getLoadStoreAddressSpace(const_cast<Instruction *>(I));
+  Type *Ty = ToVectorTy(getLoadStoreType(&Ingredient), VF);
+  const Align Alignment =
+      getLoadStoreAlignment(const_cast<Instruction *>(&Ingredient));
+  unsigned AS =
+      getLoadStoreAddressSpace(const_cast<Instruction *>(&Ingredient));
   TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
 
-  if (Consecutive) {
-    InstructionCost Cost = 0;
-    if (IsMasked) {
-      Cost += Ctx.TTI.getMaskedMemoryOpCost(I->getOpcode(), Ty, Alignment, AS,
-                                            CostKind);
-    } else {
-      TTI::OperandValueInfo OpInfo = Ctx.TTI.getOperandInfo(I->getOperand(0));
-      Cost += Ctx.TTI.getMemoryOpCost(I->getOpcode(), Ty, Alignment, AS,
-                                      CostKind, OpInfo, I);
-    }
-    if (Reverse)
-      Cost += Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
-                                     cast<VectorType>(Ty), std::nullopt,
-                                     CostKind, 0);
-
-    return Cost;
+  if (!Consecutive) {
+    // TODO: Using the original IR may not be accurate.
+    // Currently, ARM will use the underlying IR to calculate gather/scatter
+    // instruction cost.
+    const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+    return Ctx.TTI.getAddressComputationCost(Ty) +
+           Ctx.TTI.getGatherScatterOpCost(Ingredient.getOpcode(), Ty, Ptr,
+                                          IsMasked, Alignment, CostKind,
+                                          &Ingredient);
+  }
+
+  InstructionCost Cost = 0;
+  if (IsMasked) {
+    Cost += Ctx.TTI.getMaskedMemoryOpCost(Ingredient.getOpcode(), Ty, Alignment,
+                                          AS, CostKind);
+  } else {
+    TTI::OperandValueInfo OpInfo =
+        Ctx.TTI.getOperandInfo(Ingredient.getOperand(0));
+    Cost += Ctx.TTI.getMemoryOpCost(Ingredient.getOpcode(), Ty, Alignment, AS,
+                                    CostKind, OpInfo, &Ingredient);
   }
-  return Ctx.TTI.getAddressComputationCost(Ty) +
-         Ctx.TTI.getGatherScatterOpCost(I->getOpcode(), Ty, Ptr, IsMasked,
-                                        Alignment, CostKind, I);
+  if (Reverse)
+    Cost +=
+        Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
+                               cast<VectorType>(Ty), std::nullopt, CostKind, 0);
+
+  return Cost;
 }
 
 void VPWidenLoadRecipe::execute(VPTransformState &State) {

>From d1635d5fe72709fd2f6e496e6273bcd97b562f87 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 27 Aug 2024 01:56:18 -0700
Subject: [PATCH 3/4] Address comments

---
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2d9db9db1765d8..1a296acd278640 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2093,6 +2093,8 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
     // Currently, ARM will use the underlying IR to calculate gather/scatter
     // instruction cost.
     const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+    assert(!Reverse &&
+           "Inconsecutive memory access should not have the order.");
     return Ctx.TTI.getAddressComputationCost(Ty) +
            Ctx.TTI.getGatherScatterOpCost(Ingredient.getOpcode(), Ty, Ptr,
                                           IsMasked, Alignment, CostKind,
@@ -2109,12 +2111,12 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
     Cost += Ctx.TTI.getMemoryOpCost(Ingredient.getOpcode(), Ty, Alignment, AS,
                                     CostKind, OpInfo, &Ingredient);
   }
-  if (Reverse)
-    Cost +=
-        Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
-                               cast<VectorType>(Ty), std::nullopt, CostKind, 0);
+  if (!Reverse)
+    return Cost;
 
-  return Cost;
+  return Cost += Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
+                                        cast<VectorType>(Ty), std::nullopt,
+                                        CostKind, 0);
 }
 
 void VPWidenLoadRecipe::execute(VPTransformState &State) {

>From 8e7c55ce99148fbbf8b9dadd1e8a5555f47847f8 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 27 Aug 2024 22:42:54 -0700
Subject: [PATCH 4/4] Add TODO for getInstrForCost()

---
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1a296acd278640..412b2913dd0518 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -271,6 +271,12 @@ static Instruction *getInstructionForCost(const VPRecipeBase *R) {
     return dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
   if (auto *IG = dyn_cast<VPInterleaveRecipe>(R))
     return IG->getInsertPos();
+  // Currently the legacy cost model only calculates the instruction cost with
+  // underlying instruction. Removing the WidenMem here will prevent
+  // force-target-instruction-cost overwriting the cost of recipe with
+  // underlying instruction which is inconsistent with the legacy model.
+  // TODO: Remove WidenMem from this function when we don't need to compare to
+  // the latecy model.
   if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
     return &WidenMem->getIngredient();
   return nullptr;



More information about the llvm-commits mailing list