[llvm] [VPlan] Add VPExpressionRecipe, replacing extended reduction recipes. (PR #144281)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 1 13:34:05 PDT 2025
================
@@ -2930,6 +2719,122 @@ class VPBranchOnMaskRecipe : public VPRecipeBase {
}
};
+/// A recipe to combine multiple recipes into a single 'expression' recipe,
+/// which should be considered a single entity for cost-modeling and transforms.
+/// The recipe needs to be 'decomposed', i.e. replaced by its individual
+/// expression recipes, before execute. The individual expression recipes are
+/// completely disconnected from the def-use graph of other recipes not part of
+/// the expression. Def-use edges between pairs of expression recipes remain
+/// intact, whereas every edge between an expression recipe and a recipe outside
+/// the expression is elevated to connect the non-expression recipe with the
+/// VPExpressionRecipe itself.
+class VPExpressionRecipe : public VPSingleDefRecipe {
+ /// Recipes included in this VPExpressionRecipe.
+ SmallVector<VPSingleDefRecipe *> ExpressionRecipes;
+
+ /// Temporary VPValues used for external operands of the expression, i.e.
+ /// operands not defined by recipes in the expression.
+ SmallVector<VPValue *> LiveInPlaceholders;
+
+ enum class ExpressionTypes {
+ /// Represents an inloop extended reduction operation, performing a
+ /// reduction on an extended vector operand into a scalar value, and adding
+ /// the result to a chain.
+ ExtendedReduction,
+ /// Represent an inloop multiply-accumulate reduction, multiplying the
+ /// extended vector operands, performing a reduction.add on the result, and
+ /// adding the scalar result to a chain.
+ ExtMulAccReduction,
+ /// Represent an inloop multiply-accumulate reduction, multiplying the
+ /// vector operands, performing a reduction.add on the result, and adding
+ /// the scalar result to a chain.
+ MulAccReduction,
+ };
+
+ /// Type of the expression.
+ ExpressionTypes ExpressionType;
+
+ /// Construct a new VPExpressionRecipe by internalizing recipes in \p
+ /// ExpressionRecipes. External operands (i.e. not defined by another recipe
+ /// in the expression) are replaced by temporary VPValues and the original
+ /// operands are transferred to the VPExpressionRecipe itself. Clone recipes
+ /// as needed (excluding last) to ensure they are only used by other recipes
+ /// in the expression.
+ VPExpressionRecipe(ExpressionTypes ExpressionType,
+ ArrayRef<VPSingleDefRecipe *> ExpressionRecipes);
+
+public:
+ VPExpressionRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red)
+ : VPExpressionRecipe(ExpressionTypes::ExtendedReduction, {Ext, Red}) {}
+ VPExpressionRecipe(VPWidenRecipe *Mul, VPReductionRecipe *Red)
+ : VPExpressionRecipe(ExpressionTypes::MulAccReduction, {Mul, Red}) {}
+ VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
+ VPWidenRecipe *Mul, VPReductionRecipe *Red)
+ : VPExpressionRecipe(ExpressionTypes::ExtMulAccReduction,
+ {Ext0, Ext1, Mul, Red}) {}
+
+ ~VPExpressionRecipe() override {
+ for (auto *R : reverse(ExpressionRecipes))
+ delete R;
+ for (VPValue *T : LiveInPlaceholders)
+ delete T;
+ }
+
+ VP_CLASSOF_IMPL(VPDef::VPExpressionSC)
+
+ VPExpressionRecipe *clone() override {
+ assert(!ExpressionRecipes.empty() && "empty expressions should be removed");
+ SmallVector<VPSingleDefRecipe *> NewExpressiondRecipes;
+ for (auto *R : ExpressionRecipes)
+ NewExpressiondRecipes.push_back(R->clone());
+ for (auto *New : NewExpressiondRecipes) {
+ for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
+ New->replaceUsesOfWith(Old, NewExpressiondRecipes[Idx]);
+ // Update placeholder operands in the cloned recipe to use the external
+ // operands, to be internalized when the cloned expression is constructed.
+ for (const auto &[Placeholder, OutsideOp] :
+ zip(LiveInPlaceholders, operands()))
+ New->replaceUsesOfWith(Placeholder, OutsideOp);
+ }
+ return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
+ }
+
+ /// Return the VPValue to use to infer the result type of the recipe.
+ VPValue *getOperandOfResultType() const {
+ unsigned OpIdx =
+ cast<VPReductionRecipe>(ExpressionRecipes.back())->isConditional() ? 2
+ : 1;
+ return getOperand(getNumOperands() - OpIdx);
+ }
+
+ /// Insert the recipes of the expression back into the VPlan, directly before
+ /// the current recipe. Leaves the expression recipe empty, which must be
+ /// removed before codegen.
+ void decompose();
----------------
ayalz wrote:
Another name may be `dissolve` - as in `dissolveLoopRegions()`, also conveying that they should then be removed.
Compare with `expand` as in `expandVPWidenIntOrFpInduction()` which also takes place during `convertToConcreteRecipes()` but involves placing new recipes in several distinct places (i.e., is not an expression, and the function does more than dissolve a recipe or region in place), and `materialize` as in `materializeBroadcasts()` which introduces new recipes rather than replace abstract ones with concrete ones (`concretize()` them?).
https://github.com/llvm/llvm-project/pull/144281
More information about the llvm-commits
mailing list