[llvm] [VPlan] Add VPSingleDefBundleRecipe, replacing extended reduction recipes. (PR #144281)

[via llvm-commits]

================
@@ -2930,6 +2719,122 @@ class VPBranchOnMaskRecipe : public VPRecipeBase {
   }
 };
 
+/// A recipe to combine multiple recipes into a 'bundle' recipe, which should be
+/// considered a single entity for cost-modeling and transforms. The recipe
+/// needs to be 'unbundled', i.e. replaced by its bundled recipes before
+/// execute. The bundled recipes are completely disconnected from the def-use
+/// graph of other, non-bundled recipes. Def-use edges between pairs of bundled
+/// recipes remain intact, whereas every edge between a bundled and a
+/// non-bundled recipe is elevated to connect the non-bundled recipe with the
+/// VPSingleDefBundleRecipe itself.
+class VPSingleDefBundleRecipe : public VPSingleDefRecipe {
+  /// Recipes bundled together in this VPSingleDefBundleRecipe.
+  SmallVector<VPSingleDefRecipe *> BundledRecipes;
+
+  /// Temporary VPValues used for external operands of the bundle, i.e. operands
+  /// not defined by recipes in the bundle.
+  SmallVector<VPValue *> BundleLiveInPlaceholders;
+
+  enum class BundleTypes {
+    /// Represents an inloop extended reduction operation, performing a
+    /// reduction on a extended vector operand into a scalar value, and adding
+    /// the result to a chain.
+    ExtendedReduction,
+    /// Represent an inloop multiply-accumulate reduction, multiplying the
+    /// extended vector operands, performing a reduction.add on the result, and
+    /// adding the scalar result to a chain.
+    ExtMulAccumulateReduction,
+    /// Represent an inloop multiply-accumulate reduction, multiplying the
+    /// vector operands, performing a reduction.add on the result, and adding
+    /// the scalar result to a chain.
+    MulAccumulateReduction,
+  };
+
+  /// Type of the bundle.
+  BundleTypes BundleType;
+
+  /// Construct a new VPSingleDefBundleRecipe by internalizing recipes in \p
+  /// BundledRecipes. External operands (i.e. not defined by another recipe in
+  /// the bundle) are replaced by temporary VPValues and the original operands
+  /// are transferred to the VPSingleDefBundleRecipe itself. Clone recipes as
+  /// needed (excluding last) to ensure they are only used by other recipes in
+  /// the bundle.
+  VPSingleDefBundleRecipe(BundleTypes BundleType,
+                          ArrayRef<VPSingleDefRecipe *> ToBundle);
+
+public:
+  VPSingleDefBundleRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red)
+      : VPSingleDefBundleRecipe(BundleTypes::ExtendedReduction, {Ext, Red}) {}
+  VPSingleDefBundleRecipe(VPWidenRecipe *Mul, VPReductionRecipe *Red)
+      : VPSingleDefBundleRecipe(BundleTypes::MulAccumulateReduction,
+                                {Mul, Red}) {}
+  VPSingleDefBundleRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
+                          VPWidenRecipe *Mul, VPReductionRecipe *Red)
+      : VPSingleDefBundleRecipe(BundleTypes::ExtMulAccumulateReduction,
+                                {Ext0, Ext1, Mul, Red}) {}
+
+  ~VPSingleDefBundleRecipe() override {
+    SmallPtrSet<VPRecipeBase *, 4> Seen;
+    for (auto *R : reverse(BundledRecipes))
+      if (Seen.insert(R).second)
+        delete R;
+    for (VPValue *T : BundleLiveInPlaceholders)
+      delete T;
+  }
+
+  VP_CLASSOF_IMPL(VPDef::VPBundleSC)
+
+  VPSingleDefBundleRecipe *clone() override {
+    assert(!BundledRecipes.empty() && "empty bundles should be removed");
+    SmallVector<VPSingleDefRecipe *> NewBundledRecipes;
+    for (auto *R : BundledRecipes)
+      NewBundledRecipes.push_back(R->clone());
+    for (auto *New : NewBundledRecipes) {
+      for (const auto &[Idx, Old] : enumerate(BundledRecipes))
+        New->replaceUsesOfWith(Old, NewBundledRecipes[Idx]);
+      // Update placeholder operands in the cloned recipe to use the external
+      // operands, to be internalized when the cloned bundle is constructed.
+      for (const auto &[Placeholder, OutsideOp] :
+           zip(BundleLiveInPlaceholders, operands()))
+        New->replaceUsesOfWith(Placeholder, OutsideOp);
+    }
+    return new VPSingleDefBundleRecipe(BundleType, NewBundledRecipes);
+  }
+
+  /// Return the VPValue to use to infer the result type of the recipe.
+  VPValue *getTypeVPValue() const {
+    unsigned OpIdx =
+        cast<VPReductionRecipe>(BundledRecipes.back())->isConditional() ? 2 : 1;
----------------
ayalz wrote:

Last operand unless it's a mask in which case penultimate operand? Should the bundled recipe itself support isConditional()?

https://github.com/llvm/llvm-project/pull/144281