[llvm] [VPlan] Add VPBundleRecipe, replacing extended reduction recipes. (PR #144281)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 26 07:12:40 PDT 2025
================
@@ -2506,30 +2500,193 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
Ctx.CostKind);
}
-InstructionCost
-VPExtendedReductionRecipe::computeCost(ElementCount VF,
- VPCostContext &Ctx) const {
- unsigned Opcode = RecurrenceDescriptor::getOpcode(getRecurrenceKind());
- Type *RedTy = Ctx.Types.inferScalarType(this);
- auto *SrcVecTy =
- cast<VectorType>(toVectorTy(Ctx.Types.inferScalarType(getVecOp()), VF));
- assert(RedTy->isIntegerTy() &&
- "ExtendedReduction only support integer type currently.");
- return Ctx.TTI.getExtendedReductionCost(Opcode, isZExt(), RedTy, SrcVecTy,
- std::nullopt, Ctx.CostKind);
+void VPBundleRecipe::bundle(ArrayRef<VPValue *> Operands) {
+ assert(!BundledRecipes.empty() && "Nothing to bundle?");
+
+ // Bundle up the operand recipes.
+ SmallPtrSet<VPUser *, 4> BundledUsers;
+ for (auto *R : BundledRecipes)
+ BundledUsers.insert(R);
+
+ // Recipes in the bundle, except the last one, must only be used inside the
+ // bundle. If there other external users, clone the recipes for the bundle.
+ for (unsigned Idx = 0; Idx != BundledRecipes.size() - 1; ++Idx) {
+ VPSingleDefRecipe *R = BundledRecipes[Idx];
+ if (all_of(R->users(), [&BundledUsers](VPUser *U) {
+ return BundledUsers.contains(U);
+ })) {
+ if (R->getParent())
+ R->removeFromParent();
+ continue;
+ }
+ // The users external to the bundle. Clone the recipe for use in the
+ // bundle and update all its in-bundle users.
+ VPSingleDefRecipe *Copy = R->clone();
+ BundledRecipes[Idx] = Copy;
+ BundledUsers.insert(Copy);
+ R->replaceUsesWithIf(Copy, [&BundledUsers](VPUser &U, unsigned) {
+ return BundledUsers.contains(&U);
+ });
+ }
+ if (BundledRecipes.back()->getParent())
+ BundledRecipes.back()->removeFromParent();
+
+ // Internalize all external operands to the bundled operations. To do so,
+ // create new temporary VPValues for all operands not defined by recipe in
+ // the bundle. The original operands are added as operands of the
+ // VPBundleRecipe.
+ for (auto *R : BundledRecipes) {
+ for (const auto &[Idx, Op] : enumerate(R->operands())) {
+ auto *Def = Op->getDefiningRecipe();
+ if (Def && BundledUsers.contains(Def))
+ continue;
+ if (Operands.empty())
+ addOperand(Op);
+ else
----------------
fhahn wrote:
This was needed for cloning, remoed the operand and handled instead by updating the placeholder values with the original operands for the cloned recipes.
https://github.com/llvm/llvm-project/pull/144281
More information about the llvm-commits
mailing list