[llvm] [VPlan] Introduce vputils::getSingleScalarClone (PR #161667)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 6 02:05:54 PST 2025
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/161667
>From 747c1d4bae248a8669db20715b4779dbf476d02d Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Thu, 2 Oct 2025 14:21:01 +0100
Subject: [PATCH 1/2] [VPlan] Introduce vputils::getSingleScalarClone
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 26 ++++---------------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 22 ++++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 4 +++
3 files changed, 31 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 82bf79e7bd876..ef946cec4f872 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -212,19 +212,7 @@ static bool sinkScalarOperands(VPlan &Plan) {
if (NeedsDuplicating) {
if (ScalarVFOnly)
continue;
- VPSingleDefRecipe *Clone;
- if (auto *SinkCandidateRepR =
- dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
- // TODO: Handle converting to uniform recipes as separate transform,
- // then cloning should be sufficient here.
- Instruction *I = SinkCandidate->getUnderlyingInstr();
- Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true,
- nullptr /*Mask*/, *SinkCandidateRepR);
- // TODO: add ".cloned" suffix to name of Clone's VPValue.
- } else {
- Clone = SinkCandidate->clone();
- }
-
+ VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(SinkCandidate);
Clone->insertBefore(SinkCandidate);
SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) {
return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
@@ -688,8 +676,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
if (!vputils::isSingleScalar(Def) && !vputils::onlyFirstLaneUsed(Def))
continue;
- auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(),
- Def->operands(), /*IsUniform*/ true);
+ VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(Def);
Clone->insertAfter(Def);
Def->replaceAllUsesWith(Clone);
}
@@ -1399,9 +1386,8 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
auto *RepOrWidenR = cast<VPSingleDefRecipe>(&R);
if (RepR && isa<StoreInst>(RepR->getUnderlyingInstr()) &&
vputils::isSingleScalar(RepR->getOperand(1))) {
- auto *Clone = new VPReplicateRecipe(
- RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
- true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Metadata*/);
+ auto *Clone =
+ cast<VPReplicateRecipe>(vputils::getSingleScalarClone(RepOrWidenR));
Clone->insertBefore(RepOrWidenR);
unsigned ExtractOpc =
vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1))
@@ -1426,9 +1412,7 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
}))
continue;
- auto *Clone = new VPReplicateRecipe(RepOrWidenR->getUnderlyingInstr(),
- RepOrWidenR->operands(),
- true /*IsSingleScalar*/);
+ VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(RepOrWidenR);
Clone->insertBefore(RepOrWidenR);
RepOrWidenR->replaceAllUsesWith(Clone);
if (isDeadRecipe(*RepOrWidenR))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index c6380d30ab2e2..e86473e5501e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -351,6 +351,28 @@ vputils::getRecipesForUncountableExit(VPlan &Plan,
return UncountableCondition;
}
+VPSingleDefRecipe *vputils::getSingleScalarClone(VPSingleDefRecipe *R) {
+ // TODO: add ".cloned" suffix to name of Clone's VPValue.
+ return TypeSwitch<VPSingleDefRecipe *, VPSingleDefRecipe *>(R)
+ .Case<VPInstruction, VPWidenRecipe, VPWidenSelectRecipe,
+ VPWidenCallRecipe, VPReplicateRecipe>([](auto *I) {
+ return new VPReplicateRecipe(I->getUnderlyingInstr(), I->operands(),
+ /*IsSingleScalar*/ true,
+ /*Mask*/ nullptr,
+ /*Metadata*/ *I);
+ })
+ .Case<VPWidenGEPRecipe>([](auto *I) {
+ // WidenGEP does not have metadata.
+ return new VPReplicateRecipe(I->getUnderlyingInstr(), I->operands(),
+ /*IsSingleScalar*/ true, /*Mask*/ nullptr);
+ })
+ .Case<VPScalarIVStepsRecipe>([](auto *I) { return I->clone(); })
+ .Default([](auto *I) {
+ llvm_unreachable("Don't know how to convert to single-scalar");
+ return nullptr;
+ });
+}
+
bool VPBlockUtils::isHeader(const VPBlockBase *VPB,
const VPDominatorTree &VPDT) {
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index c21a0e70c1392..a4ff446b559b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -71,6 +71,10 @@ std::optional<VPValue *>
getRecipesForUncountableExit(VPlan &Plan,
SmallVectorImpl<VPRecipeBase *> &Recipes,
SmallVectorImpl<VPRecipeBase *> &GEPs);
+
+/// Returns a single-scalar version of \p R, creating a fresh single-scalar
+/// VPReplicateRecipe or just cloning the recipe.
+VPSingleDefRecipe *getSingleScalarClone(VPSingleDefRecipe *R);
} // namespace vputils
//===----------------------------------------------------------------------===//
>From 5500dcbc6e61d98d646c14dcd79a7f19bd6e9720 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Wed, 5 Nov 2025 12:42:01 +0000
Subject: [PATCH 2/2] [VPlan] Handle WidenCast, add asserts
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e86473e5501e5..17d3c99cc916d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -352,24 +352,32 @@ vputils::getRecipesForUncountableExit(VPlan &Plan,
}
VPSingleDefRecipe *vputils::getSingleScalarClone(VPSingleDefRecipe *R) {
- // TODO: add ".cloned" suffix to name of Clone's VPValue.
return TypeSwitch<VPSingleDefRecipe *, VPSingleDefRecipe *>(R)
.Case<VPInstruction, VPWidenRecipe, VPWidenSelectRecipe,
VPWidenCallRecipe, VPReplicateRecipe>([](auto *I) {
+ assert(I->getUnderlyingValue() &&
+ "Cannot narrow recipe without underlying value");
return new VPReplicateRecipe(I->getUnderlyingInstr(), I->operands(),
/*IsSingleScalar*/ true,
/*Mask*/ nullptr,
/*Metadata*/ *I);
})
.Case<VPWidenGEPRecipe>([](auto *I) {
+ assert(I->getUnderlyingValue() &&
+ "Cannot narrow recipe without underlying value");
// WidenGEP does not have metadata.
return new VPReplicateRecipe(I->getUnderlyingInstr(), I->operands(),
/*IsSingleScalar*/ true, /*Mask*/ nullptr);
})
- .Case<VPScalarIVStepsRecipe>([](auto *I) { return I->clone(); })
+ .Case<VPWidenCastRecipe>([](auto *I) {
+ return new VPInstructionWithType(I->getOpcode(), I->operands(),
+ I->getResultType(), I->getDebugLoc(),
+ /*Flags*/ *I, /*Metadata*/ *I);
+ })
.Default([](auto *I) {
- llvm_unreachable("Don't know how to convert to single-scalar");
- return nullptr;
+ assert(isa<VPScalarIVStepsRecipe>(I) &&
+ "Don't know how to convert to single-scalar");
+ return I->clone();
});
}
More information about the llvm-commits
mailing list