[llvm] [LV] Rework legacy widen/replicate in tryToBuildVPlan (NFC) (PR #224569)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 08:54:36 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/224569
>From e8c7cec556634645f33163c586008bc2151e780b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 18 Sep 2026 09:46:24 +0100
Subject: [PATCH 1/4] [LV] Rework legacy widen/replicate in tryToBuildVPlan
(NFC)
The code has been rewritten for clarity:
- An RPOT traversal is unnecessary, and a depth-first will do.
- The list of recipes to be skipped continues to grow, and its utility
is in question. Replace it with a clean make_isa_range over
VPInstructions.
- Clarify that tryToCreateWidenNonPhiRecipe is the routine that produces
VPWidenIntOrFpInductionRecipes.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 54 ++++++++-----------
1 file changed, 21 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9cec144d57daa..e19ea2ea325d3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6637,12 +6637,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// ---------------------------------------------------------------------------
VPRecipeBuilder RecipeBuilder(*Plan, Legal, *CM, Builder);
- // Scan the body of the loop in a topological order to visit each basic block
- // after having visited its predecessor basic blocks.
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
- HeaderVPBB);
-
RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
Range.Start);
@@ -6656,50 +6650,44 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::makeCallWideningDecisions, *Plan, Range,
RecipeBuilder, CostCtx);
- // Now process all other blocks and instructions.
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
- // Convert input VPInstructions to widened recipes.
- for (VPRecipeBase &R : make_early_inc_range(
- make_range(VPBB->getFirstNonPhi(), VPBB->end()))) {
- // Skip recipes that do not need transforming or have already been
- // transformed.
- if (isa<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
- VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
- VPWidenCallRecipe, VPWidenIntrinsicRecipe, VPVectorPointerRecipe,
- VPVectorEndPointerRecipe, VPHistogramRecipe>(&R) ||
- (Instruction::isCast(cast<VPInstruction>(R).getOpcode()) &&
- vputils::onlyFirstLaneUsed(R.getVPSingleValue())))
+ // Convert remaining VPInstructions to widen or replicate recipes.
+ // TODO: This legacy code should eventually be migrated to VPlan.
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(HeaderVPBB))) {
+ for (VPInstruction &VPI :
+ make_early_inc_range(make_isa_range<VPInstruction>(*VPBB))) {
+ // We represent single-scalar casts directly as VPInstructions.
+ if (Instruction::isCast(VPI.getOpcode()) &&
+ vputils::onlyFirstLaneUsed(&VPI))
continue;
- auto *VPI = cast<VPInstruction>(&R);
- if (!VPI->getUnderlyingValue())
+
+ Instruction *Instr = cast_or_null<Instruction>(VPI.getUnderlyingValue());
+ if (!Instr)
continue;
- // TODO: Gradually replace uses of underlying instruction by analyses on
- // VPlan. Migrate code relying on the underlying instruction from VPlan0
- // to construct recipes below to not use the underlying instruction.
- Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
- Builder.setInsertPoint(VPI);
+ Builder.setInsertPoint(&VPI);
VPRecipeBase *Recipe =
- RecipeBuilder.tryToCreateWidenNonPhiRecipe(VPI, Range);
- if (!Recipe)
- Recipe =
- RecipeBuilder.handleReplication(cast<VPInstruction>(VPI), Range);
+ RecipeBuilder.tryToCreateWidenNonPhiRecipe(&VPI, Range);
- if (isa<VPWidenIntOrFpInductionRecipe>(Recipe) && isa<TruncInst>(Instr)) {
+ if (isa_and_nonnull<VPWidenIntOrFpInductionRecipe>(Recipe) &&
+ isa<TruncInst>(Instr)) {
// Optimized a truncate to VPWidenIntOrFpInductionRecipe. It needs to be
// moved to the phi section in the header.
Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
} else {
+ if (!Recipe)
+ Recipe = RecipeBuilder.handleReplication(&VPI, Range);
Builder.insert(Recipe);
}
if (Recipe->getNumDefinedValues() == 1) {
- VPI->replaceAllUsesWith(Recipe->getVPSingleValue());
+ VPI.replaceAllUsesWith(Recipe->getVPSingleValue());
} else {
assert(Recipe->getNumDefinedValues() == 0 &&
"Unexpected multidef recipe");
}
- R.eraseFromParent();
+ VPI.eraseFromParent();
}
}
>From e559db8088bdc0a886975b2d9db661307c81726b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 18 Sep 2026 11:40:42 +0100
Subject: [PATCH 2/4] [LV] Clarify role of underlying value
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e19ea2ea325d3..31387599b0086 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6662,8 +6662,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
vputils::onlyFirstLaneUsed(&VPI))
continue;
- Instruction *Instr = cast_or_null<Instruction>(VPI.getUnderlyingValue());
- if (!Instr)
+ // We cannot handle VPInstructions without underlying values, as we would
+ // not be able to create a Replicate without it.
+ if (!VPI.getUnderlyingValue())
continue;
Builder.setInsertPoint(&VPI);
@@ -6672,7 +6673,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RecipeBuilder.tryToCreateWidenNonPhiRecipe(&VPI, Range);
if (isa_and_nonnull<VPWidenIntOrFpInductionRecipe>(Recipe) &&
- isa<TruncInst>(Instr)) {
+ VPI.getOpcode() == Instruction::Trunc) {
// Optimized a truncate to VPWidenIntOrFpInductionRecipe. It needs to be
// moved to the phi section in the header.
Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
>From e9ef5a50d2c068b03fcc09b357b1ffda1ae9ef74 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 21 Sep 2026 07:51:47 +0100
Subject: [PATCH 3/4] [LV] Restore assert
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 31387599b0086..a6015ddfd31e7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6655,6 +6655,15 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(HeaderVPBB))) {
+ assert(
+ all_of(
+ make_range(VPBB->getFirstNonPhi(), VPBB->end()),
+ IsaPred<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
+ VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
+ VPWidenCallRecipe, VPWidenIntrinsicRecipe,
+ VPVectorPointerRecipe, VPVectorEndPointerRecipe,
+ VPHistogramRecipe, VPInstruction>) &&
+ "Unexpected recipe");
for (VPInstruction &VPI :
make_early_inc_range(make_isa_range<VPInstruction>(*VPBB))) {
// We represent single-scalar casts directly as VPInstructions.
>From 44adcf8c7fe11dd980232da7b62878df00a92399 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 21 Sep 2026 21:40:29 +0100
Subject: [PATCH 4/4] [LV] Fix comment nits (NFC)
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a6015ddfd31e7..faebed45c584f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6655,6 +6655,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(HeaderVPBB))) {
+ // All types but VPInstructions are already widened and don't need extra
+ // processing. We process VPInstructions below.
assert(
all_of(
make_range(VPBB->getFirstNonPhi(), VPBB->end()),
@@ -6671,8 +6673,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
vputils::onlyFirstLaneUsed(&VPI))
continue;
- // We cannot handle VPInstructions without underlying values, as we would
- // not be able to create a Replicate without it.
+ // Only VPInstrutions with an underlying value need to be processed.
if (!VPI.getUnderlyingValue())
continue;
More information about the llvm-commits
mailing list