[llvm] [NFC][VPlan] Split `makeMemOpWideningDecisions` into subpasses (PR #182593)
Andrei Elovikov via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 10:44:39 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/182593
>From 54f9f2f56f1fe6642871fb21754ff09df6d5ec8a Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 19 Feb 2026 11:28:54 -0800
Subject: [PATCH] [NFC][VPlan] Split `makeMemOpWideningDecisions` into
subpasses
The idea is to have handling of strided memory operations (either from
https://github.com/llvm/llvm-project/pull/147297 or for VPlan-based
multiversioning for unit-strided accesses) done after some mandatory
processing has been performed (e.g., some types **must** be scalarized)
but before legacy CM's decision to widen (gather/scatter) or scalarize
has been committed.
And in longer term, we can uplift all other memory widening decision to
be done here directly at VPlan level. I expect this structure would also
be beneficial for that.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 10 --
.../Transforms/Vectorize/VPlanTransforms.cpp | 92 ++++++++++++++-----
.../Transforms/Vectorize/VPlanTransforms.h | 10 ++
.../VPlan/vplan-print-after-all.ll | 3 +
4 files changed, 80 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ee90c09085c3f..e3bd19ecbceb2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -378,16 +378,6 @@ static cl::opt<bool> EnableEarlyExitVectorization(
// after prolog. See `emitIterationCountCheck`.
static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
-/// A helper function that returns true if the given type is irregular. The
-/// type is irregular if its allocated size doesn't equal the store size of an
-/// element of the corresponding vector type.
-static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
- // Determine if an array of N elements of type Ty is "bitcast compatible"
- // with a <N x Ty> vector.
- // This is only true if there is no padding between the array elements.
- return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
-}
-
/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
/// ElementCount to include loops whose trip count is a function of vscale.
static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3dadd9253ad57..266cb706fbb2e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6473,38 +6473,80 @@ void VPlanTransforms::makeMemOpWideningDecisions(
}
}
- VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
+ // Few helpers to process different kinds of memory operations.
+
+ // To be used as argument to `VPlanTransforms::runPass` which explicitly
+ // specified pass name, hence `VPlan &` parameter.
+ auto ProcessSubset = [&](VPlan &, auto ProcessVPInst) {
+ SmallVector<VPInstruction *> RemainingMemOps;
+ for (VPInstruction *VPI : MemOps) {
+ if (!ProcessVPInst(VPI))
+ RemainingMemOps.push_back(VPI);
+ }
+
+ MemOps.clear();
+ std::swap(MemOps, RemainingMemOps);
+ };
+
+ auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
+ Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
+ New->insertBefore(VPI);
+ if (VPI->getOpcode() == Instruction::Load)
+ VPI->replaceAllUsesWith(New->getVPSingleValue());
+ VPI->eraseFromParent();
+
+ // VPI has been processed.
+ return true;
+ };
+
+ auto Scalarize = [&](VPInstruction *VPI) {
+ return ReplaceWith(VPI, RecipeBuilder.handleReplication(VPI, Range));
+ };
+
+ auto *MiddleVPBB = Plan.getMiddleBlock();
VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
+ VPlanTransforms::runPass(
+ "lowerMemoryIdioms", ProcessSubset, Plan, [&](VPInstruction *VPI) {
+ if (RecipeBuilder.replaceWithFinalIfReductionStore(
+ VPI, FinalRedStoresBuilder))
+ return true;
- for (VPInstruction *VPI : MemOps) {
- auto ReplaceWith = [&](VPRecipeBase *New) {
- New->insertBefore(VPI);
- if (VPI->getOpcode() == Instruction::Load)
- VPI->replaceAllUsesWith(New->getVPSingleValue());
- VPI->eraseFromParent();
- };
+ // Filter out scalar VPlan for the remaining idioms.
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [](ElementCount VF) { return VF.isScalar(); }, Range))
+ return false;
- // Note: we must do that for scalar VPlan as well.
- if (RecipeBuilder.replaceWithFinalIfReductionStore(VPI,
- FinalRedStoresBuilder))
- continue;
+ if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
+ return ReplaceWith(VPI, Histogram);
- // Filter out scalar VPlan for the remaining memory operations.
- if (LoopVectorizationPlanner::getDecisionAndClampRange(
- [](ElementCount VF) { return VF.isScalar(); }, Range))
- continue;
+ return false;
+ });
- if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI)) {
- ReplaceWith(Histogram);
- continue;
- }
+ // Filter out scalar VPlan for the remaining memory operations.
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [](ElementCount VF) { return VF.isScalar(); }, Range))
+ return;
- VPRecipeBase *Recipe = RecipeBuilder.tryToWidenMemory(VPI, Range);
- if (!Recipe)
- Recipe = RecipeBuilder.handleReplication(VPI, Range);
+ // If the instruction's allocated size doesn't equal it's type size, it
+ // requires padding and will be scalarized.
+ VPlanTransforms::runPass(
+ "scalarizeMemOpsWithIrregularTypes", ProcessSubset, Plan,
+ [&](VPInstruction *VPI) {
+ Instruction *I = VPI->getUnderlyingInstr();
+ if (hasIrregularType(getLoadStoreType(I), I->getDataLayout()))
+ return Scalarize(VPI);
- ReplaceWith(Recipe);
- }
+ return false;
+ });
+
+ VPlanTransforms::runPass("delegateMemOpWideningToLegacyCM", ProcessSubset,
+ Plan, [&](VPInstruction *VPI) {
+ if (VPRecipeBase *Recipe =
+ RecipeBuilder.tryToWidenMemory(VPI, Range))
+ return ReplaceWith(VPI, Recipe);
+
+ return Scalarize(VPI);
+ });
}
void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index d0ce6ce71d80c..afcfb76d63ebe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -546,6 +546,16 @@ struct VPlanTransforms {
static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range);
};
+/// A helper function that returns true if the given type is irregular. The
+/// type is irregular if its allocated size doesn't equal the store size of an
+/// element of the corresponding vector type.
+inline bool hasIrregularType(Type *Ty, const DataLayout &DL) {
+ // Determine if an array of N elements of type Ty is "bitcast compatible"
+ // with a <N x Ty> vector.
+ // This is only true if there is no padding between the array elements.
+ return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
+}
+
} // namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 1fb23a06401d4..57825e52d31a4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -12,6 +12,9 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createLoopRegions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::introduceMasksAndLinearize
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInLoopReductionRecipes
+; CHECK: VPlan for loop in 'foo' after lowerMemoryIdioms
+; CHECK: VPlan for loop in 'foo' after scalarizeMemOpsWithIrregularTypes
+; CHECK: VPlan for loop in 'foo' after delegateMemOpWideningToLegacyCM
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeMemOpWideningDecisions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeScalarizationDecisions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers
More information about the llvm-commits
mailing list