[llvm] [NFCI][VPlan] Split initial mem-widening into a separate transformation (PR #182592)
Andrei Elovikov via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 09:35:06 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/182592
>From 23eecfbc792ad4d195652001200e471eae634ad7 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 17 Feb 2026 13:29:13 -0800
Subject: [PATCH 01/17] [NFCI][VPlan] Split initial mem-widening into a
separate transformation
Preparation change before implementing stride-multiversioning as a
VPlan-based transformation. Might help
https://github.com/llvm/llvm-project/pull/147297/ as well.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 122 +++++++++++++-----
.../Transforms/Vectorize/VPRecipeBuilder.h | 26 ++--
.../Transforms/Vectorize/VPlanTransforms.h | 4 +
.../AArch64/predication_costs.ll | 5 +-
.../VPlan/vplan-print-after-all.ll | 1 +
5 files changed, 114 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 98bb12e8e3670..4b4c094a1c708 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7603,13 +7603,9 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
return tryToWidenCall(VPI, Range);
Instruction *Instr = R->getUnderlyingInstr();
- if (VPI->getOpcode() == Instruction::Store)
- if (auto HistInfo = Legal->getHistogramInfo(cast<StoreInst>(Instr)))
- return tryToWidenHistogram(*HistInfo, VPI);
-
- if (VPI->getOpcode() == Instruction::Load ||
- VPI->getOpcode() == Instruction::Store)
- return tryToWidenMemory(VPI, Range);
+ assert(!is_contained({Instruction::Load, Instruction::Store},
+ VPI->getOpcode()) &&
+ "Should have been handled prior to this!");
if (!shouldWiden(Instr, Range))
return nullptr;
@@ -7797,8 +7793,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
HeaderVPBB);
- VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
-
// Collect blocks that need predication for in-loop reduction recipes.
DenseSet<BasicBlock *> BlocksNeedingPredication;
for (BasicBlock *BB : OrigLoop->blocks())
@@ -7808,13 +7802,23 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
VPlanTransforms::createInLoopReductionRecipes(*Plan, BlocksNeedingPredication,
Range.Start);
+ VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind, CM.PSE,
+ OrigLoop);
+
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeMemOpWideningDecisions, *Plan,
+ Range, RecipeBuilder, CostCtx);
+
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
// Convert input VPInstructions to widened recipes.
for (VPRecipeBase &R : make_early_inc_range(
make_range(VPBB->getFirstNonPhi(), VPBB->end()))) {
- // Skip recipes that do not need transforming.
- if (isa<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe>(&R))
+ // Skip recipes that do not need transforming or have already been
+ // transformed.
+ if (isa<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
+ VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
+ VPVectorPointerRecipe, VPVectorEndPointerRecipe,
+ VPHistogramRecipe>(&R))
continue;
auto *VPI = cast<VPInstruction>(&R);
if (!VPI->getUnderlyingValue())
@@ -7826,23 +7830,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
Builder.setInsertPoint(VPI);
- // The stores with invariant address inside the loop will be deleted, and
- // in the exit block, a uniform store recipe will be created for the final
- // invariant store of the reduction.
- StoreInst *SI;
- if ((SI = dyn_cast<StoreInst>(Instr)) &&
- Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
- // Only create recipe for the final invariant store of the reduction.
- if (Legal->isInvariantStoreOfReduction(SI)) {
- auto *Recipe = new VPReplicateRecipe(
- SI, VPI->operandsWithoutMask(), true /* IsUniform */,
- nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
- Recipe->insertBefore(*MiddleVPBB, MBIP);
- }
- R.eraseFromParent();
- continue;
- }
-
VPRecipeBase *Recipe =
RecipeBuilder.tryToCreateWidenNonPhiRecipe(VPI, Range);
if (!Recipe)
@@ -7909,8 +7896,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// TODO: Enable following transform when the EVL-version of extended-reduction
// and mulacc-reduction are implemented.
if (!CM.foldTailWithEVL()) {
- VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind, CM.PSE,
- OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
Range);
RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
@@ -9549,3 +9534,80 @@ void LoopVectorizePass::printPipeline(
OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
OS << '>';
}
+
+void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
+ VPRecipeBuilder &RecipeBuilder,
+ VPCostContext &CostCtx) {
+ // Filter out scalar VPlan.
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) { return VF.isScalar(); }, Range))
+ return;
+
+ // Scan the body of the loop in a topological order to visit each basic block
+ // after having visited its predecessor basic blocks.
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ HeaderVPBB);
+
+ // Collect all loads/stores first. We will start with ones having simpler
+ // decisions followed by more complex ones that are potentially
+ // guided/dependent on the simpler ones.
+ SmallVector<VPInstruction *> MemOps;
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+ for (VPRecipeBase &R : *VPBB) {
+ auto *VPI = dyn_cast<VPInstruction>(&R);
+ if (VPI && VPI->getUnderlyingValue() &&
+ is_contained({Instruction::Load, Instruction::Store},
+ VPI->getOpcode()))
+ MemOps.push_back(VPI);
+ }
+ }
+
+ auto *Legal = CostCtx.CM.Legal;
+
+ auto *MiddleVPBB = Plan.getMiddleBlock();
+ VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
+
+ for (VPInstruction *VPI : MemOps) {
+ Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
+ RecipeBuilder.getVPBuilder().setInsertPoint(VPI);
+
+ auto ReplaceWith = [&](VPRecipeBase *New) {
+ RecipeBuilder.setRecipe(Instr, New);
+ RecipeBuilder.getVPBuilder().insert(New);
+ if (VPI->getOpcode() == Instruction::Load)
+ VPI->replaceAllUsesWith(New->getVPSingleValue());
+ VPI->eraseFromParent();
+ };
+
+ // The stores with invariant address inside the loop will be deleted, and
+ // in the exit block, a uniform store recipe will be created for the final
+ // invariant store of the reduction.
+ StoreInst *SI;
+ if ((SI = dyn_cast<StoreInst>(Instr)) &&
+ Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
+ // Only create recipe for the final invariant store of the reduction.
+ if (Legal->isInvariantStoreOfReduction(SI)) {
+ auto *Recipe = new VPReplicateRecipe(
+ SI, VPI->operandsWithoutMask(), true /* IsUniform */,
+ nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
+ Recipe->insertBefore(*MiddleVPBB, MBIP);
+ }
+ VPI->eraseFromParent();
+ continue;
+ }
+
+ if (VPI->getOpcode() == Instruction::Store)
+ if (auto HistInfo = Legal->getHistogramInfo(cast<StoreInst>(Instr))) {
+ ReplaceWith(RecipeBuilder.tryToWidenHistogram(*HistInfo, VPI));
+ continue;
+ }
+
+ VPRecipeBase *Recipe = RecipeBuilder.tryToWidenMemory(VPI, Range);
+ if (!Recipe)
+ Recipe = RecipeBuilder.handleReplication(cast<VPInstruction>(VPI), Range);
+
+ ReplaceWith(Recipe);
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 64315df74dda5..0c261373e4e1b 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -52,11 +52,6 @@ class VPRecipeBuilder {
/// Range. The function should not be called for memory instructions or calls.
bool shouldWiden(Instruction *I, VFRange &Range) const;
- /// Check if the load or store instruction \p VPI should widened for \p
- /// Range.Start and potentially masked. Such instructions are handled by a
- /// recipe that takes an additional VPInstruction for the mask.
- VPRecipeBase *tryToWidenMemory(VPInstruction *VPI, VFRange &Range);
-
/// Optimize the special case where the operand of \p VPI is a constant
/// integer induction variable.
VPWidenIntOrFpInductionRecipe *
@@ -72,24 +67,31 @@ class VPRecipeBuilder {
/// cost-model indicates that widening should be performed.
VPWidenRecipe *tryToWiden(VPInstruction *VPI);
- /// Makes Histogram count operations safe for vectorization, by emitting a
- /// llvm.experimental.vector.histogram.add intrinsic in place of the
- /// Load + Add|Sub + Store operations that perform the histogram in the
- /// original scalar loop.
- VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
- VPInstruction *VPI);
-
public:
VPRecipeBuilder(VPlan &Plan, const TargetLibraryInfo *TLI,
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM, VPBuilder &Builder)
: Plan(Plan), TLI(TLI), Legal(Legal), CM(CM), Builder(Builder) {}
+ VPBuilder &getVPBuilder() const { return Builder; }
+
/// Create and return a widened recipe for a non-phi recipe \p R if one can be
/// created within the given VF \p Range.
VPRecipeBase *tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VFRange &Range);
+ /// Check if the load or store instruction \p VPI should widened for \p
+ /// Range.Start and potentially masked. Such instructions are handled by a
+ /// recipe that takes an additional VPInstruction for the mask.
+ VPRecipeBase *tryToWidenMemory(VPInstruction *VPI, VFRange &Range);
+
+ /// Makes Histogram count operations safe for vectorization, by emitting a
+ /// llvm.experimental.vector.histogram.add intrinsic in place of the
+ /// Load + Add|Sub + Store operations that perform the histogram in the
+ /// original scalar loop.
+ VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
+ VPInstruction *VPI);
+
/// Set the recipe created for given ingredient.
void setRecipe(Instruction *I, VPRecipeBase *R) {
assert(!Ingredient2Recipe.contains(I) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6312b823e5d33..206d32021405d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -534,6 +534,10 @@ struct VPlanTransforms {
/// are only valid for a subset of VFs in Range, Range.End is updated.
static void createPartialReductions(VPlan &Plan, VPCostContext &CostCtx,
VFRange &Range);
+
+ static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
+ VPRecipeBuilder &RecipeBuilder,
+ VPCostContext &CostCtx);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index b9b91be9b7a65..fdb22beb19695 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -179,8 +179,8 @@ for.end:
; Cost of store:
; store(4) / 2 = 2
;
-; CHECK: Scalarizing: %tmp2 = add nsw i32 %tmp1, %x
; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %tmp0, align 4
+; CHECK: Scalarizing: %tmp2 = add nsw i32 %tmp1, %x
; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp2 = add nsw i32 %tmp1, %x
;
@@ -229,10 +229,11 @@ for.end:
; store(4) / 2 = 2
;
; CHECK-NOT: Scalarizing: %tmp2 = add i32 %tmp1, %x
+; CHECK: Scalarizing and predicating: store i32 %tmp5, ptr %tmp0, align 4
+; CHECK-NOT: Scalarizing: %tmp2 = add i32 %tmp1, %x
; CHECK: Scalarizing and predicating: %tmp3 = sdiv i32 %tmp1, %tmp2
; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp3, %tmp2
; CHECK: Scalarizing: %tmp5 = sub i32 %tmp4, %x
-; CHECK: Scalarizing and predicating: store i32 %tmp5, ptr %tmp0, align 4
; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp5, ptr %tmp0, align 4
; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp5 = sub i32 %tmp4, %x
; CHECK: Cost of 1 for VF 2: WIDEN ir<%tmp2> = add ir<%tmp1>, ir<%x>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index a698636aa2349..ca13a229e34a7 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -6,6 +6,7 @@
; CHECK: VPlan for loop in 'foo' after printAfterInitialConstruction
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createLoopRegions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::introduceMasksAndLinearize
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeMemOpWideningDecisions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimizeFindIVReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleMultiUseReductions
>From 44baf4c66719ee59ff415e20143f0fd3a63c7bfa Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 23 Feb 2026 09:04:23 -0800
Subject: [PATCH 02/17] Don't make unnecessary captures
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4b4c094a1c708..7468dbf99ec6e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9540,7 +9540,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPCostContext &CostCtx) {
// Filter out scalar VPlan.
if (LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) { return VF.isScalar(); }, Range))
+ [](ElementCount VF) { return VF.isScalar(); }, Range))
return;
// Scan the body of the loop in a topological order to visit each basic block
>From 19ccdc422b10f25c7f8fa004c1b874843b91c754 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 23 Feb 2026 13:58:01 -0800
Subject: [PATCH 03/17] Move to VPlanTransforms, have to pass Legal explicitly
---
.../Transforms/Vectorize/LoopVectorize.cpp | 79 +------------------
.../Transforms/Vectorize/VPlanTransforms.cpp | 76 ++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 4 +-
3 files changed, 80 insertions(+), 79 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7468dbf99ec6e..c133f2b79a647 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7806,7 +7806,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
OrigLoop);
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeMemOpWideningDecisions, *Plan,
- Range, RecipeBuilder, CostCtx);
+ Range, RecipeBuilder, CostCtx, *CM.Legal);
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
@@ -9534,80 +9534,3 @@ void LoopVectorizePass::printPipeline(
OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
OS << '>';
}
-
-void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
- VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx) {
- // Filter out scalar VPlan.
- if (LoopVectorizationPlanner::getDecisionAndClampRange(
- [](ElementCount VF) { return VF.isScalar(); }, Range))
- return;
-
- // Scan the body of the loop in a topological order to visit each basic block
- // after having visited its predecessor basic blocks.
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
- HeaderVPBB);
-
- // Collect all loads/stores first. We will start with ones having simpler
- // decisions followed by more complex ones that are potentially
- // guided/dependent on the simpler ones.
- SmallVector<VPInstruction *> MemOps;
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
- for (VPRecipeBase &R : *VPBB) {
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (VPI && VPI->getUnderlyingValue() &&
- is_contained({Instruction::Load, Instruction::Store},
- VPI->getOpcode()))
- MemOps.push_back(VPI);
- }
- }
-
- auto *Legal = CostCtx.CM.Legal;
-
- auto *MiddleVPBB = Plan.getMiddleBlock();
- VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
-
- for (VPInstruction *VPI : MemOps) {
- Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
- RecipeBuilder.getVPBuilder().setInsertPoint(VPI);
-
- auto ReplaceWith = [&](VPRecipeBase *New) {
- RecipeBuilder.setRecipe(Instr, New);
- RecipeBuilder.getVPBuilder().insert(New);
- if (VPI->getOpcode() == Instruction::Load)
- VPI->replaceAllUsesWith(New->getVPSingleValue());
- VPI->eraseFromParent();
- };
-
- // The stores with invariant address inside the loop will be deleted, and
- // in the exit block, a uniform store recipe will be created for the final
- // invariant store of the reduction.
- StoreInst *SI;
- if ((SI = dyn_cast<StoreInst>(Instr)) &&
- Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
- // Only create recipe for the final invariant store of the reduction.
- if (Legal->isInvariantStoreOfReduction(SI)) {
- auto *Recipe = new VPReplicateRecipe(
- SI, VPI->operandsWithoutMask(), true /* IsUniform */,
- nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
- Recipe->insertBefore(*MiddleVPBB, MBIP);
- }
- VPI->eraseFromParent();
- continue;
- }
-
- if (VPI->getOpcode() == Instruction::Store)
- if (auto HistInfo = Legal->getHistogramInfo(cast<StoreInst>(Instr))) {
- ReplaceWith(RecipeBuilder.tryToWidenHistogram(*HistInfo, VPI));
- continue;
- }
-
- VPRecipeBase *Recipe = RecipeBuilder.tryToWidenMemory(VPI, Range);
- if (!Recipe)
- Recipe = RecipeBuilder.handleReplication(cast<VPInstruction>(VPI), Range);
-
- ReplaceWith(Recipe);
- }
-}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b2e8b6a85a35a..8428934061052 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -42,6 +42,7 @@
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
using namespace llvm;
using namespace VPlanPatternMatch;
@@ -6487,3 +6488,78 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
for (const VPPartialReductionChain &Chain : Chains)
transformToPartialReduction(Chain, CostCtx.Types, Plan, Phi);
}
+
+void VPlanTransforms::makeMemOpWideningDecisions(
+ VPlan &Plan, VFRange &Range, VPRecipeBuilder &RecipeBuilder,
+ VPCostContext &CostCtx, LoopVectorizationLegality &Legal) {
+ // Filter out scalar VPlan.
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [](ElementCount VF) { return VF.isScalar(); }, Range))
+ return;
+
+ // Scan the body of the loop in a topological order to visit each basic block
+ // after having visited its predecessor basic blocks.
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ HeaderVPBB);
+
+ // Collect all loads/stores first. We will start with ones having simpler
+ // decisions followed by more complex ones that are potentially
+ // guided/dependent on the simpler ones.
+ SmallVector<VPInstruction *> MemOps;
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+ for (VPRecipeBase &R : *VPBB) {
+ auto *VPI = dyn_cast<VPInstruction>(&R);
+ if (VPI && VPI->getUnderlyingValue() &&
+ is_contained({Instruction::Load, Instruction::Store},
+ VPI->getOpcode()))
+ MemOps.push_back(VPI);
+ }
+ }
+
+ auto *MiddleVPBB = Plan.getMiddleBlock();
+ VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
+
+ for (VPInstruction *VPI : MemOps) {
+ Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
+ RecipeBuilder.getVPBuilder().setInsertPoint(VPI);
+
+ auto ReplaceWith = [&](VPRecipeBase *New) {
+ RecipeBuilder.setRecipe(Instr, New);
+ RecipeBuilder.getVPBuilder().insert(New);
+ if (VPI->getOpcode() == Instruction::Load)
+ VPI->replaceAllUsesWith(New->getVPSingleValue());
+ VPI->eraseFromParent();
+ };
+
+ // The stores with invariant address inside the loop will be deleted, and
+ // in the exit block, a uniform store recipe will be created for the final
+ // invariant store of the reduction.
+ StoreInst *SI;
+ if ((SI = dyn_cast<StoreInst>(Instr)) &&
+ Legal.isInvariantAddressOfReduction(SI->getPointerOperand())) {
+ // Only create recipe for the final invariant store of the reduction.
+ if (Legal.isInvariantStoreOfReduction(SI)) {
+ auto *Recipe = new VPReplicateRecipe(
+ SI, VPI->operandsWithoutMask(), true /* IsUniform */,
+ nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
+ Recipe->insertBefore(*MiddleVPBB, MBIP);
+ }
+ VPI->eraseFromParent();
+ continue;
+ }
+
+ if (VPI->getOpcode() == Instruction::Store)
+ if (auto HistInfo = Legal.getHistogramInfo(cast<StoreInst>(Instr))) {
+ ReplaceWith(RecipeBuilder.tryToWidenHistogram(*HistInfo, VPI));
+ continue;
+ }
+
+ VPRecipeBase *Recipe = RecipeBuilder.tryToWidenMemory(VPI, Range);
+ if (!Recipe)
+ Recipe = RecipeBuilder.handleReplication(cast<VPInstruction>(VPI), Range);
+
+ ReplaceWith(Recipe);
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 206d32021405d..0ff509fa86f61 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -26,6 +26,7 @@ namespace llvm {
class InductionDescriptor;
class Instruction;
class Loop;
+class LoopVectorizationLegality;
class LoopVersioning;
class OptimizationRemarkEmitter;
class PHINode;
@@ -537,7 +538,8 @@ struct VPlanTransforms {
static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx);
+ VPCostContext &CostCtx,
+ LoopVectorizationLegality &Legal);
};
} // namespace llvm
>From 1e60fcd138f8b44248faafc007af3311db1931b4 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 23 Feb 2026 14:00:58 -0800
Subject: [PATCH 04/17] Braces for outer `if`
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8428934061052..8239539210ba0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6550,11 +6550,12 @@ void VPlanTransforms::makeMemOpWideningDecisions(
continue;
}
- if (VPI->getOpcode() == Instruction::Store)
+ if (VPI->getOpcode() == Instruction::Store) {
if (auto HistInfo = Legal.getHistogramInfo(cast<StoreInst>(Instr))) {
ReplaceWith(RecipeBuilder.tryToWidenHistogram(*HistInfo, VPI));
continue;
}
+ }
VPRecipeBase *Recipe = RecipeBuilder.tryToWidenMemory(VPI, Range);
if (!Recipe)
>From 989eef5da50c1bbf6481a1af2cd5c4bd9be0cf19 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 3 Mar 2026 13:42:26 -0800
Subject: [PATCH 05/17] Fold one `Legal` use into `tryToWidenHistogram` renamed
to `widenIfHistogram`
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 12 ++++++++++--
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h | 12 ++++++------
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 +++-----
3 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c133f2b79a647..11a04794ecffc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7496,8 +7496,16 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
};
}
-VPHistogramRecipe *VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
- VPInstruction *VPI) {
+VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
+ if (VPI->getOpcode() != Instruction::Store)
+ return nullptr;
+
+ auto HistInfo =
+ Legal->getHistogramInfo(cast<StoreInst>(VPI->getUnderlyingInstr()));
+ if (!HistInfo)
+ return nullptr;
+
+ const HistogramInfo *HI = *HistInfo;
// FIXME: Support other operations.
unsigned Opcode = HI->Update->getOpcode();
assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 0c261373e4e1b..080151e7cd2cf 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -85,12 +85,12 @@ class VPRecipeBuilder {
/// recipe that takes an additional VPInstruction for the mask.
VPRecipeBase *tryToWidenMemory(VPInstruction *VPI, VFRange &Range);
- /// Makes Histogram count operations safe for vectorization, by emitting a
- /// llvm.experimental.vector.histogram.add intrinsic in place of the
- /// Load + Add|Sub + Store operations that perform the histogram in the
- /// original scalar loop.
- VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
- VPInstruction *VPI);
+ /// If \p VPI represents a histogram operation (as determined by
+ /// LoopVectorizationLegality) make that safe for vectorization, by emitting a
+ /// llvm.experimental.vector.histogram.add intrinsic in place of the Load +
+ /// Add|Sub + Store operations that perform the histogram in the original
+ /// scalar loop.
+ VPHistogramRecipe *widenIfHistogram(VPInstruction *VPI);
/// Set the recipe created for given ingredient.
void setRecipe(Instruction *I, VPRecipeBase *R) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8239539210ba0..e50e76161abdd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6550,11 +6550,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(
continue;
}
- if (VPI->getOpcode() == Instruction::Store) {
- if (auto HistInfo = Legal.getHistogramInfo(cast<StoreInst>(Instr))) {
- ReplaceWith(RecipeBuilder.tryToWidenHistogram(*HistInfo, VPI));
- continue;
- }
+ if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI)) {
+ ReplaceWith(Histogram);
+ continue;
}
VPRecipeBase *Recipe = RecipeBuilder.tryToWidenMemory(VPI, Range);
>From 8531e7798aaed3e1e09a91021f5edc0a1f5ed680 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 3 Mar 2026 15:37:00 -0800
Subject: [PATCH 06/17] Move another `Legal` use to
`VPRecipeBuilder::replaceWithFinalIfReductionStore`
---
.../Transforms/Vectorize/LoopVectorize.cpp | 22 ++++++++++++-
.../Transforms/Vectorize/VPRecipeBuilder.h | 8 +++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 31 ++++++-------------
.../Transforms/Vectorize/VPlanTransforms.h | 4 +--
4 files changed, 40 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 11a04794ecffc..2c0308a4a8a7b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7525,6 +7525,26 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
return new VPHistogramRecipe(Opcode, HGramOps, VPI->getDebugLoc());
}
+bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
+ VPBuilder &FinalRedStoresBuilder, VPInstruction *VPI) {
+ StoreInst *SI;
+ if ((SI = dyn_cast<StoreInst>(VPI->getUnderlyingInstr())) &&
+ Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
+ // Only create recipe for the final invariant store of the reduction.
+ if (Legal->isInvariantStoreOfReduction(SI)) {
+ auto *Recipe = new VPReplicateRecipe(
+ SI, VPI->operandsWithoutMask(), true /* IsUniform */,
+ nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
+ FinalRedStoresBuilder.insert(Recipe);
+ // Recipe->insertBefore(*MiddleVPBB, MBIP);
+ }
+ VPI->eraseFromParent();
+ return true;
+ }
+
+ return false;
+}
+
VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
VFRange &Range) {
auto *I = VPI->getUnderlyingInstr();
@@ -7814,7 +7834,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
OrigLoop);
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeMemOpWideningDecisions, *Plan,
- Range, RecipeBuilder, CostCtx, *CM.Legal);
+ Range, RecipeBuilder, CostCtx);
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 080151e7cd2cf..a908c25de3fd5 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -92,6 +92,14 @@ class VPRecipeBuilder {
/// scalar loop.
VPHistogramRecipe *widenIfHistogram(VPInstruction *VPI);
+ /// The stores with invariant address inside the loop will be deleted, and in
+ /// the exit block, a uniform store recipe will be created for the final
+ /// invariant store of the reduction. Returns `true` if replacement took
+ /// place. The order of stores must be preserved, hence \p
+ /// FinalRedStoresBuidler.
+ bool replaceWithFinalIfReductionStore(VPBuilder &FinalRedStoresBuilder,
+ VPInstruction *VPI);
+
/// Set the recipe created for given ingredient.
void setRecipe(Instruction *I, VPRecipeBase *R) {
assert(!Ingredient2Recipe.contains(I) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e50e76161abdd..fb28b01457d4a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -42,7 +42,6 @@
#include "llvm/Support/TypeSize.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
-#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
using namespace llvm;
using namespace VPlanPatternMatch;
@@ -6489,16 +6488,19 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
transformToPartialReduction(Chain, CostCtx.Types, Plan, Phi);
}
-void VPlanTransforms::makeMemOpWideningDecisions(
- VPlan &Plan, VFRange &Range, VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx, LoopVectorizationLegality &Legal) {
+void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
+ VPRecipeBuilder &RecipeBuilder,
+ VPCostContext &CostCtx) {
// Filter out scalar VPlan.
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[](ElementCount VF) { return VF.isScalar(); }, Range))
return;
// Scan the body of the loop in a topological order to visit each basic block
- // after having visited its predecessor basic blocks.
+ // after having visited its predecessor basic blocks. This is necessary
+ // because we need to preserve the order of the reduction stores into
+ // invariant address when transforming those to a scalar store outside the
+ // vector loop body.
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
@@ -6519,7 +6521,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(
}
auto *MiddleVPBB = Plan.getMiddleBlock();
- VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
+ VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
for (VPInstruction *VPI : MemOps) {
Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
@@ -6533,22 +6535,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(
VPI->eraseFromParent();
};
- // The stores with invariant address inside the loop will be deleted, and
- // in the exit block, a uniform store recipe will be created for the final
- // invariant store of the reduction.
- StoreInst *SI;
- if ((SI = dyn_cast<StoreInst>(Instr)) &&
- Legal.isInvariantAddressOfReduction(SI->getPointerOperand())) {
- // Only create recipe for the final invariant store of the reduction.
- if (Legal.isInvariantStoreOfReduction(SI)) {
- auto *Recipe = new VPReplicateRecipe(
- SI, VPI->operandsWithoutMask(), true /* IsUniform */,
- nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
- Recipe->insertBefore(*MiddleVPBB, MBIP);
- }
- VPI->eraseFromParent();
+ if (RecipeBuilder.replaceWithFinalIfReductionStore(FinalRedStoresBuilder,
+ VPI))
continue;
- }
if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI)) {
ReplaceWith(Histogram);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 0ff509fa86f61..206d32021405d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -26,7 +26,6 @@ namespace llvm {
class InductionDescriptor;
class Instruction;
class Loop;
-class LoopVectorizationLegality;
class LoopVersioning;
class OptimizationRemarkEmitter;
class PHINode;
@@ -538,8 +537,7 @@ struct VPlanTransforms {
static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx,
- LoopVectorizationLegality &Legal);
+ VPCostContext &CostCtx);
};
} // namespace llvm
>From 5b5cd9f3502ba1020827ec313e795e8832190110 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 10 Mar 2026 10:33:08 -0700
Subject: [PATCH 07/17] Don't use RPOT, per review feedback
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 14 +++-----------
1 file changed, 3 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index fb28b01457d4a..ef4dab5ad951e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6496,21 +6496,13 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
[](ElementCount VF) { return VF.isScalar(); }, Range))
return;
- // Scan the body of the loop in a topological order to visit each basic block
- // after having visited its predecessor basic blocks. This is necessary
- // because we need to preserve the order of the reduction stores into
- // invariant address when transforming those to a scalar store outside the
- // vector loop body.
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
- HeaderVPBB);
-
// Collect all loads/stores first. We will start with ones having simpler
// decisions followed by more complex ones that are potentially
// guided/dependent on the simpler ones.
SmallVector<VPInstruction *> MemOps;
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+ for (VPBasicBlock *VPBB :
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
for (VPRecipeBase &R : *VPBB) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (VPI && VPI->getUnderlyingValue() &&
>From 966fe9890665b15113fec4acc757b6fb22f79e59 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 11 Mar 2026 09:55:59 -0700
Subject: [PATCH 08/17] Drop leftover comment
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2c0308a4a8a7b..b10565861208d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7536,7 +7536,6 @@ bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
SI, VPI->operandsWithoutMask(), true /* IsUniform */,
nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
FinalRedStoresBuilder.insert(Recipe);
- // Recipe->insertBefore(*MiddleVPBB, MBIP);
}
VPI->eraseFromParent();
return true;
>From c285d9634a99b216d7c3738c05f42cb9c59fe5bb Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 11 Mar 2026 10:44:57 -0700
Subject: [PATCH 09/17] Avoid exposing `RecipeBuilder.getVPBuilder()`
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h | 2 --
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 +--
3 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b10565861208d..6c69a1c905dc7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7284,6 +7284,7 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
: GEPNoWrapFlags::none(),
VPI->getDebugLoc());
}
+ Builder.setInsertPoint(VPI);
Builder.insert(VectorPtr);
Ptr = VectorPtr;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index a908c25de3fd5..9f7cd97a6503f 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -73,8 +73,6 @@ class VPRecipeBuilder {
LoopVectorizationCostModel &CM, VPBuilder &Builder)
: Plan(Plan), TLI(TLI), Legal(Legal), CM(CM), Builder(Builder) {}
- VPBuilder &getVPBuilder() const { return Builder; }
-
/// Create and return a widened recipe for a non-phi recipe \p R if one can be
/// created within the given VF \p Range.
VPRecipeBase *tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ef4dab5ad951e..b2e46ed48218b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6517,11 +6517,10 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
for (VPInstruction *VPI : MemOps) {
Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
- RecipeBuilder.getVPBuilder().setInsertPoint(VPI);
auto ReplaceWith = [&](VPRecipeBase *New) {
RecipeBuilder.setRecipe(Instr, New);
- RecipeBuilder.getVPBuilder().insert(New);
+ New->insertBefore(VPI);
if (VPI->getOpcode() == Instruction::Load)
VPI->replaceAllUsesWith(New->getVPSingleValue());
VPI->eraseFromParent();
>From e474f4cfe81dc558c00f3bed05c2ba5f5cc4e63d Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 13 Mar 2026 11:48:01 -0700
Subject: [PATCH 10/17] Reduction store needs to be processed on scalar VPlan
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b2e46ed48218b..1470a4c7191cf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6491,11 +6491,6 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx) {
- // Filter out scalar VPlan.
- if (LoopVectorizationPlanner::getDecisionAndClampRange(
- [](ElementCount VF) { return VF.isScalar(); }, Range))
- return;
-
// Collect all loads/stores first. We will start with ones having simpler
// decisions followed by more complex ones that are potentially
// guided/dependent on the simpler ones.
@@ -6526,10 +6521,16 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPI->eraseFromParent();
};
+ // Note: we must do that for scalar VPlan as well.
if (RecipeBuilder.replaceWithFinalIfReductionStore(FinalRedStoresBuilder,
VPI))
continue;
+ // Filter out scalar VPlan for the remaining memory operations.
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [](ElementCount VF) { return VF.isScalar(); }, Range))
+ return;
+
if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI)) {
ReplaceWith(Histogram);
continue;
>From 8e91986b4dcb6e9e963f75372164d8b285b8d1d8 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 24 Mar 2026 11:56:37 -0700
Subject: [PATCH 11/17] Fix bug/add test
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
.../LoopVectorize/AArch64/memop_widening.ll | 114 ++++++++++++++++++
2 files changed, 115 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/memop_widening.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1470a4c7191cf..231c3d2fee06b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6529,7 +6529,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
// Filter out scalar VPlan for the remaining memory operations.
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[](ElementCount VF) { return VF.isScalar(); }, Range))
- return;
+ continue;
if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI)) {
ReplaceWith(Histogram);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/memop_widening.ll b/llvm/test/Transforms/LoopVectorize/AArch64/memop_widening.ll
new file mode 100644
index 0000000000000..0fc24d79a32cb
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/memop_widening.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S -p loop-vectorize -mtriple=arm64 < %s | FileCheck %s
+
+; Crashed during refactoring if reduction store is not sunk out of the loop.
+define void @ordered_reduction(ptr %dst, float %a, float %b) {
+; CHECK-LABEL: define void @ordered_reduction(
+; CHECK-SAME: ptr [[DST:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[B]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[A]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call float @llvm.vector.reduce.fadd.v2f32(float [[VEC_PHI]], <2 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2]] = call float @llvm.vector.reduce.fadd.v2f32(float [[TMP1]], <2 x float> [[TMP0]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store float [[TMP2]], ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SUM:%.*]] = phi float [ [[TMP2]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[A]], float [[B]], float [[SUM]])
+; CHECK-NEXT: store float [[MULADD]], ptr [[DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV]], 100
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sum = phi float [ 0.000000e+00, %entry ], [ %muladd, %loop ]
+ %muladd = tail call float @llvm.fmuladd.f32(float %a, float %b, float %sum)
+ store float %muladd, ptr %dst, align 4
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv, 100
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Crashed due to not updating "return -> continue" after cut-paste during refactoring.
+define void @ordered_reduction2(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @ordered_reduction2(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.vector.reduce.fadd.v2f32(float [[VEC_PHI]], <2 x float> splat (float 6.000000e+00))
+; CHECK-NEXT: [[TMP1]] = call float @llvm.vector.reduce.fadd.v2f32(float [[TMP0]], <2 x float> splat (float 6.000000e+00))
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store float [[TMP1]], ptr [[DST]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
+; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 97, %[[MIDDLE_BLOCK]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[SRC]], align 4
+; CHECK-NEXT: [[TMP5]] = tail call float @llvm.fmuladd.f32(float 2.000000e+00, float 3.000000e+00, float [[TMP3]])
+; CHECK-NEXT: store float [[TMP5]], ptr [[DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %0 = phi float [ 0.000000e+00, %entry ], [ %2, %loop ]
+ %1 = load float, ptr %src, align 4
+ %2 = tail call float @llvm.fmuladd.f32(float 2.000000e+00, float 3.000000e+00, float %0)
+ store float %2, ptr %dst, align 4
+ %iv.next = add i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 100
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 1cc17f4fc1ce994795199c46ab3f78cec737a75d Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 24 Mar 2026 12:24:52 -0700
Subject: [PATCH 12/17] Add doc comment
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 206d32021405d..59a3b4e9bab36 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -535,6 +535,8 @@ struct VPlanTransforms {
static void createPartialReductions(VPlan &Plan, VPCostContext &CostCtx,
VFRange &Range);
+ /// Convert input load/store instructions into widened or replicate recipes.
+ /// Non load/store input instructions are left unchanged.
static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx);
>From 00b0c5f5e35b833490f1a7d13af19df44787946a Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 26 Mar 2026 15:37:01 -0700
Subject: [PATCH 13/17] Apply code review suggestions
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 12 +++++-------
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 7 +++----
3 files changed, 9 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6c69a1c905dc7..2897bff780e48 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7834,7 +7834,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
OrigLoop);
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeMemOpWideningDecisions, *Plan,
- Range, RecipeBuilder, CostCtx);
+ Range, RecipeBuilder);
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 231c3d2fee06b..303f485863889 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6488,9 +6488,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
transformToPartialReduction(Chain, CostCtx.Types, Plan, Phi);
}
-void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
- VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx) {
+void VPlanTransforms::makeMemOpWideningDecisions(
+ VPlan &Plan, VFRange &Range, VPRecipeBuilder &RecipeBuilder) {
// Collect all loads/stores first. We will start with ones having simpler
// decisions followed by more complex ones that are potentially
// guided/dependent on the simpler ones.
@@ -6511,10 +6510,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
for (VPInstruction *VPI : MemOps) {
- Instruction *Instr = cast<Instruction>(VPI->getUnderlyingValue());
-
auto ReplaceWith = [&](VPRecipeBase *New) {
- RecipeBuilder.setRecipe(Instr, New);
+ RecipeBuilder.setRecipe(cast<Instruction>(VPI->getUnderlyingValue()),
+ New);
New->insertBefore(VPI);
if (VPI->getOpcode() == Instruction::Load)
VPI->replaceAllUsesWith(New->getVPSingleValue());
@@ -6538,7 +6536,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBase *Recipe = RecipeBuilder.tryToWidenMemory(VPI, Range);
if (!Recipe)
- Recipe = RecipeBuilder.handleReplication(cast<VPInstruction>(VPI), Range);
+ Recipe = RecipeBuilder.handleReplication(VPI, Range);
ReplaceWith(Recipe);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 59a3b4e9bab36..bebf7ff1e9262 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -535,11 +535,10 @@ struct VPlanTransforms {
static void createPartialReductions(VPlan &Plan, VPCostContext &CostCtx,
VFRange &Range);
- /// Convert input load/store instructions into widened or replicate recipes.
- /// Non load/store input instructions are left unchanged.
+ /// Convert load/store VPInstructions in \p Plan into widened or replicate
+ /// recipes. Non load/store input instructions are left unchanged.
static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
- VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx);
+ VPRecipeBuilder &RecipeBuilder);
};
} // namespace llvm
>From 2208dae99d541ae5d392cccdbbaaf732306590ca Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 13 Apr 2026 09:14:51 -0700
Subject: [PATCH 14/17] Swap operands order in
VPRecipeBuilder::replaceWithFinalIfReductionStore
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h | 4 ++--
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2897bff780e48..b3a5c39ec2999 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7527,7 +7527,7 @@ VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
}
bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
- VPBuilder &FinalRedStoresBuilder, VPInstruction *VPI) {
+ VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
StoreInst *SI;
if ((SI = dyn_cast<StoreInst>(VPI->getUnderlyingInstr())) &&
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 9f7cd97a6503f..fce2b3169fcbd 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -95,8 +95,8 @@ class VPRecipeBuilder {
/// invariant store of the reduction. Returns `true` if replacement took
/// place. The order of stores must be preserved, hence \p
/// FinalRedStoresBuidler.
- bool replaceWithFinalIfReductionStore(VPBuilder &FinalRedStoresBuilder,
- VPInstruction *VPI);
+ bool replaceWithFinalIfReductionStore(VPInstruction *VPI,
+ VPBuilder &FinalRedStoresBuilder);
/// Set the recipe created for given ingredient.
void setRecipe(Instruction *I, VPRecipeBase *R) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 303f485863889..ebf22f8561673 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6520,8 +6520,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(
};
// Note: we must do that for scalar VPlan as well.
- if (RecipeBuilder.replaceWithFinalIfReductionStore(FinalRedStoresBuilder,
- VPI))
+ if (RecipeBuilder.replaceWithFinalIfReductionStore(VPI,
+ FinalRedStoresBuilder))
continue;
// Filter out scalar VPlan for the remaining memory operations.
>From 1475157a7418f42ec7d2fb39d2e73e05a42584c5 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 13 Apr 2026 09:17:38 -0700
Subject: [PATCH 15/17] noalias + updated comment in/rename the test
---
...rdered-reduction-with-invariant-stores.ll} | 31 +++++++------------
1 file changed, 12 insertions(+), 19 deletions(-)
rename llvm/test/Transforms/LoopVectorize/AArch64/{memop_widening.ll => ordered-reduction-with-invariant-stores.ll} (77%)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/memop_widening.ll b/llvm/test/Transforms/LoopVectorize/AArch64/ordered-reduction-with-invariant-stores.ll
similarity index 77%
rename from llvm/test/Transforms/LoopVectorize/AArch64/memop_widening.ll
rename to llvm/test/Transforms/LoopVectorize/AArch64/ordered-reduction-with-invariant-stores.ll
index 0fc24d79a32cb..1179e83a50392 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/memop_widening.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/ordered-reduction-with-invariant-stores.ll
@@ -54,45 +54,38 @@ exit:
ret void
}
-; Crashed due to not updating "return -> continue" after cut-paste during refactoring.
-define void @ordered_reduction2(ptr %dst, ptr %src) {
+; Same as above but with an additional load (used to catch an error where "early
+; return" was used instead of "early continue" due to copy-paste error during
+; refactoring).
+define void @ordered_reduction2(ptr noalias %dst, ptr noalias %src) {
; CHECK-LABEL: define void @ordered_reduction2(
-; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 4
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_MEMCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_MEMCHECK]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.vector.reduce.fadd.v2f32(float [[VEC_PHI]], <2 x float> splat (float 6.000000e+00))
; CHECK-NEXT: [[TMP1]] = call float @llvm.vector.reduce.fadd.v2f32(float [[TMP0]], <2 x float> splat (float 6.000000e+00))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store float [[TMP1]], ptr [[DST]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
-; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK-NEXT: store float [[TMP1]], ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 97, %[[MIDDLE_BLOCK]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 97, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi float [ [[TMP1]], %[[SCALAR_PH]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[SRC]], align 4
; CHECK-NEXT: [[TMP5]] = tail call float @llvm.fmuladd.f32(float 2.000000e+00, float 3.000000e+00, float [[TMP3]])
; CHECK-NEXT: store float [[TMP5]], ptr [[DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 04a2e03e20e649b071e77d88fc7946f02d0898c6 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 13 Apr 2026 09:25:37 -0700
Subject: [PATCH 16/17] Update doc comment for replaceWithFinalIfReductionStore
---
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index fce2b3169fcbd..e7a295cd5fcb7 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -90,10 +90,10 @@ class VPRecipeBuilder {
/// scalar loop.
VPHistogramRecipe *widenIfHistogram(VPInstruction *VPI);
- /// The stores with invariant address inside the loop will be deleted, and in
- /// the exit block, a uniform store recipe will be created for the final
- /// invariant store of the reduction. Returns `true` if replacement took
- /// place. The order of stores must be preserved, hence \p
+ /// If \p VPI is a store of a reduction into an invariant address, delete it.
+ /// If it is the final store of a reduction result, a uniform store recipe
+ /// will be created for it in the middle block. Returns `true` if replacement
+ /// took place. The order of stores must be preserved, hence \p
/// FinalRedStoresBuidler.
bool replaceWithFinalIfReductionStore(VPInstruction *VPI,
VPBuilder &FinalRedStoresBuilder);
>From cb11242f00d0fd64c9c8bb34ae6226d25a18e2b9 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 13 Apr 2026 09:27:37 -0700
Subject: [PATCH 17/17] Expand `auto *MiddleBlock = ...`
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ebf22f8561673..738f6bc92f541 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6506,7 +6506,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(
}
}
- auto *MiddleVPBB = Plan.getMiddleBlock();
+ VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
for (VPInstruction *VPI : MemOps) {
More information about the llvm-commits
mailing list