[llvm] [VPlan] Add createWiden{Load,Store} VPBuilder members (NFC) (PR #210543)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 22 09:50:29 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/210543
>From 62b5a0b7a219effc0d52ae71b83245a69fbb8aeb Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 18 Jul 2026 13:15:10 +0100
Subject: [PATCH 1/2] [VPlan] Add createWiden{Load,Store} VPBuilder members
(NFC)
Add member functions to create wide load/store and migrate various sites
to use them.
It also updates makeMemOpWideningDecisions to use it. For consistency,
ReplaceWith has been updated to always require inserted recipes.
---
.../Vectorize/LoopVectorizationPlanner.h | 21 ++++++++++
.../Transforms/Vectorize/LoopVectorize.cpp | 18 ++++-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 40 +++++++++----------
3 files changed, 49 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index ed02d9475fc6f..87348557878d0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -536,6 +536,27 @@ class VPBuilder {
VectorIntrinsicID, CallArguments, Ty, Alignment, MD, DL));
}
+ /// Create a recipe widening \p Load, loading from \p Addr with an optional
+ /// \p Mask.
+ VPWidenLoadRecipe *createWidenLoad(LoadInst &Load, VPValue *Addr,
+ VPValue *Mask, bool Consecutive,
+ const VPIRMetadata &Metadata,
+ DebugLoc DL) {
+ return tryInsertInstruction(
+ new VPWidenLoadRecipe(Load, Addr, Mask, Consecutive, Metadata, DL));
+ }
+
+ /// Create a recipe widening \p Store, storing \p StoredVal to \p Addr with an
+ /// optional \p Mask.
+ VPWidenStoreRecipe *createWidenStore(StoreInst &Store, VPValue *Addr,
+ VPValue *StoredVal, VPValue *Mask,
+ bool Consecutive,
+ const VPIRMetadata &Metadata,
+ DebugLoc DL) {
+ return tryInsertInstruction(new VPWidenStoreRecipe(
+ Store, Addr, StoredVal, Mask, Consecutive, Metadata, DL));
+ }
+
//===--------------------------------------------------------------------===//
// RAII helpers.
//===--------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1929f52ae7d95..e957d0d45fabd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6193,8 +6193,8 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
: VPI->getOperand(1);
+ Builder.setInsertPoint(VPI);
if (Consecutive) {
- Builder.setInsertPoint(VPI);
Ptr = Builder.createConsecutiveVectorPointer(Ptr, getLoadStoreType(I),
Reverse, VPI->getDebugLoc());
}
@@ -6204,13 +6204,11 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
- auto *LoadR = new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, *VPI,
- Load->getDebugLoc());
- if (Reverse) {
- Builder.insert(LoadR);
- return new VPInstruction(VPInstruction::Reverse, LoadR, {}, {},
- LoadR->getDebugLoc());
- }
+ auto *LoadR = Builder.createWidenLoad(*Load, Ptr, Mask, Consecutive, *VPI,
+ Load->getDebugLoc());
+ if (Reverse)
+ return Builder.createNaryOp(VPInstruction::Reverse, LoadR,
+ LoadR->getDebugLoc());
return LoadR;
}
@@ -6219,8 +6217,8 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (Reverse)
StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
Store->getDebugLoc());
- return new VPWidenStoreRecipe(*Store, Ptr, StoredVal, Mask, Consecutive, *VPI,
- Store->getDebugLoc());
+ return Builder.createWidenStore(*Store, Ptr, StoredVal, Mask, Consecutive,
+ *VPI, Store->getDebugLoc());
}
VPWidenIntOrFpInductionRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 32ce8392ed08d..9a65649ac642d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5208,10 +5208,9 @@ static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
// Narrow interleave group to wide load, as transformed VPlan will only
// process one original iteration.
auto *LI = cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos());
- auto *L = new VPWidenLoadRecipe(*LI, LoadGroup->getAddr(),
- LoadGroup->getMask(), /*Consecutive=*/true,
- *LoadGroup, LoadGroup->getDebugLoc());
- L->insertBefore(LoadGroup);
+ auto *L = VPBuilder(LoadGroup).createWidenLoad(
+ *LI, LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
+ *LoadGroup, LoadGroup->getDebugLoc());
NarrowedOps.insert(L);
return L;
}
@@ -5371,10 +5370,10 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
NarrowedOps, Preheader);
auto *SI =
cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos());
- auto *S = new VPWidenStoreRecipe(*SI, StoreGroup->getAddr(), Res, nullptr,
- /*Consecutive=*/true, *StoreGroup,
- StoreGroup->getDebugLoc());
- S->insertBefore(StoreGroup);
+ VPBuilder(StoreGroup)
+ .createWidenStore(*SI, StoreGroup->getAddr(), Res, nullptr,
+ /*Consecutive=*/true, *StoreGroup,
+ StoreGroup->getDebugLoc());
StoreGroup->eraseFromParent();
}
@@ -6542,7 +6541,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
};
auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
- New->insertBefore(VPI);
+ assert(New->getParent() && "New recipe must have been inserted");
if (VPI->getOpcode() == Instruction::Load)
VPI->replaceAllUsesWith(New->getVPSingleValue());
VPI->eraseFromParent();
@@ -6552,7 +6551,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
};
auto Scalarize = [&](VPInstruction *VPI) {
- return ReplaceWith(VPI, RecipeBuilder.handleReplication(VPI, Range));
+ return ReplaceWith(VPI, VPBuilder(VPI).insert(
+ RecipeBuilder.handleReplication(VPI, Range)));
};
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
@@ -6569,7 +6569,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
return false;
if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
- return ReplaceWith(VPI, Histogram);
+ return ReplaceWith(VPI, VPBuilder(VPI).insert(Histogram));
return false;
});
@@ -6612,9 +6612,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
CostCtx.PSE.getSE()->isLoopInvariant(PtrSCEV, CostCtx.L);
ReplaceWith(VPI,
- new VPReplicateRecipe(
+ VPBuilder(VPI).insert(new VPReplicateRecipe(
I, Ptr, /*IsSingleScalar=*/IsSingleScalarLoad,
- /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc()));
+ /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc())));
return true;
});
}
@@ -6636,18 +6636,18 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
Type *StrideTy =
Plan.getDataLayout().getIndexType(Ptr->getScalarType());
VPValue *StrideOne = Plan.getConstantInt(StrideTy, 1);
- auto *VectorPtr = new VPVectorPointerRecipe(
+ VPBuilder Builder(VPI);
+ auto *VectorPtr = Builder.createVectorPointer(
Ptr, ScalarTy, StrideOne, vputils::getGEPFlagsForPtr(Ptr),
VPI->getDebugLoc());
- VectorPtr->insertBefore(VPI);
VPRecipeBase *WidenedR;
if (IsLoad)
- WidenedR = new VPWidenLoadRecipe(*cast<LoadInst>(I), VectorPtr,
- /*Mask=*/nullptr,
- /*Consecutive=*/true, *VPI,
- VPI->getDebugLoc());
+ WidenedR = Builder.createWidenLoad(*cast<LoadInst>(I), VectorPtr,
+ /*Mask=*/nullptr,
+ /*Consecutive=*/true, *VPI,
+ VPI->getDebugLoc());
else
- WidenedR = new VPWidenStoreRecipe(
+ WidenedR = Builder.createWidenStore(
*cast<StoreInst>(I), VectorPtr, VPI->getOperand(0),
/*Mask=*/nullptr, /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
return ReplaceWith(VPI, WidenedR);
>From 4ac282f401f3276f7a054d0d3e29577863486aa2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 22 Jul 2026 17:49:55 +0100
Subject: [PATCH 2/2] !fixup update comment for mask
---
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 4ac26637d98d6..2a88d424b449b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -536,8 +536,8 @@ class VPBuilder {
VectorIntrinsicID, CallArguments, Ty, Alignment, MD, DL));
}
- /// Create a recipe widening \p Load, loading from \p Addr with an optional
- /// \p Mask.
+ /// Create a recipe widening \p Load, loading from \p Addr with \p Mask (may
+ /// be null).
VPWidenLoadRecipe *createWidenLoad(LoadInst &Load, VPValue *Addr,
VPValue *Mask, bool Consecutive,
const VPIRMetadata &Metadata,
@@ -546,8 +546,8 @@ class VPBuilder {
new VPWidenLoadRecipe(Load, Addr, Mask, Consecutive, Metadata, DL));
}
- /// Create a recipe widening \p Store, storing \p StoredVal to \p Addr with an
- /// optional \p Mask.
+ /// Create a recipe widening \p Store, storing \p StoredVal to \p Addr with
+ /// \p Mask (may be null).
VPWidenStoreRecipe *createWidenStore(StoreInst &Store, VPValue *Addr,
VPValue *StoredVal, VPValue *Mask,
bool Consecutive,
More information about the llvm-commits
mailing list