[llvm] b47174d - [VPlan] Add createWiden{Load,Store} VPBuilder members (NFC) (#210543)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 22 11:54:04 PDT 2026
Author: Florian Hahn
Date: 2026-07-22T19:53:59+01:00
New Revision: b47174d1d1adca974494d12ba54bd793075c930a
URL: https://github.com/llvm/llvm-project/commit/b47174d1d1adca974494d12ba54bd793075c930a
DIFF: https://github.com/llvm/llvm-project/commit/b47174d1d1adca974494d12ba54bd793075c930a.diff
LOG: [VPlan] Add createWiden{Load,Store} VPBuilder members (NFC) (#210543)
Add member functions to create wide load/store and migrate various sites
to use them.
It also updates makeMemOpWideningDecisions to use it. For consistency,
ReplaceWith has been updated to always require inserted recipes.
PR: https://github.com/llvm/llvm-project/pull/210543
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 89b16f9e27045..2a88d424b449b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -536,6 +536,27 @@ class VPBuilder {
VectorIntrinsicID, CallArguments, Ty, Alignment, MD, DL));
}
+ /// Create a recipe widening \p Load, loading from \p Addr with \p Mask (may
+ /// be null).
+ VPWidenLoadRecipe *createWidenLoad(LoadInst &Load, VPValue *Addr,
+ VPValue *Mask, bool Consecutive,
+ const VPIRMetadata &Metadata,
+ DebugLoc DL) {
+ return tryInsertInstruction(
+ new VPWidenLoadRecipe(Load, Addr, Mask, Consecutive, Metadata, DL));
+ }
+
+ /// Create a recipe widening \p Store, storing \p StoredVal to \p Addr with
+ /// \p Mask (may be null).
+ VPWidenStoreRecipe *createWidenStore(StoreInst &Store, VPValue *Addr,
+ VPValue *StoredVal, VPValue *Mask,
+ bool Consecutive,
+ const VPIRMetadata &Metadata,
+ DebugLoc DL) {
+ return tryInsertInstruction(new VPWidenStoreRecipe(
+ Store, Addr, StoredVal, Mask, Consecutive, Metadata, DL));
+ }
+
//===--------------------------------------------------------------------===//
// RAII helpers.
//===--------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f9e4eb45f9783..db900cadc3a5b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6194,8 +6194,8 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
: VPI->getOperand(1);
+ Builder.setInsertPoint(VPI);
if (Consecutive) {
- Builder.setInsertPoint(VPI);
Ptr = Builder.createConsecutiveVectorPointer(Ptr, getLoadStoreType(I),
Reverse, VPI->getDebugLoc());
}
@@ -6205,13 +6205,11 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
- auto *LoadR = new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, *VPI,
- Load->getDebugLoc());
- if (Reverse) {
- Builder.insert(LoadR);
- return new VPInstruction(VPInstruction::Reverse, LoadR, {}, {},
- LoadR->getDebugLoc());
- }
+ auto *LoadR = Builder.createWidenLoad(*Load, Ptr, Mask, Consecutive, *VPI,
+ Load->getDebugLoc());
+ if (Reverse)
+ return Builder.createNaryOp(VPInstruction::Reverse, LoadR,
+ LoadR->getDebugLoc());
return LoadR;
}
@@ -6220,8 +6218,8 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (Reverse)
StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
Store->getDebugLoc());
- return new VPWidenStoreRecipe(*Store, Ptr, StoredVal, Mask, Consecutive, *VPI,
- Store->getDebugLoc());
+ return Builder.createWidenStore(*Store, Ptr, StoredVal, Mask, Consecutive,
+ *VPI, Store->getDebugLoc());
}
VPWidenIntOrFpInductionRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 192f25a0d5f54..23d7db861e6aa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4036,10 +4036,9 @@ static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
// Narrow interleave group to wide load, as transformed VPlan will only
// process one original iteration.
auto *LI = cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos());
- auto *L = new VPWidenLoadRecipe(*LI, LoadGroup->getAddr(),
- LoadGroup->getMask(), /*Consecutive=*/true,
- *LoadGroup, LoadGroup->getDebugLoc());
- L->insertBefore(LoadGroup);
+ auto *L = VPBuilder(LoadGroup).createWidenLoad(
+ *LI, LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
+ *LoadGroup, LoadGroup->getDebugLoc());
NarrowedOps.insert(L);
return L;
}
@@ -4199,10 +4198,10 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
NarrowedOps, Preheader);
auto *SI =
cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos());
- auto *S = new VPWidenStoreRecipe(*SI, StoreGroup->getAddr(), Res, nullptr,
- /*Consecutive=*/true, *StoreGroup,
- StoreGroup->getDebugLoc());
- S->insertBefore(StoreGroup);
+ VPBuilder(StoreGroup)
+ .createWidenStore(*SI, StoreGroup->getAddr(), Res, nullptr,
+ /*Consecutive=*/true, *StoreGroup,
+ StoreGroup->getDebugLoc());
StoreGroup->eraseFromParent();
}
@@ -5332,7 +5331,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
};
auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
- New->insertBefore(VPI);
+ assert(New->getParent() && "New recipe must have been inserted");
if (VPI->getOpcode() == Instruction::Load)
VPI->replaceAllUsesWith(New->getVPSingleValue());
VPI->eraseFromParent();
@@ -5342,7 +5341,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
};
auto Scalarize = [&](VPInstruction *VPI) {
- return ReplaceWith(VPI, RecipeBuilder.handleReplication(VPI, Range));
+ return ReplaceWith(VPI, VPBuilder(VPI).insert(
+ RecipeBuilder.handleReplication(VPI, Range)));
};
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
@@ -5359,7 +5359,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
return false;
if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
- return ReplaceWith(VPI, Histogram);
+ return ReplaceWith(VPI, VPBuilder(VPI).insert(Histogram));
return false;
});
@@ -5402,9 +5402,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
CostCtx.PSE.getSE()->isLoopInvariant(PtrSCEV, CostCtx.L);
ReplaceWith(VPI,
- new VPReplicateRecipe(
+ VPBuilder(VPI).insert(new VPReplicateRecipe(
I, Ptr, /*IsSingleScalar=*/IsSingleScalarLoad,
- /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc()));
+ /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc())));
return true;
});
}
@@ -5426,18 +5426,18 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
Type *StrideTy =
Plan.getDataLayout().getIndexType(Ptr->getScalarType());
VPValue *StrideOne = Plan.getConstantInt(StrideTy, 1);
- auto *VectorPtr = new VPVectorPointerRecipe(
+ VPBuilder Builder(VPI);
+ auto *VectorPtr = Builder.createVectorPointer(
Ptr, ScalarTy, StrideOne, vputils::getGEPFlagsForPtr(Ptr),
VPI->getDebugLoc());
- VectorPtr->insertBefore(VPI);
VPRecipeBase *WidenedR;
if (IsLoad)
- WidenedR = new VPWidenLoadRecipe(*cast<LoadInst>(I), VectorPtr,
- /*Mask=*/nullptr,
- /*Consecutive=*/true, *VPI,
- VPI->getDebugLoc());
+ WidenedR = Builder.createWidenLoad(*cast<LoadInst>(I), VectorPtr,
+ /*Mask=*/nullptr,
+ /*Consecutive=*/true, *VPI,
+ VPI->getDebugLoc());
else
- WidenedR = new VPWidenStoreRecipe(
+ WidenedR = Builder.createWidenStore(
*cast<StoreInst>(I), VectorPtr, VPI->getOperand(0),
/*Mask=*/nullptr, /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
return ReplaceWith(VPI, WidenedR);
More information about the llvm-commits
mailing list