[llvm] [VPlan] Support masked VPInsts, use for predication (NFC)(WIP) (PR #142285)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Jan 11 11:25:14 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/142285
>From 0fc89ee3265b87dca0907981ac34f55cf262c640 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 31 Oct 2025 22:19:32 +0000
Subject: [PATCH] [VPlan] Support masked VPInsts, use for predication (NFC).
Add support for mask operands to most VPInstructions, using
getNumOperandsForOpcode.
This allows VPlan predication to predicate VPInstructions directly. The
mask will then be dropped or handled when creating wide recipes.
Depends on #142284.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 66 ++++++++-----------
.../Transforms/Vectorize/VPRecipeBuilder.h | 25 +------
llvm/lib/Transforms/Vectorize/VPlan.h | 46 ++++++++++++-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 4 +-
.../Vectorize/VPlanConstruction.cpp | 10 +--
.../Transforms/Vectorize/VPlanPatternMatch.h | 16 ++++-
.../Transforms/Vectorize/VPlanPredicator.cpp | 30 +++++----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 15 +++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 50 +-------------
.../Transforms/Vectorize/VPlanTransforms.h | 14 ++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 42 ++++++++++++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 6 ++
12 files changed, 173 insertions(+), 151 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index cdc6ecfa21bcb..ef789121ace85 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7697,7 +7697,7 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
VPValue *Mask = nullptr;
if (Legal->isMaskRequired(I))
- Mask = getBlockInMask(Builder.getInsertBlock());
+ Mask = VPI->getMask();
// Determine if the pointer operand of the access is either consecutive or
// reverse consecutive.
@@ -7868,14 +7868,13 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
// 2) No mask is required for the block, but the only available
// vector variant at this VF requires a mask, so we synthesize an
// all-true mask.
- VPValue *Mask = Legal->isMaskRequired(CI)
- ? getBlockInMask(Builder.getInsertBlock())
- : Plan.getTrue();
+ VPValue *Mask =
+ Legal->isMaskRequired(CI) ? VPI->getMask() : Plan.getTrue();
Ops.insert(Ops.begin() + *MaskPos, Mask);
}
- Ops.push_back(VPI->getOperand(VPI->getNumOperands() - 1));
+ Ops.push_back(VPI->getOperand(size(VPI->operandsWithoutMask()) - 1));
return new VPWidenCallRecipe(CI, Variant, Ops, *VPI, *VPI,
VPI->getDebugLoc());
}
@@ -7909,8 +7908,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
// If not provably safe, use a select to form a safe divisor before widening the
// div/rem operation itself. Otherwise fall through to general handling below.
if (CM.isPredicatedInst(I)) {
- SmallVector<VPValue *> Ops(VPI->operands());
- VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
+ SmallVector<VPValue *> Ops(VPI->operandsWithoutMask());
+ VPValue *Mask = VPI->getMask();
VPValue *One = Plan.getConstantInt(I->getType(), 1u);
auto *SafeRHS =
Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
@@ -7938,10 +7937,10 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
case Instruction::Sub:
case Instruction::Xor:
case Instruction::Freeze:
- return new VPWidenRecipe(*I, VPI->operands(), *VPI, *VPI,
+ return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
VPI->getDebugLoc());
case Instruction::ExtractValue: {
- SmallVector<VPValue *> NewOps(VPI->operands());
+ SmallVector<VPValue *> NewOps(VPI->operandsWithoutMask());
auto *EVI = cast<ExtractValueInst>(I);
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
@@ -7967,7 +7966,7 @@ VPHistogramRecipe *VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
// In case of predicated execution (due to tail-folding, or conditional
// execution, or both), pass the relevant mask.
if (Legal->isMaskRequired(HI->Store))
- HGramOps.push_back(getBlockInMask(Builder.getInsertBlock()));
+ HGramOps.push_back(VPI->getMask());
return new VPHistogramRecipe(Opcode, HGramOps, VPI->getDebugLoc());
}
@@ -8021,7 +8020,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
// added initially. Masked replicate recipes will later be placed under an
// if-then construct to prevent side-effects. Generate recipes to compute
// the block mask for this region.
- BlockInMask = getBlockInMask(Builder.getInsertBlock());
+ BlockInMask = VPI->getMask();
}
// Note that there is some custom logic to mark some intrinsics as uniform
@@ -8031,8 +8030,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
auto *Recipe =
- new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI,
- *VPI, VPI->getDebugLoc());
+ new VPReplicateRecipe(I, VPI->operandsWithoutMask(), IsUniform,
+ BlockInMask, *VPI, *VPI, VPI->getDebugLoc());
return Recipe;
}
@@ -8271,8 +8270,9 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
return nullptr;
if (VPI->getOpcode() == Instruction::GetElementPtr)
- return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands(),
- *VPI, VPI->getDebugLoc());
+ return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr),
+ VPI->operandsWithoutMask(), *VPI,
+ VPI->getDebugLoc());
if (Instruction::isCast(VPI->getOpcode())) {
auto *CI = cast<CastInst>(Instr);
@@ -8288,7 +8288,9 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VPRecipeBase *
VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
unsigned ScaleFactor) {
- assert(Reduction->getNumOperands() == 2 &&
+ // Reduction must have 2 operands (or 3 with mask).
+ assert((Reduction->getNumOperands() == 2 ||
+ (Reduction->isMasked() && Reduction->getNumOperands() == 3)) &&
"Unexpected number of operands for partial reduction");
VPValue *BinOp = Reduction->getOperand(0);
@@ -8318,7 +8320,7 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
VPValue *Cond = nullptr;
if (CM.blockNeedsPredicationForAnyReason(ReductionI->getParent()))
- Cond = getBlockInMask(Builder.getInsertBlock());
+ Cond = Reduction->getMask();
return new VPReductionRecipe(
RecurKind::Add, FastMathFlags(), ReductionI, Accumulator, BinOp, Cond,
@@ -8452,15 +8454,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// ---------------------------------------------------------------------------
// Predicate and linearize the top-level loop region.
// ---------------------------------------------------------------------------
- auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(
- *Plan, CM.foldTailByMasking());
+ VPlanTransforms::introduceMasksAndLinearize(*Plan, CM.foldTailByMasking());
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, Builder,
- BlockMaskCache);
+ VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, Builder);
// TODO: Handle partial reductions with EVL tail folding.
if (!CM.foldTailWithEVL())
RecipeBuilder.collectScaledReductions(Range);
@@ -8473,9 +8473,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
auto *MiddleVPBB = Plan->getMiddleBlock();
VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
- // Mapping from VPValues in the initial plan to their widened VPValues. Needed
- // temporarily to update created block masks.
- DenseMap<VPValue *, VPValue *> Old2New;
// Collect blocks that need predication for in-loop reduction recipes.
DenseSet<BasicBlock *> BlocksNeedingPredication;
@@ -8483,8 +8480,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (CM.blockNeedsPredicationForAnyReason(BB))
BlocksNeedingPredication.insert(BB);
- VPlanTransforms::createInLoopReductionRecipes(
- *Plan, BlockMaskCache, BlocksNeedingPredication, Range.Start);
+ VPlanTransforms::createInLoopReductionRecipes(*Plan, BlocksNeedingPredication,
+ Range.Start);
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
@@ -8513,8 +8510,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
auto *Recipe = new VPReplicateRecipe(
- SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, *VPI,
- *VPI, VPI->getDebugLoc());
+ SI, VPI->operandsWithoutMask(), true /* IsUniform */,
+ nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
@@ -8537,23 +8534,14 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
}
if (Recipe->getNumDefinedValues() == 1) {
VPI->replaceAllUsesWith(Recipe->getVPSingleValue());
- Old2New[VPI] = Recipe->getVPSingleValue();
} else {
assert(Recipe->getNumDefinedValues() == 0 &&
"Unexpected multidef recipe");
- R.eraseFromParent();
}
+ R.eraseFromParent();
}
}
- // replaceAllUsesWith above may invalidate the block masks. Update them here.
- // TODO: Include the masks as operands in the predicated VPlan directly
- // to remove the need to keep a map of masks beyond the predication
- // transform.
- RecipeBuilder.updateBlockMaskCache(Old2New);
- for (VPValue *Old : Old2New.keys())
- Old->getDefiningRecipe()->eraseFromParent();
-
assert(isa<VPRegionBlock>(LoopRegion) &&
!LoopRegion->getEntryBasicBlock()->empty() &&
"entry block must be set to a VPRegionBlock having a non-empty entry "
@@ -8706,7 +8694,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
auto *RR = dyn_cast<VPReductionRecipe>(OrigExitingVPV->getDefiningRecipe());
if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
(!RR || !RR->isPartialReduction())) {
- VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
+ VPValue *Cond = vputils::findHeaderMask(*Plan);
std::optional<FastMathFlags> FMFs =
PhiTy->isFloatingPointTy()
? std::make_optional(RdxDesc.getFastMathFlags())
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 534f51e86a9b7..4999d66298493 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -65,11 +65,6 @@ class VPRecipeBuilder {
VPBuilder &Builder;
- /// The mask of each VPBB, generated earlier and used for predicating recipes
- /// in VPBB.
- /// TODO: remove by applying predication when generating the masks.
- DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache;
-
// VPlan construction support: Hold a mapping from ingredients to
// their recipe.
DenseMap<Instruction *, VPRecipeBase *> Ingredient2Recipe;
@@ -131,10 +126,9 @@ class VPRecipeBuilder {
VPRecipeBuilder(VPlan &Plan, Loop *OrigLoop, const TargetLibraryInfo *TLI,
const TargetTransformInfo *TTI,
LoopVectorizationLegality *Legal,
- LoopVectorizationCostModel &CM, VPBuilder &Builder,
- DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache)
+ LoopVectorizationCostModel &CM, VPBuilder &Builder)
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(CM), Builder(Builder), BlockMaskCache(BlockMaskCache) {}
+ CM(CM), Builder(Builder) {}
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
auto It = ScaledReductionMap.find(ExitInst);
@@ -163,12 +157,6 @@ class VPRecipeBuilder {
Ingredient2Recipe[I] = R;
}
- /// Returns the *entry* mask for block \p VPBB or null if the mask is
- /// all-true.
- VPValue *getBlockInMask(VPBasicBlock *VPBB) const {
- return BlockMaskCache.lookup(VPBB);
- }
-
/// Return the recipe created for given ingredient.
VPRecipeBase *getRecipe(Instruction *I) {
assert(Ingredient2Recipe.count(I) &&
@@ -190,15 +178,6 @@ class VPRecipeBuilder {
}
return Plan.getOrAddLiveIn(V);
}
-
- void updateBlockMaskCache(DenseMap<VPValue *, VPValue *> &Old2New) {
- for (auto &[_, V] : BlockMaskCache) {
- if (auto *New = Old2New.lookup(V)) {
- V->replaceAllUsesWith(New);
- V = New;
- }
- }
- }
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 83fe45bfd0bbf..272981879e4a7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1166,12 +1166,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// existing value is returned rather than a generated one.
Value *generate(VPTransformState &State);
-#if !defined(NDEBUG)
/// Return the number of operands determined by the opcode of the
/// VPInstruction. Returns -1u if the number of operands cannot be determined
/// directly by the opcode.
- static unsigned getNumOperandsForOpcode(unsigned Opcode);
-#endif
+ unsigned getNumOperandsForOpcode() const;
public:
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
@@ -1227,6 +1225,48 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
}
}
+ /// Returns true if the VPInstruction has a mask operand.
+ bool isMasked() const {
+ return getNumOperandsForOpcode() + 1 == getNumOperands();
+ }
+
+ /// Returns true if the opcode needs and supports masking.
+ bool needsMask() const {
+ // Don't use masks to synthetic VPInstructions for now, i.e. ones that do
+ // not have underlying values.
+ if (!getUnderlyingValue())
+ return false;
+
+ return Opcode != Instruction::ExtractValue && Opcode != Instruction::PHI &&
+ Opcode != Instruction::ICmp && Opcode != Instruction::FCmp &&
+ Opcode != Instruction::GetElementPtr;
+ }
+
+ /// Add mask \p Mask to an unmasked VPInstruction, if it needs masking.
+ void addMask(VPValue *Mask) {
+ if (!needsMask())
+ return;
+ assert(!isMasked() && "recipe is already masked");
+ addOperand(Mask);
+ }
+
+ /// Returns the mask for the VPInstruction. Returns nullptr for unmasked
+ /// VPInstructions.
+ VPValue *getMask() const {
+ return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
+ }
+
+ /// Returns an iterator range over the operands excluding the mask operand
+ /// if present.
+ iterator_range<operand_iterator> operandsWithoutMask() {
+ unsigned NumOps = getNumOperands() - (isMasked() ? 1 : 0);
+ return make_range(op_begin(), op_begin() + NumOps);
+ }
+ iterator_range<const_operand_iterator> operandsWithoutMask() const {
+ unsigned NumOps = getNumOperands() - (isMasked() ? 1 : 0);
+ return make_range(op_begin(), op_begin() + NumOps);
+ }
+
/// Returns true if the underlying opcode may read from or write to memory.
bool opcodeMayReadOrWriteFromMemory() const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 27e7915a71675..39d2d345a22f0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -59,7 +59,9 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
// other operands match and cache them.
auto SetResultTyFromOp = [this, R]() {
Type *ResTy = inferScalarType(R->getOperand(0));
- for (unsigned Op = 1; Op != R->getNumOperands(); ++Op) {
+ unsigned NumOperands =
+ R->isMasked() ? R->getNumOperands() - 1 : R->getNumOperands();
+ for (unsigned Op = 1; Op != NumOperands; ++Op) {
VPValue *OtherV = R->getOperand(Op);
assert(inferScalarType(OtherV) == ResTy &&
"different types inferred for different operands");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 33355f9dcd88c..401a966297dec 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -705,8 +705,7 @@ void VPlanTransforms::createHeaderPhiRecipes(
}
void VPlanTransforms::createInLoopReductionRecipes(
- VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
- const DenseSet<BasicBlock *> &BlocksNeedingPredication,
+ VPlan &Plan, const DenseSet<BasicBlock *> &BlocksNeedingPredication,
ElementCount MinVF) {
VPTypeAnalysis TypeInfo(Plan);
VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
@@ -829,17 +828,12 @@ void VPlanTransforms::createInLoopReductionRecipes(
? IndexOfFirstOperand + 1
: IndexOfFirstOperand;
VecOp = CurrentLink->getOperand(VecOpId);
- assert(VecOp != PreviousLink &&
- CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
- (VecOpId - IndexOfFirstOperand)) ==
- PreviousLink &&
- "PreviousLink must be the operand other than VecOp");
}
// Get block mask from BlockMaskCache if the block needs predication.
VPValue *CondOp = nullptr;
if (BlocksNeedingPredication.contains(CurrentLinkI->getParent()))
- CondOp = BlockMaskCache.lookup(LinkVPBB);
+ CondOp = cast<VPInstruction>(CurrentLink)->getMask();
assert(PhiR->getVFScaleFactor() == 1 &&
"inloop reductions must be unscaled");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 3732d009b9537..d7c2cb651dc6b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -289,7 +289,7 @@ struct Recipe_match {
if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
return false;
- if (R->getNumOperands() != std::tuple_size_v<Ops_t>) {
+ if (R->getNumOperands() < std::tuple_size<Ops_t>::value) {
[[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(R);
assert((Opcode == Instruction::PHI ||
(RepR && std::tuple_size_v<Ops_t> ==
@@ -299,6 +299,20 @@ struct Recipe_match {
return false;
}
+ // If the recipe has more operands than expected, check if it's valid, i.e.
+ // either a masked VPInstructions or which either the masked or unmasked
+ // form is matched or a variadic opcode like phi nodes.
+ if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
+ if (auto *VPI = dyn_cast<VPInstruction>(R)) {
+ // Masked VPInstructions have an extra mask operand at the end.
+ if (!VPI->isMasked() ||
+ VPI->getNumOperands() != std::tuple_size<Ops_t>::value + 1)
+ return false;
+ } else if (Opcode != Instruction::PHI) {
+ return false;
+ }
+ }
+
auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
return Op.match(R->getOperand(Idx));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index f7e7fc29bc203..9e7c20c0bf01a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -78,12 +78,10 @@ class VPPredicator {
/// Compute and return the predicate of \p VPBB, assuming that the header
/// block of the loop is set to True, or to the loop mask when tail folding.
- VPValue *createBlockInMask(VPBasicBlock *VPBB);
+ void createBlockInMask(VPBasicBlock *VPBB);
/// Convert phi recipes in \p VPBB to VPBlendRecipes.
void convertPhisToBlends(VPBasicBlock *VPBB);
-
- const BlockMaskCacheTy getBlockMaskCache() const { return BlockMaskCache; }
};
} // namespace
@@ -128,7 +126,7 @@ VPValue *VPPredicator::createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst) {
return setEdgeMask(Src, Dst, EdgeMask);
}
-VPValue *VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
+void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// Start inserting after the block's phis, which be replaced by blends later.
Builder.setInsertPoint(VPBB, VPBB->getFirstNonPhi());
// All-one mask is modelled as no-mask following the convention for masked
@@ -141,7 +139,7 @@ VPValue *VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
if (!EdgeMask) { // Mask of predecessor is all-one so mask of block is
// too.
setBlockInMask(VPBB, EdgeMask);
- return EdgeMask;
+ return;
}
if (!BlockMask) { // BlockMask has its initial nullptr value.
@@ -153,7 +151,6 @@ VPValue *VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
}
setBlockInMask(VPBB, BlockMask);
- return BlockMask;
}
void VPPredicator::createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail) {
@@ -260,8 +257,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
}
-DenseMap<VPBasicBlock *, VPValue *>
-VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
+void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -277,11 +273,22 @@ VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
// header.
if (VPBB == Header) {
Predicator.createHeaderMask(Header, FoldTail);
- continue;
+ } else {
+ Predicator.createBlockInMask(VPBB);
+ Predicator.convertPhisToBlends(VPBB);
}
- Predicator.createBlockInMask(VPBB);
- Predicator.convertPhisToBlends(VPBB);
+ VPValue *BlockMask = Predicator.getBlockInMask(VPBB);
+ if (!BlockMask)
+ continue;
+
+ // Mask all VPInstructions in the block.
+ for (VPRecipeBase &R : *VPBB) {
+ auto *VPI = dyn_cast<VPInstruction>(&R);
+ if (!VPI)
+ continue;
+ VPI->addMask(BlockMask);
+ }
}
// Linearize the blocks of the loop into one serial chain.
@@ -327,5 +334,4 @@ VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
R.getVPSingleValue()->replaceAllUsesWith(Ext);
}
}
- return Predicator.getBlockMaskCache();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2c0772320c3cf..b9e7c2904a76b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -410,13 +410,12 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
- assert((getNumOperandsForOpcode(Opcode) == -1u ||
- getNumOperandsForOpcode(Opcode) == getNumOperands()) &&
+ assert((getNumOperandsForOpcode() == -1u ||
+ getNumOperandsForOpcode() == getNumOperands()) &&
"number of operands does not match opcode");
}
-#ifndef NDEBUG
-unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
+unsigned VPInstruction::getNumOperandsForOpcode() const {
if (Instruction::isUnaryOp(Opcode) || Instruction::isCast(Opcode))
return 1;
@@ -467,7 +466,12 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
return 3;
case VPInstruction::ComputeFindIVResult:
return 4;
- case Instruction::Call:
+ case Instruction::Call: {
+ VPValue *LastOp = getOperand(getNumOperands() - 1);
+ if (isa<VPIRValue>(LastOp) && isa<Function>(LastOp->getLiveInIRValue()))
+ return getNumOperands();
+ return getNumOperands() - 1;
+ }
case Instruction::GetElementPtr:
case Instruction::PHI:
case Instruction::Switch:
@@ -481,7 +485,6 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
}
llvm_unreachable("all cases should be handled above");
}
-#endif
bool VPInstruction::doesGeneratePerAllLanes() const {
return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a430f13f0c9c0..6cd41e060cf0a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2790,52 +2790,6 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
return LaneMaskPhi;
}
-/// Collect the header mask with the pattern:
-/// (ICMP_ULE, WideCanonicalIV, backedge-taken-count)
-/// TODO: Introduce explicit recipe for header-mask instead of searching
-/// for the header-mask pattern manually.
-static VPSingleDefRecipe *findHeaderMask(VPlan &Plan) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- SmallVector<VPValue *> WideCanonicalIVs;
- auto *FoundWidenCanonicalIVUser = find_if(
- LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
- assert(count_if(LoopRegion->getCanonicalIV()->users(),
- IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
- "Must have at most one VPWideCanonicalIVRecipe");
- if (FoundWidenCanonicalIVUser !=
- LoopRegion->getCanonicalIV()->users().end()) {
- auto *WideCanonicalIV =
- cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
- WideCanonicalIVs.push_back(WideCanonicalIV);
- }
-
- // Also include VPWidenIntOrFpInductionRecipes that represent a widened
- // version of the canonical induction.
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
- auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
- if (WidenOriginalIV && WidenOriginalIV->isCanonical())
- WideCanonicalIVs.push_back(WidenOriginalIV);
- }
-
- // Walk users of wide canonical IVs and find the single compare of the form
- // (ICMP_ULE, WideCanonicalIV, backedge-taken-count).
- VPSingleDefRecipe *HeaderMask = nullptr;
- for (auto *Wide : WideCanonicalIVs) {
- for (VPUser *U : Wide->users()) {
- auto *VPI = dyn_cast<VPInstruction>(U);
- if (!VPI || !vputils::isHeaderMask(VPI, Plan))
- continue;
-
- assert(VPI->getOperand(0) == Wide &&
- "WidenCanonicalIV must be the first operand of the compare");
- assert(!HeaderMask && "Multiple header masks found?");
- HeaderMask = VPI;
- }
- }
- return HeaderMask;
-}
-
void VPlanTransforms::addActiveLaneMask(
VPlan &Plan, bool UseActiveLaneMaskForControlFlow,
bool DataAndControlFlowWithoutRuntimeCheck) {
@@ -2849,7 +2803,7 @@ void VPlanTransforms::addActiveLaneMask(
LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
assert(FoundWidenCanonicalIVUser &&
"Must have widened canonical IV when tail folding!");
- VPSingleDefRecipe *HeaderMask = findHeaderMask(Plan);
+ VPSingleDefRecipe *HeaderMask = vputils::findHeaderMask(Plan);
auto *WideCanonicalIV =
cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
VPSingleDefRecipe *LaneMask;
@@ -3067,7 +3021,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
}
}
- VPValue *HeaderMask = findHeaderMask(Plan);
+ VPValue *HeaderMask = vputils::findHeaderMask(Plan);
if (!HeaderMask)
return;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index f4ecee4e7f04f..1165723fa235a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -115,11 +115,9 @@ struct VPlanTransforms {
/// Create VPReductionRecipes for in-loop reductions. This processes chains
/// of operations contributing to in-loop reductions and creates appropriate
- /// VPReductionRecipe instances. Block masks from \p BlockMaskCache are used
- /// to add predication for blocks in \p BlocksNeedingPredication.
+ /// VPReductionRecipe instances.
static void createInLoopReductionRecipes(
- VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
- const DenseSet<BasicBlock *> &BlocksNeedingPredication,
+ VPlan &Plan, const DenseSet<BasicBlock *> &BlocksNeedingPredication,
ElementCount MinVF);
/// Update \p Plan to account for all early exits.
@@ -401,12 +399,8 @@ struct VPlanTransforms {
/// Predicate and linearize the control-flow in the only loop region of
/// \p Plan. If \p FoldTail is true, create a mask guarding the loop
- /// header, otherwise use all-true for the header mask. Masks for blocks are
- /// added to a block-to-mask map which is returned in order to be used later
- /// for wide recipe construction. This argument is temporary and will be
- /// removed in the future.
- static DenseMap<VPBasicBlock *, VPValue *>
- introduceMasksAndLinearize(VPlan &Plan, bool FoldTail);
+ /// header, otherwise use all-true for the header mask.
+ static void introduceMasksAndLinearize(VPlan &Plan, bool FoldTail);
/// Add branch weight metadata, if the \p Plan's middle block is terminated by
/// a BranchOnCond recipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index ae36df06bb81c..297756581ed91 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -481,6 +481,48 @@ vputils::getRecipesForUncountableExit(VPlan &Plan,
return UncountableCondition;
}
+VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ SmallVector<VPValue *> WideCanonicalIVs;
+ auto *FoundWidenCanonicalIVUser = find_if(
+ LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
+ assert(count_if(LoopRegion->getCanonicalIV()->users(),
+ IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
+ "Must have at most one VPWideCanonicalIVRecipe");
+ if (FoundWidenCanonicalIVUser !=
+ LoopRegion->getCanonicalIV()->users().end()) {
+ auto *WideCanonicalIV =
+ cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
+ WideCanonicalIVs.push_back(WideCanonicalIV);
+ }
+
+ // Also include VPWidenIntOrFpInductionRecipes that represent a widened
+ // version of the canonical induction.
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
+ auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
+ if (WidenOriginalIV && WidenOriginalIV->isCanonical())
+ WideCanonicalIVs.push_back(WidenOriginalIV);
+ }
+
+ // Walk users of wide canonical IVs and find the single compare of the form
+ // (ICMP_ULE, WideCanonicalIV, backedge-taken-count).
+ VPSingleDefRecipe *HeaderMask = nullptr;
+ for (auto *Wide : WideCanonicalIVs) {
+ for (VPUser *U : SmallVector<VPUser *>(Wide->users())) {
+ auto *VPI = dyn_cast<VPInstruction>(U);
+ if (!VPI || !vputils::isHeaderMask(VPI, Plan))
+ continue;
+
+ assert(VPI->getOperand(0) == Wide &&
+ "WidenCanonicalIV must be the first operand of the compare");
+ assert(!HeaderMask && "Multiple header masks found?");
+ HeaderMask = VPI;
+ }
+ }
+ return HeaderMask;
+}
+
bool VPBlockUtils::isHeader(const VPBlockBase *VPB,
const VPDominatorTree &VPDT) {
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 4e7ed1f5a4ab7..d26d7aeca28a2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -103,6 +103,12 @@ inline VPIRFlags getFlagsFromIndDesc(const InductionDescriptor &ID) {
"Expected int induction");
return VPIRFlags::WrapFlagsTy(false, false);
}
+
+/// Collect the header mask with the pattern:
+/// ICMP_ULE, WideCanonicalIV, backedge-taken-count)
+/// TODO: Introduce explicit recipe for header-mask instead of searching
+/// the header-mask pattern manually.
+VPSingleDefRecipe *findHeaderMask(VPlan &Plan);
} // namespace vputils
//===----------------------------------------------------------------------===//
More information about the llvm-commits
mailing list