[llvm] [VPlan] Support masked VPInsts, use for predication (NFC) (PR #142285)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 1 11:47:39 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/142285
>From 5a69d046d00366593bfbf63d13db6f82e7bf66f6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 31 Oct 2025 22:19:32 +0000
Subject: [PATCH 1/3] [VPlan] Support masked VPInsts, use for predication
(NFC).
Add support for mask operands to most VPInstructions, using
getNumOperandsForOpcode.
This allows VPlan predication to predicate VPInstructions directly. The
mask will then be dropped or handled when creating wide recipes.
Depends on #142284.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 66 ++++++++-----------
.../Transforms/Vectorize/VPRecipeBuilder.h | 25 +------
llvm/lib/Transforms/Vectorize/VPlan.h | 44 ++++++++++++-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 4 +-
.../Vectorize/VPlanConstruction.cpp | 10 +--
.../Transforms/Vectorize/VPlanPatternMatch.h | 18 ++++-
.../Transforms/Vectorize/VPlanPredicator.cpp | 30 +++++----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 15 +++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 50 +-------------
.../Transforms/Vectorize/VPlanTransforms.h | 14 ++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 42 ++++++++++++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 6 ++
12 files changed, 174 insertions(+), 150 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0b4546636e584..0f9a90dd37d5a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7702,7 +7702,7 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
VPValue *Mask = nullptr;
if (Legal->isMaskRequired(I))
- Mask = getBlockInMask(Builder.getInsertBlock());
+ Mask = VPI->getMask();
// Determine if the pointer operand of the access is either consecutive or
// reverse consecutive.
@@ -7873,14 +7873,13 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
// 2) No mask is required for the block, but the only available
// vector variant at this VF requires a mask, so we synthesize an
// all-true mask.
- VPValue *Mask = Legal->isMaskRequired(CI)
- ? getBlockInMask(Builder.getInsertBlock())
- : Plan.getTrue();
+ VPValue *Mask =
+ Legal->isMaskRequired(CI) ? VPI->getMask() : Plan.getTrue();
Ops.insert(Ops.begin() + *MaskPos, Mask);
}
- Ops.push_back(VPI->getOperand(VPI->getNumOperands() - 1));
+ Ops.push_back(VPI->getOperand(size(VPI->operandsWithoutMask()) - 1));
return new VPWidenCallRecipe(CI, Variant, Ops, *VPI, *VPI,
VPI->getDebugLoc());
}
@@ -7914,8 +7913,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
// If not provably safe, use a select to form a safe divisor before widening the
// div/rem operation itself. Otherwise fall through to general handling below.
if (CM.isPredicatedInst(I)) {
- SmallVector<VPValue *> Ops(VPI->operands());
- VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
+ SmallVector<VPValue *> Ops(VPI->operandsWithoutMask());
+ VPValue *Mask = VPI->getMask();
VPValue *One = Plan.getConstantInt(I->getType(), 1u);
auto *SafeRHS =
Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
@@ -7943,10 +7942,10 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
case Instruction::Sub:
case Instruction::Xor:
case Instruction::Freeze:
- return new VPWidenRecipe(*I, VPI->operands(), *VPI, *VPI,
+ return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
VPI->getDebugLoc());
case Instruction::ExtractValue: {
- SmallVector<VPValue *> NewOps(VPI->operands());
+ SmallVector<VPValue *> NewOps(VPI->operandsWithoutMask());
auto *EVI = cast<ExtractValueInst>(I);
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
@@ -7972,7 +7971,7 @@ VPHistogramRecipe *VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
// In case of predicated execution (due to tail-folding, or conditional
// execution, or both), pass the relevant mask.
if (Legal->isMaskRequired(HI->Store))
- HGramOps.push_back(getBlockInMask(Builder.getInsertBlock()));
+ HGramOps.push_back(VPI->getMask());
return new VPHistogramRecipe(Opcode, HGramOps, VPI->getDebugLoc());
}
@@ -8026,7 +8025,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
// added initially. Masked replicate recipes will later be placed under an
// if-then construct to prevent side-effects. Generate recipes to compute
// the block mask for this region.
- BlockInMask = getBlockInMask(Builder.getInsertBlock());
+ BlockInMask = VPI->getMask();
}
// Note that there is some custom logic to mark some intrinsics as uniform
@@ -8036,8 +8035,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
auto *Recipe =
- new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI,
- *VPI, VPI->getDebugLoc());
+ new VPReplicateRecipe(I, VPI->operandsWithoutMask(), IsUniform,
+ BlockInMask, *VPI, *VPI, VPI->getDebugLoc());
return Recipe;
}
@@ -8276,8 +8275,9 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
return nullptr;
if (VPI->getOpcode() == Instruction::GetElementPtr)
- return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands(),
- *VPI, VPI->getDebugLoc());
+ return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr),
+ VPI->operandsWithoutMask(), *VPI,
+ VPI->getDebugLoc());
if (Instruction::isCast(VPI->getOpcode())) {
auto *CI = cast<CastInst>(Instr);
@@ -8293,7 +8293,9 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VPRecipeBase *
VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
unsigned ScaleFactor) {
- assert(Reduction->getNumOperands() == 2 &&
+ // Reduction must have 2 operands (or 3 with mask).
+ assert((Reduction->getNumOperands() == 2 ||
+ (Reduction->isMasked() && Reduction->getNumOperands() == 3)) &&
"Unexpected number of operands for partial reduction");
VPValue *BinOp = Reduction->getOperand(0);
@@ -8323,7 +8325,7 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
VPValue *Cond = nullptr;
if (CM.blockNeedsPredicationForAnyReason(ReductionI->getParent()))
- Cond = getBlockInMask(Builder.getInsertBlock());
+ Cond = Reduction->getMask();
return new VPReductionRecipe(
RecurKind::Add, FastMathFlags(), ReductionI, Accumulator, BinOp, Cond,
@@ -8459,15 +8461,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// ---------------------------------------------------------------------------
// Predicate and linearize the top-level loop region.
// ---------------------------------------------------------------------------
- auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(
- *Plan, CM.foldTailByMasking());
+ VPlanTransforms::introduceMasksAndLinearize(*Plan, CM.foldTailByMasking());
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, Builder,
- BlockMaskCache);
+ VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, Builder);
// TODO: Handle partial reductions with EVL tail folding.
if (!CM.foldTailWithEVL())
RecipeBuilder.collectScaledReductions(Range);
@@ -8480,9 +8480,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
auto *MiddleVPBB = Plan->getMiddleBlock();
VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
- // Mapping from VPValues in the initial plan to their widened VPValues. Needed
- // temporarily to update created block masks.
- DenseMap<VPValue *, VPValue *> Old2New;
// Collect blocks that need predication for in-loop reduction recipes.
DenseSet<BasicBlock *> BlocksNeedingPredication;
@@ -8490,8 +8487,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (CM.blockNeedsPredicationForAnyReason(BB))
BlocksNeedingPredication.insert(BB);
- VPlanTransforms::createInLoopReductionRecipes(
- *Plan, BlockMaskCache, BlocksNeedingPredication, Range.Start);
+ VPlanTransforms::createInLoopReductionRecipes(*Plan, BlocksNeedingPredication,
+ Range.Start);
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
@@ -8520,8 +8517,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
auto *Recipe = new VPReplicateRecipe(
- SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, *VPI,
- *VPI, VPI->getDebugLoc());
+ SI, VPI->operandsWithoutMask(), true /* IsUniform */,
+ nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
@@ -8544,23 +8541,14 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
}
if (Recipe->getNumDefinedValues() == 1) {
VPI->replaceAllUsesWith(Recipe->getVPSingleValue());
- Old2New[VPI] = Recipe->getVPSingleValue();
} else {
assert(Recipe->getNumDefinedValues() == 0 &&
"Unexpected multidef recipe");
- R.eraseFromParent();
}
+ R.eraseFromParent();
}
}
- // replaceAllUsesWith above may invalidate the block masks. Update them here.
- // TODO: Include the masks as operands in the predicated VPlan directly
- // to remove the need to keep a map of masks beyond the predication
- // transform.
- RecipeBuilder.updateBlockMaskCache(Old2New);
- for (VPValue *Old : Old2New.keys())
- Old->getDefiningRecipe()->eraseFromParent();
-
assert(isa<VPRegionBlock>(LoopRegion) &&
!LoopRegion->getEntryBasicBlock()->empty() &&
"entry block must be set to a VPRegionBlock having a non-empty entry "
@@ -8719,7 +8707,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
auto *RR = dyn_cast<VPReductionRecipe>(OrigExitingVPV->getDefiningRecipe());
if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
(!RR || !RR->isPartialReduction())) {
- VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
+ VPValue *Cond = vputils::findHeaderMask(*Plan);
std::optional<FastMathFlags> FMFs =
PhiTy->isFloatingPointTy()
? std::make_optional(RdxDesc.getFastMathFlags())
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 534f51e86a9b7..4999d66298493 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -65,11 +65,6 @@ class VPRecipeBuilder {
VPBuilder &Builder;
- /// The mask of each VPBB, generated earlier and used for predicating recipes
- /// in VPBB.
- /// TODO: remove by applying predication when generating the masks.
- DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache;
-
// VPlan construction support: Hold a mapping from ingredients to
// their recipe.
DenseMap<Instruction *, VPRecipeBase *> Ingredient2Recipe;
@@ -131,10 +126,9 @@ class VPRecipeBuilder {
VPRecipeBuilder(VPlan &Plan, Loop *OrigLoop, const TargetLibraryInfo *TLI,
const TargetTransformInfo *TTI,
LoopVectorizationLegality *Legal,
- LoopVectorizationCostModel &CM, VPBuilder &Builder,
- DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache)
+ LoopVectorizationCostModel &CM, VPBuilder &Builder)
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(CM), Builder(Builder), BlockMaskCache(BlockMaskCache) {}
+ CM(CM), Builder(Builder) {}
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
auto It = ScaledReductionMap.find(ExitInst);
@@ -163,12 +157,6 @@ class VPRecipeBuilder {
Ingredient2Recipe[I] = R;
}
- /// Returns the *entry* mask for block \p VPBB or null if the mask is
- /// all-true.
- VPValue *getBlockInMask(VPBasicBlock *VPBB) const {
- return BlockMaskCache.lookup(VPBB);
- }
-
/// Return the recipe created for given ingredient.
VPRecipeBase *getRecipe(Instruction *I) {
assert(Ingredient2Recipe.count(I) &&
@@ -190,15 +178,6 @@ class VPRecipeBuilder {
}
return Plan.getOrAddLiveIn(V);
}
-
- void updateBlockMaskCache(DenseMap<VPValue *, VPValue *> &Old2New) {
- for (auto &[_, V] : BlockMaskCache) {
- if (auto *New = Old2New.lookup(V)) {
- V->replaceAllUsesWith(New);
- V = New;
- }
- }
- }
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 94a19beb75a8f..aa09442b367bd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1210,7 +1210,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Return the number of operands determined by the opcode of the
/// VPInstruction. Returns -1u if the number of operands cannot be determined
/// directly by the opcode.
- static unsigned getNumOperandsForOpcode(unsigned Opcode);
+ unsigned getNumOperandsForOpcode() const;
private:
typedef unsigned char OpcodeTy;
@@ -1282,6 +1282,48 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
}
}
+ /// Returns true if the VPInstruction has a mask operand.
+ bool isMasked() const {
+ return getNumOperandsForOpcode() + 1 == getNumOperands();
+ }
+
+ /// Returns true if the opcode needs and supports masking.
+ bool needsMask() const {
+ // Don't use masks to synthetic VPInstructions for now, i.e. ones that do
+ // not have underlying values.
+ if (!getUnderlyingValue())
+ return false;
+
+ return Opcode != Instruction::ExtractValue && Opcode != Instruction::PHI &&
+ Opcode != Instruction::ICmp && Opcode != Instruction::FCmp &&
+ Opcode != Instruction::GetElementPtr;
+ }
+
+ /// Add mask \p Mask to an unmasked VPInstruction, if it needs masking.
+ void addMask(VPValue *Mask) {
+ if (!needsMask())
+ return;
+ assert(!isMasked() && "recipe is already masked");
+ addOperand(Mask);
+ }
+
+ /// Returns the mask for the VPInstruction. Returns nullptr for unmasked
+ /// VPInstructions.
+ VPValue *getMask() const {
+ return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
+ }
+
+ /// Returns an iterator range over the operands excluding the mask operand
+ /// if present.
+ iterator_range<operand_iterator> operandsWithoutMask() {
+ unsigned NumOps = getNumOperands() - (isMasked() ? 1 : 0);
+ return make_range(op_begin(), op_begin() + NumOps);
+ }
+ iterator_range<const_operand_iterator> operandsWithoutMask() const {
+ unsigned NumOps = getNumOperands() - (isMasked() ? 1 : 0);
+ return make_range(op_begin(), op_begin() + NumOps);
+ }
+
/// Returns true if the underlying opcode may read from or write to memory.
bool opcodeMayReadOrWriteFromMemory() const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 39e7a044dfff5..c4e698ac2183e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -59,7 +59,9 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
// other operands match and cache them.
auto SetResultTyFromOp = [this, R]() {
Type *ResTy = inferScalarType(R->getOperand(0));
- for (unsigned Op = 1; Op != R->getNumOperands(); ++Op) {
+ unsigned NumOperands =
+ R->isMasked() ? R->getNumOperands() - 1 : R->getNumOperands();
+ for (unsigned Op = 1; Op != NumOperands; ++Op) {
VPValue *OtherV = R->getOperand(Op);
assert(inferScalarType(OtherV) == ResTy &&
"different types inferred for different operands");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1f8243d5f6c72..a245dd66b989a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -705,8 +705,7 @@ void VPlanTransforms::createHeaderPhiRecipes(
}
void VPlanTransforms::createInLoopReductionRecipes(
- VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
- const DenseSet<BasicBlock *> &BlocksNeedingPredication,
+ VPlan &Plan, const DenseSet<BasicBlock *> &BlocksNeedingPredication,
ElementCount MinVF) {
VPTypeAnalysis TypeInfo(Plan);
VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
@@ -829,17 +828,12 @@ void VPlanTransforms::createInLoopReductionRecipes(
? IndexOfFirstOperand + 1
: IndexOfFirstOperand;
VecOp = CurrentLink->getOperand(VecOpId);
- assert(VecOp != PreviousLink &&
- CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
- (VecOpId - IndexOfFirstOperand)) ==
- PreviousLink &&
- "PreviousLink must be the operand other than VecOp");
}
// Get block mask from BlockMaskCache if the block needs predication.
VPValue *CondOp = nullptr;
if (BlocksNeedingPredication.contains(CurrentLinkI->getParent()))
- CondOp = BlockMaskCache.lookup(LinkVPBB);
+ CondOp = cast<VPInstruction>(CurrentLink)->getMask();
assert(PhiR->getVFScaleFactor() == 1 &&
"inloop reductions must be unscaled");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 148ee4119f372..7b3a084aeadff 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -279,10 +279,10 @@ struct Recipe_match {
if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
return false;
- if (R->getNumOperands() != std::tuple_size_v<Ops_t>) {
+ if (R->getNumOperands() < std::tuple_size<Ops_t>::value) {
[[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(R);
assert(((isa<VPInstruction>(R) &&
- VPInstruction::getNumOperandsForOpcode(Opcode) == -1u) ||
+ cast<VPInstruction>(R)->getNumOperandsForOpcode() == -1u) ||
(RepR && std::tuple_size_v<Ops_t> ==
RepR->getNumOperands() - RepR->isPredicated())) &&
"non-variadic recipe with matched opcode does not have the "
@@ -290,6 +290,20 @@ struct Recipe_match {
return false;
}
+ // If the recipe has more operands than expected, check if it's valid, i.e.
+ // either a masked VPInstructions or which either the masked or unmasked
+ // form is matched or a variadic opcode like phi nodes.
+ if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
+ if (auto *VPI = dyn_cast<VPInstruction>(R)) {
+ // Masked VPInstructions have an extra mask operand at the end.
+ if (!VPI->isMasked() ||
+ VPI->getNumOperands() != std::tuple_size<Ops_t>::value + 1)
+ return false;
+ } else if (Opcode != Instruction::PHI) {
+ return false;
+ }
+ }
+
auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
return Op.match(R->getOperand(Idx));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index f7e7fc29bc203..9e7c20c0bf01a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -78,12 +78,10 @@ class VPPredicator {
/// Compute and return the predicate of \p VPBB, assuming that the header
/// block of the loop is set to True, or to the loop mask when tail folding.
- VPValue *createBlockInMask(VPBasicBlock *VPBB);
+ void createBlockInMask(VPBasicBlock *VPBB);
/// Convert phi recipes in \p VPBB to VPBlendRecipes.
void convertPhisToBlends(VPBasicBlock *VPBB);
-
- const BlockMaskCacheTy getBlockMaskCache() const { return BlockMaskCache; }
};
} // namespace
@@ -128,7 +126,7 @@ VPValue *VPPredicator::createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst) {
return setEdgeMask(Src, Dst, EdgeMask);
}
-VPValue *VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
+void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// Start inserting after the block's phis, which be replaced by blends later.
Builder.setInsertPoint(VPBB, VPBB->getFirstNonPhi());
// All-one mask is modelled as no-mask following the convention for masked
@@ -141,7 +139,7 @@ VPValue *VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
if (!EdgeMask) { // Mask of predecessor is all-one so mask of block is
// too.
setBlockInMask(VPBB, EdgeMask);
- return EdgeMask;
+ return;
}
if (!BlockMask) { // BlockMask has its initial nullptr value.
@@ -153,7 +151,6 @@ VPValue *VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
}
setBlockInMask(VPBB, BlockMask);
- return BlockMask;
}
void VPPredicator::createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail) {
@@ -260,8 +257,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
}
-DenseMap<VPBasicBlock *, VPValue *>
-VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
+void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -277,11 +273,22 @@ VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
// header.
if (VPBB == Header) {
Predicator.createHeaderMask(Header, FoldTail);
- continue;
+ } else {
+ Predicator.createBlockInMask(VPBB);
+ Predicator.convertPhisToBlends(VPBB);
}
- Predicator.createBlockInMask(VPBB);
- Predicator.convertPhisToBlends(VPBB);
+ VPValue *BlockMask = Predicator.getBlockInMask(VPBB);
+ if (!BlockMask)
+ continue;
+
+ // Mask all VPInstructions in the block.
+ for (VPRecipeBase &R : *VPBB) {
+ auto *VPI = dyn_cast<VPInstruction>(&R);
+ if (!VPI)
+ continue;
+ VPI->addMask(BlockMask);
+ }
}
// Linearize the blocks of the loop into one serial chain.
@@ -327,5 +334,4 @@ VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
R.getVPSingleValue()->replaceAllUsesWith(Ext);
}
}
- return Predicator.getBlockMaskCache();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b10dd17fbfc89..8bbc80018e3f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -421,13 +421,12 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
- assert((getNumOperandsForOpcode(Opcode) == -1u ||
- getNumOperandsForOpcode(Opcode) == getNumOperands()) &&
+ assert((getNumOperandsForOpcode() == -1u ||
+ getNumOperandsForOpcode() == getNumOperands()) &&
"number of operands does not match opcode");
}
-#ifndef NDEBUG
-unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
+unsigned VPInstruction::getNumOperandsForOpcode() const {
if (Instruction::isUnaryOp(Opcode) || Instruction::isCast(Opcode))
return 1;
@@ -477,7 +476,12 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
case VPInstruction::ReductionStartVector:
case VPInstruction::ExtractLastActive:
return 3;
- case Instruction::Call:
+ case Instruction::Call: {
+ VPValue *LastOp = getOperand(getNumOperands() - 1);
+ if (isa<VPIRValue>(LastOp) && isa<Function>(LastOp->getLiveInIRValue()))
+ return getNumOperands();
+ return getNumOperands() - 1;
+ }
case Instruction::GetElementPtr:
case Instruction::PHI:
case Instruction::Switch:
@@ -492,7 +496,6 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
}
llvm_unreachable("all cases should be handled above");
}
-#endif
bool VPInstruction::doesGeneratePerAllLanes() const {
return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a39b171ab4cd6..805115103acd4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2801,52 +2801,6 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
return LaneMaskPhi;
}
-/// Collect the header mask with the pattern:
-/// (ICMP_ULE, WideCanonicalIV, backedge-taken-count)
-/// TODO: Introduce explicit recipe for header-mask instead of searching
-/// for the header-mask pattern manually.
-static VPSingleDefRecipe *findHeaderMask(VPlan &Plan) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- SmallVector<VPValue *> WideCanonicalIVs;
- auto *FoundWidenCanonicalIVUser = find_if(
- LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
- assert(count_if(LoopRegion->getCanonicalIV()->users(),
- IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
- "Must have at most one VPWideCanonicalIVRecipe");
- if (FoundWidenCanonicalIVUser !=
- LoopRegion->getCanonicalIV()->users().end()) {
- auto *WideCanonicalIV =
- cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
- WideCanonicalIVs.push_back(WideCanonicalIV);
- }
-
- // Also include VPWidenIntOrFpInductionRecipes that represent a widened
- // version of the canonical induction.
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
- auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
- if (WidenOriginalIV && WidenOriginalIV->isCanonical())
- WideCanonicalIVs.push_back(WidenOriginalIV);
- }
-
- // Walk users of wide canonical IVs and find the single compare of the form
- // (ICMP_ULE, WideCanonicalIV, backedge-taken-count).
- VPSingleDefRecipe *HeaderMask = nullptr;
- for (auto *Wide : WideCanonicalIVs) {
- for (VPUser *U : Wide->users()) {
- auto *VPI = dyn_cast<VPInstruction>(U);
- if (!VPI || !vputils::isHeaderMask(VPI, Plan))
- continue;
-
- assert(VPI->getOperand(0) == Wide &&
- "WidenCanonicalIV must be the first operand of the compare");
- assert(!HeaderMask && "Multiple header masks found?");
- HeaderMask = VPI;
- }
- }
- return HeaderMask;
-}
-
void VPlanTransforms::addActiveLaneMask(
VPlan &Plan, bool UseActiveLaneMaskForControlFlow,
bool DataAndControlFlowWithoutRuntimeCheck) {
@@ -2860,7 +2814,7 @@ void VPlanTransforms::addActiveLaneMask(
LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
assert(FoundWidenCanonicalIVUser &&
"Must have widened canonical IV when tail folding!");
- VPSingleDefRecipe *HeaderMask = findHeaderMask(Plan);
+ VPSingleDefRecipe *HeaderMask = vputils::findHeaderMask(Plan);
auto *WideCanonicalIV =
cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
VPSingleDefRecipe *LaneMask;
@@ -3112,7 +3066,7 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
}
}
- VPValue *HeaderMask = findHeaderMask(Plan);
+ VPValue *HeaderMask = vputils::findHeaderMask(Plan);
if (!HeaderMask)
return;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e0d09a099647a..7c0c8539a9bd5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -115,11 +115,9 @@ struct VPlanTransforms {
/// Create VPReductionRecipes for in-loop reductions. This processes chains
/// of operations contributing to in-loop reductions and creates appropriate
- /// VPReductionRecipe instances. Block masks from \p BlockMaskCache are used
- /// to add predication for blocks in \p BlocksNeedingPredication.
+ /// VPReductionRecipe instances.
static void createInLoopReductionRecipes(
- VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
- const DenseSet<BasicBlock *> &BlocksNeedingPredication,
+ VPlan &Plan, const DenseSet<BasicBlock *> &BlocksNeedingPredication,
ElementCount MinVF);
/// Update \p Plan to account for all early exits.
@@ -415,12 +413,8 @@ struct VPlanTransforms {
/// Predicate and linearize the control-flow in the only loop region of
/// \p Plan. If \p FoldTail is true, create a mask guarding the loop
- /// header, otherwise use all-true for the header mask. Masks for blocks are
- /// added to a block-to-mask map which is returned in order to be used later
- /// for wide recipe construction. This argument is temporary and will be
- /// removed in the future.
- static DenseMap<VPBasicBlock *, VPValue *>
- introduceMasksAndLinearize(VPlan &Plan, bool FoldTail);
+ /// header, otherwise use all-true for the header mask.
+ static void introduceMasksAndLinearize(VPlan &Plan, bool FoldTail);
/// Add branch weight metadata, if the \p Plan's middle block is terminated by
/// a BranchOnCond recipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 795d137a3f47c..8112163531565 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -548,6 +548,48 @@ vputils::getRecipesForUncountableExit(VPlan &Plan,
return UncountableCondition;
}
+VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ SmallVector<VPValue *> WideCanonicalIVs;
+ auto *FoundWidenCanonicalIVUser = find_if(
+ LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
+ assert(count_if(LoopRegion->getCanonicalIV()->users(),
+ IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
+ "Must have at most one VPWideCanonicalIVRecipe");
+ if (FoundWidenCanonicalIVUser !=
+ LoopRegion->getCanonicalIV()->users().end()) {
+ auto *WideCanonicalIV =
+ cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
+ WideCanonicalIVs.push_back(WideCanonicalIV);
+ }
+
+ // Also include VPWidenIntOrFpInductionRecipes that represent a widened
+ // version of the canonical induction.
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
+ auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
+ if (WidenOriginalIV && WidenOriginalIV->isCanonical())
+ WideCanonicalIVs.push_back(WidenOriginalIV);
+ }
+
+ // Walk users of wide canonical IVs and find the single compare of the form
+ // (ICMP_ULE, WideCanonicalIV, backedge-taken-count).
+ VPSingleDefRecipe *HeaderMask = nullptr;
+ for (auto *Wide : WideCanonicalIVs) {
+ for (VPUser *U : SmallVector<VPUser *>(Wide->users())) {
+ auto *VPI = dyn_cast<VPInstruction>(U);
+ if (!VPI || !vputils::isHeaderMask(VPI, Plan))
+ continue;
+
+ assert(VPI->getOperand(0) == Wide &&
+ "WidenCanonicalIV must be the first operand of the compare");
+ assert(!HeaderMask && "Multiple header masks found?");
+ HeaderMask = VPI;
+ }
+ }
+ return HeaderMask;
+}
+
bool VPBlockUtils::isHeader(const VPBlockBase *VPB,
const VPDominatorTree &VPDT) {
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 4e7ed1f5a4ab7..d26d7aeca28a2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -103,6 +103,12 @@ inline VPIRFlags getFlagsFromIndDesc(const InductionDescriptor &ID) {
"Expected int induction");
return VPIRFlags::WrapFlagsTy(false, false);
}
+
+/// Collect the header mask with the pattern:
+/// ICMP_ULE, WideCanonicalIV, backedge-taken-count)
+/// TODO: Introduce explicit recipe for header-mask instead of searching
+/// the header-mask pattern manually.
+VPSingleDefRecipe *findHeaderMask(VPlan &Plan);
} // namespace vputils
//===----------------------------------------------------------------------===//
>From eadd44ffda71988cb03100826b3c9f146fb1888b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 22 Jan 2026 12:12:40 +0000
Subject: [PATCH 2/3] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 25 +++++-----
llvm/lib/Transforms/Vectorize/VPlan.h | 48 +++++++++++--------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 3 +-
.../Vectorize/VPlanConstruction.cpp | 8 +++-
.../Transforms/Vectorize/VPlanPatternMatch.h | 16 +++----
.../Transforms/Vectorize/VPlanPredicator.cpp | 10 ++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +
7 files changed, 59 insertions(+), 53 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 51d80106eb1c3..7b872343fd093 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7719,9 +7719,9 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (!LoopVectorizationPlanner::getDecisionAndClampRange(WillWiden, Range))
return nullptr;
- VPValue *Mask = nullptr;
- if (Legal->isMaskRequired(I))
- Mask = VPI->getMask();
+ // If a mask is not required drop it - use unmasked version for safe loads.
+ // TODO: Determine if mask is needed in VPlan.
+ VPValue *Mask = Legal->isMaskRequired(I) ? VPI->getMask() : nullptr;
// Determine if the pointer operand of the access is either consecutive or
// reverse consecutive.
@@ -7886,19 +7886,18 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
if (ShouldUseVectorCall) {
if (MaskPos.has_value()) {
// We have 2 cases that would require a mask:
- // 1) The block needs to be predicated, either due to a conditional
+ // 1) The call needs to be predicated, either due to a conditional
// in the scalar loop or use of an active lane mask with
// tail-folding, and we use the appropriate mask for the block.
- // 2) No mask is required for the block, but the only available
- // vector variant at this VF requires a mask, so we synthesize an
- // all-true mask.
- VPValue *Mask =
- Legal->isMaskRequired(CI) ? VPI->getMask() : Plan.getTrue();
+ // 2) No mask is required for the call instruction, but the only
+ // available vector variant at this VF requires a mask, so we
+ // synthesize an all-true mask.
+ VPValue *Mask = VPI->isMasked() ? VPI->getMask() : Plan.getTrue();
Ops.insert(Ops.begin() + *MaskPos, Mask);
}
- Ops.push_back(VPI->getOperand(size(VPI->operandsWithoutMask()) - 1));
+ Ops.push_back(VPI->getOperand(VPI->getNumOperandsWithoutMask() - 1));
return new VPWidenCallRecipe(CI, Variant, Ops, *VPI, *VPI,
VPI->getDebugLoc());
}
@@ -8315,7 +8314,7 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
// Reduction must have 2 operands (or 3 with mask).
assert((Reduction->getNumOperands() == 2 ||
(Reduction->isMasked() && Reduction->getNumOperands() == 3)) &&
- "Unexpected number of operands for partial reduction");
+ "Partial reduction expected with 2 operands plus an optional mask");
VPValue *BinOp = Reduction->getOperand(0);
VPValue *Accumulator = Reduction->getOperand(1);
@@ -8342,9 +8341,7 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
Builder.insert(BinOp->getDefiningRecipe());
}
- VPValue *Cond = nullptr;
- if (CM.blockNeedsPredicationForAnyReason(ReductionI->getParent()))
- Cond = Reduction->getMask();
+ VPValue *Cond = Reduction->isMasked() ? Reduction->getMask() : nullptr;
return new VPReductionRecipe(
RecurKind::Add, FastMathFlags(), ReductionI, Accumulator, BinOp, Cond,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 2a9304af9b27a..c28faae385b65 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1079,7 +1079,9 @@ class VPIRMetadata {
/// This is a concrete Recipe that models a single VPlan-level instruction.
/// While as any Recipe it may generate a sequence of IR instructions when
/// executed, these instructions would always form a single-def expression as
-/// the VPInstruction is also a single def-use vertex.
+/// the VPInstruction is also a single def-use vertex. Most VPInstructions
+/// opcodes can take an optional mask. Masks may be assigned during
+/// predication.
class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
public VPIRMetadata,
public VPUnrollPartAccessor<1> {
@@ -1207,8 +1209,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
bool doesGeneratePerAllLanes() const;
/// Return the number of operands determined by the opcode of the
- /// VPInstruction. Returns -1u if the number of operands cannot be determined
- /// directly by the opcode.
+ /// VPInstruction, excluding mask. Returns -1u if the number of operands
+ /// cannot be determined directly by the opcode.
unsigned getNumOperandsForOpcode() const;
private:
@@ -1227,6 +1229,16 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// existing value is returned rather than a generated one.
Value *generate(VPTransformState &State);
+ /// Returns true if the VPInstruction does not need masking.
+ bool alwaysUnmasked() const {
+ // For now only VPInstructions with underlying values use masks.
+ // TODO: provide masks to VPInstructions w/o underlying values.
+ if (!getUnderlyingValue())
+ return true;
+
+ return Opcode == Instruction::PHI || Opcode == Instruction::GetElementPtr;
+ }
+
public:
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
@@ -1283,26 +1295,24 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns true if the VPInstruction has a mask operand.
bool isMasked() const {
- return getNumOperandsForOpcode() + 1 == getNumOperands();
- }
-
- /// Returns true if the opcode needs and supports masking.
- bool needsMask() const {
- // Don't use masks to synthetic VPInstructions for now, i.e. ones that do
- // not have underlying values.
- if (!getUnderlyingValue())
+ unsigned NumOpsForOpcode = getNumOperandsForOpcode();
+ // VPInstructions for without fixed number of operands cannot be masked.
+ if (NumOpsForOpcode == -1u)
return false;
+ return NumOpsForOpcode + 1 == getNumOperands();
+ }
- return Opcode != Instruction::ExtractValue && Opcode != Instruction::PHI &&
- Opcode != Instruction::ICmp && Opcode != Instruction::FCmp &&
- Opcode != Instruction::GetElementPtr;
+ /// Returns the number of operands, excluding the mask if the VPInstruction is
+ /// masked.
+ unsigned getNumOperandsWithoutMask() const {
+ return getNumOperands() - isMasked();
}
/// Add mask \p Mask to an unmasked VPInstruction, if it needs masking.
void addMask(VPValue *Mask) {
- if (!needsMask())
- return;
assert(!isMasked() && "recipe is already masked");
+ if (alwaysUnmasked())
+ return;
addOperand(Mask);
}
@@ -1315,12 +1325,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns an iterator range over the operands excluding the mask operand
/// if present.
iterator_range<operand_iterator> operandsWithoutMask() {
- unsigned NumOps = getNumOperands() - (isMasked() ? 1 : 0);
- return make_range(op_begin(), op_begin() + NumOps);
+ return make_range(op_begin(), op_begin() + getNumOperandsWithoutMask());
}
iterator_range<const_operand_iterator> operandsWithoutMask() const {
- unsigned NumOps = getNumOperands() - (isMasked() ? 1 : 0);
- return make_range(op_begin(), op_begin() + NumOps);
+ return make_range(op_begin(), op_begin() + getNumOperandsWithoutMask());
}
/// Returns true if the underlying opcode may read from or write to memory.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index c4e698ac2183e..bddc1acb8a03a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -59,8 +59,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
// other operands match and cache them.
auto SetResultTyFromOp = [this, R]() {
Type *ResTy = inferScalarType(R->getOperand(0));
- unsigned NumOperands =
- R->isMasked() ? R->getNumOperands() - 1 : R->getNumOperands();
+ unsigned NumOperands = R->getNumOperandsWithoutMask();
for (unsigned Op = 1; Op != NumOperands; ++Op) {
VPValue *OtherV = R->getOperand(Op);
assert(inferScalarType(OtherV) == ResTy &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a245dd66b989a..b9e58afd709d3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -828,9 +828,15 @@ void VPlanTransforms::createInLoopReductionRecipes(
? IndexOfFirstOperand + 1
: IndexOfFirstOperand;
VecOp = CurrentLink->getOperand(VecOpId);
+ assert(
+ VecOp != PreviousLink &&
+ CurrentLink->getOperand(
+ cast<VPInstruction>(CurrentLink)->getNumOperandsWithoutMask() -
+ 1 - (VecOpId - IndexOfFirstOperand)) == PreviousLink &&
+ "PreviousLink must be the operand other than VecOp");
}
- // Get block mask from BlockMaskCache if the block needs predication.
+ // Get block mask from CurrentLink, if it needs predication.
VPValue *CondOp = nullptr;
if (BlocksNeedingPredication.contains(CurrentLinkI->getParent()))
CondOp = cast<VPInstruction>(CurrentLink)->getMask();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 6d7b19b49dbfb..322e535bd3c5b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -290,18 +290,14 @@ struct Recipe_match {
return false;
}
- // If the recipe has more operands than expected, check if it's valid, i.e.
- // either a masked VPInstructions or which either the masked or unmasked
- // form is matched or a variadic opcode like phi nodes.
+ // If the recipe has more operands than expected, we only support matching
+ // masked VPInstructions where the number of operands of the matcher is the
+ // same as the number of operands excluding mask.
if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
- if (auto *VPI = dyn_cast<VPInstruction>(R)) {
- // Masked VPInstructions have an extra mask operand at the end.
- if (!VPI->isMasked() ||
- VPI->getNumOperands() != std::tuple_size<Ops_t>::value + 1)
- return false;
- } else if (Opcode != Instruction::PHI) {
+ auto *VPI = dyn_cast<VPInstruction>(R);
+ if (!VPI || !VPI->isMasked() ||
+ VPI->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
return false;
- }
}
auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 9e7c20c0bf01a..e4c1b333cc315 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -76,8 +76,8 @@ class VPPredicator {
/// Compute and return the mask for the vector loop header block.
void createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail);
- /// Compute and return the predicate of \p VPBB, assuming that the header
- /// block of the loop is set to True, or to the loop mask when tail folding.
+ /// Compute the predicate of \p VPBB, assuming that the header block of the
+ /// loop is set to True, or to the loop mask when tail folding.
void createBlockInMask(VPBasicBlock *VPBB);
/// Convert phi recipes in \p VPBB to VPBlendRecipes.
@@ -284,10 +284,8 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
// Mask all VPInstructions in the block.
for (VPRecipeBase &R : *VPBB) {
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (!VPI)
- continue;
- VPI->addMask(BlockMask);
+ if (auto *VPI = dyn_cast<VPInstruction>(&R))
+ VPI->addMask(BlockMask);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f5f88e43c9428..bd0af2e1a3f17 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -477,6 +477,8 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::ExtractLastActive:
return 3;
case Instruction::Call: {
+ // For unmasked calls, the last argument will the called function. Use that
+ // to compute the number of operands without the mask.
VPValue *LastOp = getOperand(getNumOperands() - 1);
if (isa<VPIRValue>(LastOp) && isa<Function>(LastOp->getLiveInIRValue()))
return getNumOperands();
>From 6cea8275fbfb6fa7a136372cbe6792348cd6edcc Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 1 Feb 2026 16:35:02 +0000
Subject: [PATCH 3/3] !fixup assert we don't execute masked VPInsts.
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7d1f77bc40d3b..3e7759a7e8a85 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1276,6 +1276,7 @@ bool VPInstruction::isSingleScalar() const {
}
void VPInstruction::execute(VPTransformState &State) {
+ assert(!isMasked() && "cannot execute masked VPInstruction");
assert(!State.Lane && "VPInstruction executing an Lane");
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
assert(flagsValidForOpcode(getOpcode()) &&
More information about the llvm-commits
mailing list