[llvm] [VPlan] Introduce refinement of vputils::isSingleScalar (NFCI) (PR #196181)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu May 7 02:08:38 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/196181
>From 8f2e1434ea995386c60d54acffa49e277783a21d Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 6 May 2026 16:26:11 +0100
Subject: [PATCH] [VPlan] Introduce refinement of vputils::isSingleScalar
(NFCI)
We introduce VPWideningInfo, a distillation of widening semantics of
recipes, to communicate more fine-grained information about widening
information than vputils::isSingleScalar, and demonstrate its utility in
a few places.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 8 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 18 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 7 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 123 ++++++++++++------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 37 ++++++
6 files changed, 143 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6a1ea6b3439bf..55b8acc0682e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1489,6 +1489,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// e.g. by performing a reduction or extracting a lane.
bool isVectorToScalar() const;
+ /// Returns true if this VPInstruction produces a vector value from scalar
+ /// values.
+ bool isScalarToVector() const;
+
/// Returns true if this VPInstruction's operands are single scalars and the
/// result is also a single scalar.
bool isSingleScalar() const;
@@ -3438,8 +3442,8 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
/// effects.
bool mayHaveSideEffects() const;
- /// Returns true if the result of this VPExpressionRecipe is a single-scalar.
- bool isSingleScalar() const;
+ /// Returns true if this VPExpressionRecipe produces a scalar.
+ bool isVectorToScalar() const;
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d04b5edcfc212..59469c1eab296 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -550,7 +550,8 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
}
bool VPInstruction::doesGeneratePerAllLanes() const {
- return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
+ return Opcode == VPInstruction::Unpack ||
+ (Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this));
}
bool VPInstruction::canGenerateScalarForFirstLane() const {
@@ -1320,6 +1321,17 @@ bool VPInstruction::isVectorToScalar() const {
getOpcode() == VPInstruction::AnyOf;
}
+bool VPInstruction::isScalarToVector() const {
+ switch (getOpcode()) {
+ case VPInstruction::BuildStructVector:
+ case VPInstruction::BuildVector:
+ case VPInstruction::Broadcast:
+ return true;
+ default:
+ return false;
+ }
+}
+
bool VPInstruction::isSingleScalar() const {
switch (getOpcode()) {
case Instruction::Load:
@@ -3119,9 +3131,7 @@ bool VPExpressionRecipe::mayHaveSideEffects() const {
return false;
}
-bool VPExpressionRecipe::isSingleScalar() const {
- // Cannot use vputils::isSingleScalar(), because all external operands
- // of the expression will be live-ins while bundled.
+bool VPExpressionRecipe::isVectorToScalar() const {
auto *RR = dyn_cast<VPReductionRecipe>(ExpressionRecipes.back());
return RR && !RR->isPartialReduction();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bcdb91a54e305..40abce8883ce2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4897,18 +4897,15 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
for (VPBasicBlock *VPBB :
concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!isa<VPScalarIVStepsRecipe, VPReplicateRecipe, VPInstruction>(&R))
+ if (!vputils::getWideningInfo(R).couldReplicatePerPart())
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
auto UsesVectorOrInsideReplicateRegion = [DefR, LoopRegion](VPUser *U) {
VPRegionBlock *ParentRegion = cast<VPRecipeBase>(U)->getRegion();
return !U->usesScalars(DefR) || ParentRegion != LoopRegion;
};
- if ((isa<VPReplicateRecipe>(DefR) &&
- cast<VPReplicateRecipe>(DefR)->isSingleScalar()) ||
- (isa<VPInstruction>(DefR) &&
- (vputils::onlyFirstLaneUsed(DefR) ||
- !cast<VPInstruction>(DefR)->doesGeneratePerAllLanes())) ||
+ if ((isa<VPInstruction>(DefR) &&
+ !cast<VPInstruction>(DefR)->doesGeneratePerAllLanes()) ||
none_of(DefR->users(), UsesVectorOrInsideReplicateRegion))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index f1b9efae08377..d31059d5de1d7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -940,12 +940,9 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
SmallVector<VPRecipeBase *> ToRemove;
for (VPBasicBlock *VPBB : VPBBsToUnroll) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!isa<VPInstruction, VPReplicateRecipe, VPScalarIVStepsRecipe>(&R) ||
- (isa<VPReplicateRecipe>(&R) &&
- cast<VPReplicateRecipe>(&R)->isSingleScalar()) ||
+ if (!vputils::getWideningInfo(R).couldReplicatePerPart() ||
(isa<VPInstruction>(&R) &&
- !cast<VPInstruction>(&R)->doesGeneratePerAllLanes() &&
- cast<VPInstruction>(&R)->getOpcode() != VPInstruction::Unpack))
+ !cast<VPInstruction>(&R)->doesGeneratePerAllLanes()))
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 3327c4b188bb3..8dc06bbe98984 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -357,11 +357,8 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
match(Addr, m_scev_AffineAddRec(m_SCEV(), m_SCEV()));
}
-/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
-/// uniform, the result will also be uniform.
-static bool preservesUniformity(unsigned Opcode) {
- if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
- return true;
+static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
+ VPWideningInfo WideOrRep) {
switch (Opcode) {
case Instruction::Freeze:
case Instruction::GetElementPtr:
@@ -369,50 +366,94 @@ static bool preservesUniformity(unsigned Opcode) {
case Instruction::FCmp:
case Instruction::Select:
case VPInstruction::Not:
- case VPInstruction::Broadcast:
case VPInstruction::MaskedCond:
- case VPInstruction::PtrAdd:
- return true;
+ return WideOrRep | VPWideningInfo::Narrow;
default:
- return false;
+ if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
+ return WideOrRep | VPWideningInfo::Narrow;
+ return WideOrRep;
}
}
-bool vputils::isSingleScalar(const VPValue *VPV) {
- // Live-in, symbolic and region-values represent single-scalar values.
- if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(VPV))
- return true;
+VPWideningInfo vputils::getWideningInfo(const VPRecipeBase &R) {
+ switch (R.getVPRecipeID()) {
+ default:
+ return VPWideningInfo::Wide;
+ case VPRecipeBase::VPVectorPointerSC:
+ case VPRecipeBase::VPVectorEndPointerSC:
+ case VPRecipeBase::VPDerivedIVSC:
+ case VPRecipeBase::VPExpandSCEVSC:
+ case VPRecipeBase::VPIRInstructionSC:
+ case VPRecipeBase::VPBranchOnMaskSC:
+ return VPWideningInfo::Narrow;
+ case VPRecipeBase::VPScalarIVStepsSC:
+ return VPWideningInfo::ReplicatePart;
+ case VPRecipeBase::VPWidenCastSC:
+ // FIXME: This should be Wide | Narrow.
+ return VPWideningInfo::Wide;
+ case VPRecipeBase::VPWidenGEPSC:
+ case VPRecipeBase::VPPredInstPHISC:
+ case VPRecipeBase::VPBlendSC:
+ return VPWideningInfo::Wide | VPWideningInfo::Narrow;
+ case VPRecipeBase::VPInstructionSC: {
+ auto *VPI = cast<VPInstruction>(&R);
+ if (VPI->isVectorToScalar())
+ return VPWideningInfo::Narrow | VPWideningInfo::Agnostic;
+ if (VPI->isScalarToVector())
+ return VPWideningInfo::Wide | VPWideningInfo::Agnostic;
+ if (VPI->isSingleScalar())
+ return VPWideningInfo::Narrow;
+ if (VPI->getOpcode() == VPInstruction::Unpack)
+ return VPWideningInfo::ReplicatePart;
+ if (VPI->getOpcode() == VPInstruction::PtrAdd)
+ return VPWideningInfo::ReplicatePart | VPWideningInfo::Narrow;
+ return getNarrowableWideningInfo(VPI->getOpcode(), VPWideningInfo::Wide);
+ }
+ case VPRecipeBase::VPExpressionSC: {
+ auto *Expr = cast<VPExpressionRecipe>(&R);
+ if (Expr->isVectorToScalar())
+ return VPWideningInfo::Narrow | VPWideningInfo::Agnostic;
+ return VPWideningInfo::Wide;
+ }
+ case VPRecipeBase::VPReductionSC: {
+ auto *Red = cast<VPReductionRecipe>(&R);
+ return Red->isPartialReduction()
+ ? VPWideningInfo::Wide
+ : (VPWideningInfo::Narrow | VPWideningInfo::Agnostic);
+ }
+ case VPRecipeBase::VPReplicateSC: {
+ auto *Rep = cast<VPReplicateRecipe>(&R);
+ if (Rep->isSingleScalar())
+ return VPWideningInfo::Narrow;
+ return getNarrowableWideningInfo(Rep->getOpcode(),
+ VPWideningInfo::ReplicatePart);
+ }
+ case VPRecipeBase::VPWidenSC: {
+ auto *Wide = dyn_cast<VPWidenRecipe>(&R);
+ return getNarrowableWideningInfo(Wide->getOpcode(), VPWideningInfo::Wide);
+ }
+ }
+}
+
+VPWideningInfo vputils::getWideningInfo(const VPValue *VPV) {
+ if (!VPV->hasDefiningRecipe())
+ return VPWideningInfo::Narrow;
+ return getWideningInfo(*VPV->getDefiningRecipe());
+}
- if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
+bool vputils::isSingleScalar(const VPValue *VPV) {
+ if (auto *Rep = dyn_cast_or_null<VPReplicateRecipe>(VPV)) {
const VPRegionBlock *RegionOfR = Rep->getRegion();
// Don't consider recipes in replicate regions as uniform yet; their first
// lane cannot be accessed when executing the replicate region for other
// lanes.
if (RegionOfR && RegionOfR->isReplicator())
return false;
- return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
- all_of(Rep->operands(), isSingleScalar));
- }
- if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
- return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
- if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
- return preservesUniformity(WidenR->getOpcode()) &&
- all_of(WidenR->operands(), isSingleScalar);
}
- if (auto *VPI = dyn_cast<VPInstruction>(VPV))
- return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- (preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isSingleScalar));
- if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
- return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
- return true;
- if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
- return Expr->isSingleScalar();
-
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ VPWideningInfo Info = getWideningInfo(VPV);
+ return Info.producesNarrowResult() || match(VPV, m_Broadcast(m_VPValue())) ||
+ (Info.couldProduceNarrowResult() &&
+ all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar));
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
@@ -443,14 +484,12 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
- .Case([](const VPWidenRecipe *R) {
- return preservesUniformity(R->getOpcode()) &&
+ .Case<VPWidenRecipe, VPInstruction>([](const auto *R) {
+ VPWideningInfo Info = getWideningInfo(R);
+ return (Info.couldProduceNarrowResult() ||
+ match(R, m_Broadcast(m_VPValue()))) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
- .Case([](const VPInstruction *VPI) {
- return preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isUniformAcrossVFsAndUFs);
- })
.Case([](const VPWidenCastRecipe *R) {
// A cast is uniform according to its operand.
return isUniformAcrossVFsAndUFs(R->getOperand(0));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index ac3a1005c8f24..7272f2b952a08 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -21,6 +21,38 @@ class PredicatedScalarEvolution;
} // namespace llvm
namespace llvm {
+/// A class keeping track of widening information of various recipes.
+/// A recipe necessarily produces a scalar value if only the Narrow bit is set,
+/// a wide value if only the Wide bit is set, and scalar values for each unroll
+/// part if only the ReplicatePart bit is set. The Narrow bit can be set on Wide
+/// and ReplicatePart recipes, which indicates that the recipe could be
+/// considered narrow if profitable. For instructions not producing values, like
+/// an assume or store, the bits talk about the inherent widening of the recipe.
+/// Finally, there is a class of instructions that necessarily take vector
+/// operands and produce a scalar result, like (Insert|Extract)Element, or
+/// necessarily take a scalar values and produce a vector, like BuildVector or
+/// Broadcast: there is no widening decision to make on this class, and it is
+/// marked with the Agnostic bit.
+class VPWideningInfo {
+ unsigned char Info : 4;
+
+public:
+ using VPWideningTy = enum {
+ Narrow = 1 << 0,
+ Wide = 1 << 1,
+ ReplicatePart = 1 << 2,
+ Agnostic = 1 << 3
+ };
+
+ VPWideningInfo(unsigned char Info) : Info(Info) {}
+ operator unsigned char() const { return Info; }
+ bool isVectorToScalar() const { return (Info & Narrow) && (Info & Agnostic); }
+ bool isScalarToVector() const { return (Info & Wide) && (Info & Agnostic); }
+ bool producesNarrowResult() const { return !(Info & (Wide | ReplicatePart)); }
+ bool couldProduceNarrowResult() const { return Info & Narrow; }
+ bool couldProduceWideResult() const { return Info & Wide; }
+ bool couldReplicatePerPart() const { return Info & ReplicatePart; }
+};
namespace vputils {
/// Returns true if only the first lane of \p Def is used.
@@ -51,6 +83,11 @@ const SCEV *getSCEVExprForVPValue(const VPValue *V,
/// sign-extended AddRec.
bool isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE, const Loop *L);
+/// Get widening information for a given \p VPV, a live-in or a recipe with a
+/// possible opcode.
+VPWideningInfo getWideningInfo(const VPValue *VPV);
+VPWideningInfo getWideningInfo(const VPRecipeBase &R);
+
/// Returns true if \p VPV is a single scalar, either because it produces the
/// same value for all lanes or only has its first lane used.
bool isSingleScalar(const VPValue *VPV);
More information about the llvm-commits
mailing list