[llvm] [VPlan] Use WideInfo to improve select-cond-query (PR #216445)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 19:23:30 PDT 2026
https://github.com/artagnon created https://github.com/llvm/llvm-project/pull/216445
The WideInfo abstraction is useful in improving the accuracy of the scalar nature of a select condition, and it is much cheaper than vputils::isSingleScalar.
-- 8< --
Based on https://github.com/llvm/llvm-project/pull/208910.
>From f77c147fa2f2de726a249cd439fb803ef156b3ce Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 6 May 2026 16:26:11 +0100
Subject: [PATCH 1/3] [VPlan] Introduce distillation of widening semantics
Introduce VPWideningInfo, a distillation of widening semantics of
recipes, and demonstrate its utility in vputils.
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 243 +++++++++++++------
1 file changed, 167 insertions(+), 76 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index daf8114d482d0..f88432dd7a2df 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -380,11 +380,40 @@ vputils::getOpcodeOrIntrinsicID(const VPValue *V) {
return {};
}
-/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
-/// uniform, the result will also be uniform.
-static bool preservesUniformity(unsigned Opcode) {
+/// A class keeping track of widening information of various recipes.
+/// A recipe necessarily produces a single scalar value if only the SingleScalar
+/// bit is set, a wide value if only the Wide bit is set, and scalar values for
+/// all VF lanes only the GenPerAllLanes bit is set. The SingleScalar bit can be
+/// set on Wide or GenPerAllLanes recipes, which indicates that the recipe could
+/// be narrowed to single-scalar if legal and profitable. For instructions not
+/// producing values, like an assume or store, the bits talk about the
+/// appropriate operands. Finally, there is a class of instructions that
+/// necessarily take vector operands and produce a scalar result termed
+/// VectorToScalar, or necessarily take a scalar values and produce a vector,
+/// termed ScalarToVector. These are marked with the Agnostic bit.
+class VPWideningInfo {
+ unsigned char Info : 4;
+
+public:
+ using VPWideningTy = enum {
+ SingleScalar = 1 << 0,
+ Wide = 1 << 1,
+ GenPerAllLanes = 1 << 2,
+ Agnostic = 1 << 3
+ };
+
+ VPWideningInfo(unsigned char Info) : Info(Info) {}
+ operator unsigned char() const { return Info; }
+ bool producesSingleScalarResult() const {
+ return Info == SingleScalar || Info == (SingleScalar | Agnostic);
+ }
+ bool couldProduceSingleScalarResult() const { return Info & SingleScalar; }
+};
+
+static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
+ VPWideningInfo WideOrRep) {
if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
- return true;
+ return WideOrRep | VPWideningInfo::SingleScalar;
switch (Opcode) {
case Instruction::Freeze:
case Instruction::GetElementPtr:
@@ -392,13 +421,107 @@ static bool preservesUniformity(unsigned Opcode) {
case Instruction::FCmp:
case Instruction::Select:
case VPInstruction::Not:
- case VPInstruction::Broadcast:
case VPInstruction::MaskedCond:
case VPInstruction::PtrAdd:
- return true;
+ return WideOrRep | VPWideningInfo::SingleScalar;
default:
- return false;
+ return WideOrRep;
+ }
+}
+
+static VPWideningInfo getWideningInfo(const VPRecipeBase &R) {
+ switch (R.getVPRecipeID()) {
+ case VPRecipeBase::VPVectorPointerSC:
+ case VPRecipeBase::VPVectorEndPointerSC:
+ case VPRecipeBase::VPDerivedIVSC:
+ case VPRecipeBase::VPExpandSCEVSC:
+ case VPRecipeBase::VPIRInstructionSC:
+ case VPRecipeBase::VPBranchOnMaskSC:
+ return VPWideningInfo::SingleScalar;
+ case VPRecipeBase::VPScalarIVStepsSC:
+ return VPWideningInfo::GenPerAllLanes;
+ case VPRecipeBase::VPWidenCastSC:
+ case VPRecipeBase::VPWidenGEPSC:
+ case VPRecipeBase::VPPredInstPHISC:
+ case VPRecipeBase::VPBlendSC:
+ return VPWideningInfo::Wide | VPWideningInfo::SingleScalar;
+ case VPRecipeBase::VPInstructionSC: {
+ auto *VPI = cast<VPInstruction>(&R);
+ // Broadcast is a special case of a vector-to-scalar.
+ if (VPI->isVectorToScalar() || VPI->getOpcode() == VPInstruction::Broadcast)
+ return VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
+ // These opcodes take multiple scalars are produce a vector.
+ if (is_contained({VPInstruction::BuildStructVector,
+ VPInstruction::BuildVector,
+ VPInstruction::ActiveLaneMask},
+ VPI->getOpcode()))
+ return VPWideningInfo::Wide | VPWideningInfo::Agnostic;
+ if (VPI->isSingleScalar())
+ return VPWideningInfo::SingleScalar;
+ if (VPI->doesGeneratePerAllLanes())
+ return VPWideningInfo::GenPerAllLanes;
+ return getNarrowableWideningInfo(VPI->getOpcode(), VPWideningInfo::Wide);
+ }
+ case VPRecipeBase::VPExpressionSC: {
+ auto *Expr = cast<VPExpressionRecipe>(&R);
+ return Expr->isVectorToScalar()
+ ? (VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic)
+ : VPWideningInfo::Wide;
+ }
+ case VPRecipeBase::VPReductionSC:
+ case VPRecipeBase::VPReductionEVLSC: {
+ auto *Red = cast<VPReductionRecipe>(&R);
+ return Red->isPartialReduction()
+ ? VPWideningInfo::Wide
+ : (VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic);
+ }
+ case VPRecipeBase::VPReplicateSC: {
+ auto *Rep = cast<VPReplicateRecipe>(&R);
+ if (Rep->isSingleScalar())
+ return VPWideningInfo::SingleScalar;
+ return getNarrowableWideningInfo(Rep->getOpcode(),
+ VPWideningInfo::GenPerAllLanes);
+ }
+ case VPRecipeBase::VPWidenSC: {
+ auto *Wide = cast<VPWidenRecipe>(&R);
+ return getNarrowableWideningInfo(Wide->getOpcode(), VPWideningInfo::Wide);
+ }
+ case VPRecipeBase::VPWidenCanonicalIVSC:
+ case VPRecipeBase::VPWidenPHISC:
+ case VPRecipeBase::VPWidenCallSC:
+ case VPRecipeBase::VPWidenIntrinsicSC:
+ case VPRecipeBase::VPWidenMemIntrinsicSC:
+ case VPRecipeBase::VPWidenLoadSC:
+ case VPRecipeBase::VPWidenLoadEVLSC:
+ case VPRecipeBase::VPWidenStoreSC:
+ case VPRecipeBase::VPWidenStoreEVLSC:
+ case VPRecipeBase::VPInterleaveSC:
+ case VPRecipeBase::VPInterleaveEVLSC:
+ case VPRecipeBase::VPHistogramSC:
+ case VPRecipeBase::VPCurrentIterationPHISC:
+ case VPRecipeBase::VPActiveLaneMaskPHISC:
+ case VPRecipeBase::VPFirstOrderRecurrencePHISC:
+ case VPRecipeBase::VPWidenIntOrFpInductionSC:
+ case VPRecipeBase::VPWidenPointerInductionSC:
+ case VPRecipeBase::VPReductionPHISC:
+ return VPWideningInfo::Wide;
+ }
+ llvm_unreachable("Fell off end of switch: unknown recipe class");
+}
+
+static VPWideningInfo getWideningInfo(const VPValue *VPV) {
+ if (!VPV->hasDefiningRecipe()) {
+ // Only a CanonicalIV region value is single scalar.
+ if (auto *RV = dyn_cast<VPRegionValue>(VPV))
+ return RV == RV->getDefiningRegion()->getCanonicalIV()
+ ? VPWideningInfo::SingleScalar
+ : VPWideningInfo::Wide;
+ // A non-constant live-in may be introduce a Broadcast.
+ return isa<VPConstant>(VPV)
+ ? VPWideningInfo::SingleScalar
+ : VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
}
+ return getWideningInfo(*VPV->getDefiningRecipe());
}
bool vputils::isElementwise(const VPValue *V) {
@@ -410,12 +533,6 @@ bool vputils::isElementwise(const VPValue *V) {
}
bool vputils::isSingleScalar(const VPValue *VPV) {
- // Live-in, symbolic and canonical-IV region values are single-scalar.
- if (auto *RV = dyn_cast<VPRegionValue>(VPV))
- return RV == RV->getDefiningRegion()->getCanonicalIV();
- if (isa<VPIRValue, VPSymbolicValue>(VPV))
- return true;
-
if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
const VPRegionBlock *RegionOfR = Rep->getRegion();
// Don't consider recipes in replicate regions as uniform yet; their first
@@ -423,29 +540,13 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
// lanes.
if (RegionOfR && RegionOfR->isReplicator())
return false;
- return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
- all_of(Rep->operands(), isSingleScalar));
}
- if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
- return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
- if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
- return preservesUniformity(WidenR->getOpcode()) &&
- all_of(WidenR->operands(), isSingleScalar);
- }
- if (auto *VPI = dyn_cast<VPInstruction>(VPV))
- return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- (preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isSingleScalar));
- if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
- return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
- return true;
- if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
- return Expr->isVectorToScalar();
-
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ // FIXME: Marking WidenCast as a single-scalar leads to regressions.
+ VPWideningInfo Info = getWideningInfo(VPV);
+ return Info.producesSingleScalarResult() ||
+ (!isa<VPWidenCastRecipe>(VPV) &&
+ Info.couldProduceSingleScalarResult() &&
+ all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar));
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
@@ -455,50 +556,40 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
if (isa<VPIRValue, VPSymbolicValue>(V))
return true;
- const VPRecipeBase *R = V->getDefiningRecipe();
- const VPBasicBlock *VPBB = R ? R->getParent() : nullptr;
- const VPlan *Plan = VPBB ? VPBB->getPlan() : nullptr;
- if (VPBB &&
- (VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) {
- if (match(R,
+ // Bail out on VPPhi, as we can end up in infinite cycles.
+ if (isa<VPPhi>(V))
+ return false;
+
+ if (const VPRecipeBase *R = V->getDefiningRecipe()) {
+ const VPBasicBlock *VPBB = R->getParent();
+ const VPlan *Plan = VPBB->getPlan();
+ if (VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry()) {
+ if (match(
+ R,
m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()) ||
- match(R, m_ExtractVectorForPart(m_VPValue(), m_VPValue())))
- return false;
- return all_of(R->operands(), isUniformAcrossVFsAndUFs);
+ match(R, m_ExtractVectorForPart(m_VPValue(), m_VPValue())))
+ return false;
+ return all_of(R->operands(), isUniformAcrossVFsAndUFs);
+ }
+ if (auto *RepR = dyn_cast<VPReplicateRecipe>(R)) {
+ // Be conservative about side-effects, except for the
+ // known-side-effecting assumes and stores, which we know will be
+ // uniform.
+ return RepR->isSingleScalar() &&
+ (!RepR->mayHaveSideEffects() ||
+ isa<AssumeInst, StoreInst>(RepR->getUnderlyingInstr())) &&
+ all_of(RepR->operands(), isUniformAcrossVFsAndUFs);
+ }
}
- return TypeSwitch<const VPRecipeBase *, bool>(R)
- .Case([](const VPDerivedIVRecipe *R) { return true; })
- .Case([](const VPReplicateRecipe *R) {
- // Be conservative about side-effects, except for the
- // known-side-effecting assumes and stores, which we know will be
- // uniform.
- return R->isSingleScalar() &&
- (!R->mayHaveSideEffects() ||
- isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
- all_of(R->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPWidenRecipe *R) {
- return preservesUniformity(R->getOpcode()) &&
- all_of(R->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPPhi *) {
- // Bail out on VPPhi, as we can end up in infinite cycles.
- return false;
- })
- .Case([](const VPInstruction *VPI) {
- return (VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- preservesUniformity(VPI->getOpcode())) &&
- all_of(VPI->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPWidenCastRecipe *R) {
- // A cast is uniform according to its operand.
- return isUniformAcrossVFsAndUFs(R->getOperand(0));
- })
- .Default([](const VPRecipeBase *) { // A value is considered non-uniform
- // unless proven otherwise.
- return false;
- });
+ // TODO: Match more recipes.
+ if (!isa<VPDerivedIVRecipe, VPWidenRecipe, VPWidenCastRecipe, VPInstruction>(
+ V))
+ return false;
+
+ VPWideningInfo Info = getWideningInfo(V);
+ return Info.couldProduceSingleScalarResult() &&
+ all_of(V->getDefiningRecipe()->operands(), isUniformAcrossVFsAndUFs);
}
bool vputils::doesGeneratePerAllLanes(const VPRecipeBase *R) {
>From 3dcbfa69d573e5b6abdec28116e2183dbf2c77d5 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sat, 11 Jul 2026 13:43:32 +0100
Subject: [PATCH 2/3] [VPlan] Generalize usesScalars using WideInfo (NFCI)
Generalize VPUser::usesScalars, making it non-virtual, by querying
vputils::getWideningInfo.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 27 -------------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 +++
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 38 ++-----------------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 35 +++++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanValue.h | 9 +----
5 files changed, 47 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b2e87a8f4f52d..bb53559e7f8e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1598,13 +1598,6 @@ class VPInstructionWithType : public VPInstruction {
Type *getResultType() const { return getScalarType(); }
- /// Cast recipes always use scalars of their operand.
- bool usesScalars(const VPValue *Op) const override {
- if (Instruction::isCast(getOpcode()))
- return true;
- return VPInstruction::usesScalars(Op);
- }
-
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@@ -1762,12 +1755,6 @@ class VPIRInstruction : public VPRecipeBase {
Instruction &getInstruction() const { return I; }
- bool usesScalars(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-
bool usesFirstPartOnly(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
"Op must be an operand of the recipe");
@@ -3467,13 +3454,6 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
return isSingleScalar();
}
- /// Returns true if the recipe uses scalars of operand \p Op.
- bool usesScalars(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-
/// Return the mask of a predicated VPReplicateRecipe.
VPValue *getMask() {
assert(isPredicated() && "Trying to get the mask of a unpredicated recipe");
@@ -3529,13 +3509,6 @@ class LLVM_ABI_FOR_TEST VPBranchOnMaskRecipe : public VPRecipeBase {
printOperands(O, SlotTracker);
}
#endif
-
- /// Returns true if the recipe uses scalars of operand \p Op.
- bool usesScalars(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
};
/// A recipe to combine multiple recipes into a single 'expression' recipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 53ab77fbd28b1..91f6754b4e395 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -56,6 +56,12 @@ static cl::opt<bool> VPlanPrintMetadata(
cl::desc("Controls the printing of recipe metadata when debugging."));
#endif
+bool VPUser::usesScalars(const VPValue *Op) const {
+ assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
+ VPWideningInfo WideInfo = vputils::getWideningInfo(*cast<VPRecipeBase>(this));
+ return WideInfo.usesScalarOperands() || usesFirstLaneOnly(Op);
+}
+
bool VPRecipeBase::mayWriteToMemory() const {
switch (getVPRecipeID()) {
case VPExpressionSC:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index f88432dd7a2df..4ec13b52383ea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -380,36 +380,6 @@ vputils::getOpcodeOrIntrinsicID(const VPValue *V) {
return {};
}
-/// A class keeping track of widening information of various recipes.
-/// A recipe necessarily produces a single scalar value if only the SingleScalar
-/// bit is set, a wide value if only the Wide bit is set, and scalar values for
-/// all VF lanes only the GenPerAllLanes bit is set. The SingleScalar bit can be
-/// set on Wide or GenPerAllLanes recipes, which indicates that the recipe could
-/// be narrowed to single-scalar if legal and profitable. For instructions not
-/// producing values, like an assume or store, the bits talk about the
-/// appropriate operands. Finally, there is a class of instructions that
-/// necessarily take vector operands and produce a scalar result termed
-/// VectorToScalar, or necessarily take a scalar values and produce a vector,
-/// termed ScalarToVector. These are marked with the Agnostic bit.
-class VPWideningInfo {
- unsigned char Info : 4;
-
-public:
- using VPWideningTy = enum {
- SingleScalar = 1 << 0,
- Wide = 1 << 1,
- GenPerAllLanes = 1 << 2,
- Agnostic = 1 << 3
- };
-
- VPWideningInfo(unsigned char Info) : Info(Info) {}
- operator unsigned char() const { return Info; }
- bool producesSingleScalarResult() const {
- return Info == SingleScalar || Info == (SingleScalar | Agnostic);
- }
- bool couldProduceSingleScalarResult() const { return Info & SingleScalar; }
-};
-
static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
VPWideningInfo WideOrRep) {
if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
@@ -429,7 +399,7 @@ static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
}
}
-static VPWideningInfo getWideningInfo(const VPRecipeBase &R) {
+VPWideningInfo vputils::getWideningInfo(const VPRecipeBase &R) {
switch (R.getVPRecipeID()) {
case VPRecipeBase::VPVectorPointerSC:
case VPRecipeBase::VPVectorEndPointerSC:
@@ -521,7 +491,7 @@ static VPWideningInfo getWideningInfo(const VPValue *VPV) {
? VPWideningInfo::SingleScalar
: VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
}
- return getWideningInfo(*VPV->getDefiningRecipe());
+ return vputils::getWideningInfo(*VPV->getDefiningRecipe());
}
bool vputils::isElementwise(const VPValue *V) {
@@ -542,7 +512,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
return false;
}
// FIXME: Marking WidenCast as a single-scalar leads to regressions.
- VPWideningInfo Info = getWideningInfo(VPV);
+ VPWideningInfo Info = ::getWideningInfo(VPV);
return Info.producesSingleScalarResult() ||
(!isa<VPWidenCastRecipe>(VPV) &&
Info.couldProduceSingleScalarResult() &&
@@ -587,7 +557,7 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
V))
return false;
- VPWideningInfo Info = getWideningInfo(V);
+ VPWideningInfo Info = ::getWideningInfo(V);
return Info.couldProduceSingleScalarResult() &&
all_of(V->getDefiningRecipe()->operands(), isUniformAcrossVFsAndUFs);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..1927244ae8eb8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -22,6 +22,38 @@ class VPBuilder;
} // namespace llvm
namespace llvm {
+/// A class keeping track of widening information of various recipes.
+/// A recipe necessarily produces a single scalar value if only the SingleScalar
+/// bit is set, a wide value if only the Wide bit is set, and scalar values for
+/// all VF lanes only the GenPerAllLanes bit is set. The SingleScalar bit can be
+/// set on Wide or GenPerAllLanes recipes, which indicates that the recipe could
+/// be narrowed to single-scalar if legal and profitable. For instructions not
+/// producing values, like an assume or store, the bits talk about the
+/// appropriate operands. Finally, there is a class of instructions that
+/// necessarily take vector operands and produce a scalar result termed
+/// VectorToScalar, or necessarily take a scalar values and produce a vector,
+/// termed ScalarToVector. These are marked with the Agnostic bit.
+class VPWideningInfo {
+ unsigned char Info : 4;
+
+public:
+ using VPWideningTy = enum {
+ SingleScalar = 1 << 0,
+ Wide = 1 << 1,
+ GenPerAllLanes = 1 << 2,
+ Agnostic = 1 << 3
+ };
+
+ VPWideningInfo(unsigned char Info) : Info(Info) {}
+ operator unsigned char() const { return Info; }
+ bool producesSingleScalarResult() const {
+ return Info == SingleScalar || Info == (SingleScalar | Agnostic);
+ }
+ bool couldProduceSingleScalarResult() const { return Info & SingleScalar; }
+ bool usesScalarOperands() const {
+ return !(Info & (Agnostic | Wide)) || Info == (Wide | Agnostic);
+ }
+};
namespace vputils {
/// Returns true if only the first lane of \p Def is used.
@@ -62,6 +94,9 @@ bool isSingleScalar(const VPValue *VPV);
/// VPDerivedIV or the canonical IV).
bool isUniformAcrossVFsAndUFs(const VPValue *V);
+/// Get the WideningInfo for \p R.
+VPWideningInfo getWideningInfo(const VPRecipeBase &R);
+
/// Return true if \p V is elementwise, i.e. none of the lanes are permuted.
bool isElementwise(const VPValue *V);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index b318c38c796a3..48f768b4523ac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -476,13 +476,8 @@ class LLVM_ABI_FOR_TEST VPUser {
return const_operand_range(op_begin(), op_end());
}
- /// Returns true if the VPUser uses scalars of operand \p Op. Conservatively
- /// returns if only first (scalar) lane is used, as default.
- virtual bool usesScalars(const VPValue *Op) const {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return usesFirstLaneOnly(Op);
- }
+ /// Returns true if the VPUser uses scalars of operand \p Op.
+ bool usesScalars(const VPValue *Op) const;
/// Returns true if the VPUser only uses the first lane of operand \p Op.
/// Conservatively returns false.
>From d26953c9018eefc380bedfba55c747165ed24985 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sat, 15 Aug 2026 03:17:22 +0100
Subject: [PATCH 3/3] [VPlan] Use WideInfo to improve select-cond-query
The WideInfo abstraction is useful in improving the accuracy of the
scalar nature of a select condition, and it is much cheaper than
vputils::isSingleScalar.
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 ++++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 8 ++++----
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 3 +++
.../LoopVectorize/AArch64/masked-call-scalarize.ll | 6 ++----
llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll | 12 ++++--------
.../LoopVectorize/X86/invariant-load-gather.ll | 8 +++-----
6 files changed, 20 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 91f6754b4e395..0f32349fab022 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -796,7 +796,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
Value *Cond =
State.get(getOperand(0),
- OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0)));
+ OnlyFirstLaneUsed || vputils::getWideningInfo(getOperand(0))
+ .producesSingleScalarResult());
Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(),
@@ -2882,7 +2883,8 @@ void VPWidenRecipe::execute(VPTransformState &State) {
}
case Instruction::Select: {
VPValue *CondOp = getOperand(0);
- Value *Cond = State.get(CondOp, vputils::isSingleScalar(CondOp));
+ Value *Cond = State.get(
+ CondOp, vputils::getWideningInfo(CondOp).producesSingleScalarResult());
Value *Op0 = State.get(getOperand(1));
Value *Op1 = State.get(getOperand(2));
Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 4ec13b52383ea..db5ef8a77d8c2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -479,7 +479,7 @@ VPWideningInfo vputils::getWideningInfo(const VPRecipeBase &R) {
llvm_unreachable("Fell off end of switch: unknown recipe class");
}
-static VPWideningInfo getWideningInfo(const VPValue *VPV) {
+VPWideningInfo vputils::getWideningInfo(const VPValue *VPV) {
if (!VPV->hasDefiningRecipe()) {
// Only a CanonicalIV region value is single scalar.
if (auto *RV = dyn_cast<VPRegionValue>(VPV))
@@ -491,7 +491,7 @@ static VPWideningInfo getWideningInfo(const VPValue *VPV) {
? VPWideningInfo::SingleScalar
: VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
}
- return vputils::getWideningInfo(*VPV->getDefiningRecipe());
+ return getWideningInfo(*VPV->getDefiningRecipe());
}
bool vputils::isElementwise(const VPValue *V) {
@@ -512,7 +512,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
return false;
}
// FIXME: Marking WidenCast as a single-scalar leads to regressions.
- VPWideningInfo Info = ::getWideningInfo(VPV);
+ VPWideningInfo Info = getWideningInfo(VPV);
return Info.producesSingleScalarResult() ||
(!isa<VPWidenCastRecipe>(VPV) &&
Info.couldProduceSingleScalarResult() &&
@@ -557,7 +557,7 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
V))
return false;
- VPWideningInfo Info = ::getWideningInfo(V);
+ VPWideningInfo Info = getWideningInfo(V);
return Info.couldProduceSingleScalarResult() &&
all_of(V->getDefiningRecipe()->operands(), isUniformAcrossVFsAndUFs);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 1927244ae8eb8..cd6862c90b625 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -97,6 +97,9 @@ bool isUniformAcrossVFsAndUFs(const VPValue *V);
/// Get the WideningInfo for \p R.
VPWideningInfo getWideningInfo(const VPRecipeBase &R);
+/// Get the WideningInfo for \p VPV.
+VPWideningInfo getWideningInfo(const VPValue *VPV);
+
/// Return true if \p V is elementwise, i.e. none of the lanes are permuted.
bool isElementwise(const VPValue *V);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
index 8aa223c0f927c..6dc3ed281d683 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
@@ -69,8 +69,7 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFCOMMON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
; TFCOMMON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
; TFCOMMON-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
-; TFCOMMON-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
-; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
; TFCOMMON-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TFCOMMON-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TFCOMMON: [[PRED_STORE_IF]]:
@@ -114,8 +113,7 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
-; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
-; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TFA_INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TFA_INTERLEAVE: [[PRED_STORE_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index d02dc36f7a6ae..8c910c8b22fe5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -279,8 +279,7 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.udiv.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[TMP6]], i32 [[TMP12]])
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP6]], i64 0
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
@@ -305,8 +304,7 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; FIXED-NEXT: [[TMP8:%.*]] = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i1> [[TMP0]])
-; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; FIXED-NEXT: [[PREDPHI2:%.*]] = select i1 [[TMP6]], <4 x i64> [[TMP8]], <4 x i64> [[WIDE_LOAD1]]
+; FIXED-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP8]], <4 x i64> [[WIDE_LOAD1]]
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -355,8 +353,7 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.sdiv.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[TMP6]], i32 [[TMP12]])
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP6]], i64 0
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
@@ -381,8 +378,7 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; FIXED-NEXT: [[TMP8:%.*]] = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i1> [[TMP0]])
-; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; FIXED-NEXT: [[PREDPHI2:%.*]] = select i1 [[TMP6]], <4 x i64> [[TMP8]], <4 x i64> [[WIDE_LOAD1]]
+; FIXED-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP8]], <4 x i64> [[WIDE_LOAD1]]
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
index fe772e7c17e96..e713bd5b9ed11 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
@@ -41,8 +41,7 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) {
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP10]], <16 x i32> [[WIDE_MASKED_GATHER]], <16 x i32> splat (i32 1)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <16 x i1> [[TMP1]], <16 x i32> [[WIDE_MASKED_GATHER]], <16 x i32> splat (i32 1)
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x i32> [[PREDPHI]], i64 15
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
@@ -63,13 +62,12 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) {
; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX10]]
; CHECK-NEXT: store <8 x i32> [[BROADCAST_SPLAT12]], ptr [[TMP6]], align 4, !alias.scope [[META0]], !noalias [[META3]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER13:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[BROADCAST_SPLAT9]], <8 x i1> [[TMP5]], <8 x i32> poison), !alias.scope [[META3]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[BROADCAST_SPLAT9]], <8 x i1> [[TMP5]], <8 x i32> poison), !alias.scope [[META3]]
; CHECK-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX10]], 8
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC7]]
; CHECK-NEXT: br i1 [[TMP7]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: vec.epilog.middle.block:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
-; CHECK-NEXT: [[PREDPHI14:%.*]] = select i1 [[TMP9]], <8 x i32> [[WIDE_MASKED_GATHER13]], <8 x i32> splat (i32 1)
+; CHECK-NEXT: [[PREDPHI14:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[WIDE_MASKED_GATHER12]], <8 x i32> splat (i32 1)
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[PREDPHI14]], i64 7
; CHECK-NEXT: [[CMP_N16:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC7]]
; CHECK-NEXT: br i1 [[CMP_N16]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
More information about the llvm-commits
mailing list