[llvm] 36e9bff - [VPlan] Demarcate VPIs that gen scalars (NFC) (#196207)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 01:58:16 PDT 2026
Author: Ramkumar Ramachandra
Date: 2026-09-30T08:58:10Z
New Revision: 36e9bff64557a2f4b4bf537c8327fec2a1d5b0cf
URL: https://github.com/llvm/llvm-project/commit/36e9bff64557a2f4b4bf537c8327fec2a1d5b0cf
DIFF: https://github.com/llvm/llvm-project/commit/36e9bff64557a2f4b4bf537c8327fec2a1d5b0cf.diff
LOG: [VPlan] Demarcate VPIs that gen scalars (NFC) (#196207)
Certain VPInstruction opcodes generate single-scalars, others generate
vectors, and there are yet others that could generate either. Make this
clear in the code, by factoring the exact condition and asserting on
each opcode. This also makes it clear that the handling for
Logical(And|Or) is incomplete, and lays the groundwork for further
extending the opcodes handles by VPInstruction.
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b5b832ec31d0e..aa93dbaa65170 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1434,12 +1434,12 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns true if we can generate a scalar for the first lane only if
/// needed.
- bool canGenerateScalarForFirstLane() const;
+ bool doesGenerateSingleScalar() const;
- /// Utility methods serving execute(): generates a single vector instance of
- /// the modeled instruction. \returns the generated value. . In some cases an
- /// existing value is returned rather than a generated one.
- Value *generate(VPTransformState &State);
+ /// Utility method serving execute: Generates either a single-scalar or vector
+ /// value. \p GenerateSingleScalar determines whether to generate a
+ /// single-scalar value.
+ Value *generate(VPTransformState &State, bool GenerateSingleScalar);
/// Returns true if the VPInstruction does not need masking.
bool alwaysUnmasked() const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2a6349dd670db..5c31590283dbb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -711,9 +711,7 @@ bool VPInstruction::doesGeneratePerAllLanes() const {
(Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this));
}
-bool VPInstruction::canGenerateScalarForFirstLane() const {
- if (Instruction::isBinaryOp(getOpcode()) || Instruction::isCast(getOpcode()))
- return true;
+bool VPInstruction::doesGenerateSingleScalar() const {
if (isSingleScalar() || isVectorToScalar())
return true;
switch (Opcode) {
@@ -729,9 +727,9 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
case VPInstruction::ExplicitVectorLength:
case VPInstruction::AnyOf:
case VPInstruction::Not:
- return true;
+ return vputils::onlyFirstLaneUsed(this);
default:
- return false;
+ return Instruction::isBinaryOp(Opcode) && vputils::onlyFirstLaneUsed(this);
}
}
@@ -743,13 +741,13 @@ static Instruction::BinaryOps getSubRecurOpcode(RecurKind Kind) {
llvm_unreachable("RecurKind should be Sub/FSub.");
}
-Value *VPInstruction::generate(VPTransformState &State) {
+Value *VPInstruction::generate(VPTransformState &State,
+ bool GenerateSingleScalar) {
IRBuilderBase &Builder = State.Builder;
if (Instruction::isBinaryOp(getOpcode())) {
- bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
- Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
- Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
+ Value *A = State.get(getOperand(0), GenerateSingleScalar);
+ Value *B = State.get(getOperand(1), GenerateSingleScalar);
auto *Res =
Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
if (auto *I = dyn_cast<Instruction>(Res))
@@ -769,11 +767,24 @@ Value *VPInstruction::generate(VPTransformState &State) {
switch (getOpcode()) {
case VPInstruction::Not: {
- bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
- Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
+ Value *A = State.get(getOperand(0), GenerateSingleScalar);
return Builder.CreateNot(A, Name);
}
+ case VPInstruction::LogicalAnd: {
+ // TODO: Use IsSingleScalar to produce a scalar value.
+ Value *A = State.get(getOperand(0));
+ Value *B = State.get(getOperand(1));
+ return Builder.CreateLogicalAnd(A, B, Name);
+ }
+ case VPInstruction::LogicalOr: {
+ // TODO: Use IsSingleScalar to produce a scalar value.
+ Value *A = State.get(getOperand(0));
+ Value *B = State.get(getOperand(1));
+ return Builder.CreateLogicalOr(A, B, Name);
+ }
case Instruction::ExtractElement: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for ExtractElement");
assert(State.VF.isVector() && "Only extract elements from vectors");
if (auto *Idx = dyn_cast<VPConstantInt>(getOperand(1)))
return State.get(getOperand(0), VPLane(Idx->getZExtValue()));
@@ -782,6 +793,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateExtractElement(Vec, Idx, Name);
}
case Instruction::InsertElement: {
+ assert(!GenerateSingleScalar &&
+ "Cannot generate scalar value for InsertElement");
assert(State.VF.isVector() && "Can only insert elements into vectors");
Value *Vec = State.get(getOperand(0), /*NeedsSingleScalar=*/false);
Value *Elt = State.get(getOperand(1), /*NeedsSingleScalar=*/true);
@@ -789,31 +802,31 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateInsertElement(Vec, Elt, Idx, Name);
}
case Instruction::Freeze: {
- Value *Op = State.get(getOperand(0), vputils::onlyFirstLaneUsed(this));
+ Value *Op = State.get(getOperand(0), GenerateSingleScalar);
return Builder.CreateFreeze(Op, Name);
}
case Instruction::FCmp:
case Instruction::ICmp: {
- bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
- Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
- Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
+ Value *A = State.get(getOperand(0), GenerateSingleScalar);
+ Value *B = State.get(getOperand(1), GenerateSingleScalar);
return Builder.CreateCmp(getPredicate(), A, B, Name);
}
case Instruction::PHI: {
llvm_unreachable("should be handled by VPPhi::execute");
}
case Instruction::Select: {
- bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
Value *Cond =
- State.get(getOperand(0),
- OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0)));
- Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
- Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
+ State.get(getOperand(0), GenerateSingleScalar ||
+ vputils::isSingleScalar(getOperand(0)));
+ Value *Op1 = State.get(getOperand(1), GenerateSingleScalar);
+ Value *Op2 = State.get(getOperand(2), GenerateSingleScalar);
return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(),
Name);
}
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask: {
+ // Can produce either a scalar value as a icmp of a phi, or a vector value,
+ // as a get.active.lane.mask intrinsic.
// Get first lane of vector induction variable.
Value *VIVElem0 = State.get(getOperand(0), VPLane(0));
// Get the original loop tripcount.
@@ -836,6 +849,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
{VIVElem0, ScalarTC}, nullptr, Name);
}
case VPInstruction::NumActiveLanes: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for NumActiveLanes");
Value *Op = State.get(getOperand(0));
auto *VecTy = cast<VectorType>(Op->getType());
assert(VecTy->getScalarSizeInBits() == 1 &&
@@ -870,6 +885,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
case VPInstruction::ExplicitVectorLength: {
// TODO: Restructure this code with an explicit remainder loop, vsetvli can
// be outside of the main loop.
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for ExplicitVectorLength");
Value *AVL = State.get(getOperand(0), /*NeedsSingleScalar=*/true);
// Compute EVL
assert(AVL->getType()->isIntegerTy() &&
@@ -884,6 +901,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
return EVL;
}
case VPInstruction::BranchOnCond: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for BranchOnCond");
Value *Cond = State.get(getOperand(0), VPLane(0));
// Replace the temporary unreachable terminator with a new conditional
// branch, hooking it up to backward destination for latch blocks now, and
@@ -901,11 +920,15 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Br;
}
case VPInstruction::Broadcast: {
+ assert(!GenerateSingleScalar &&
+ "Cannot generate scalar value for Broadcast");
return Builder.CreateVectorSplat(
State.VF, State.get(getOperand(0), /*NeedsSingleScalar=*/true),
"broadcast");
}
case VPInstruction::BuildStructVector: {
+ assert(!GenerateSingleScalar &&
+ "Cannot generate scalar value for BuildStructVector");
// For struct types, we need to build a new 'wide' struct type, where each
// element is widened, i.e., we create a struct of vectors.
auto *StructTy = cast<StructType>(getOperand(0)->getScalarType());
@@ -924,6 +947,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Res;
}
case VPInstruction::BuildVector: {
+ assert(!GenerateSingleScalar &&
+ "Cannot generate scalar value for BuildVector");
auto *ScalarTy = getOperand(0)->getScalarType();
auto NumOfElements = ElementCount::getFixed(getNumOperands());
Value *Res = PoisonValue::get(toVectorizedTy(ScalarTy, NumOfElements));
@@ -946,6 +971,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
Builder.getInt64(0));
}
case VPInstruction::ComputeReductionResult: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for ComputeReductionResult");
RecurKind RK = getRecurKind();
bool IsOrdered = isReductionOrdered();
bool IsInLoop = isReductionInLoop();
@@ -997,6 +1024,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
case VPInstruction::ExtractLastLane:
case VPInstruction::ExtractPenultimateElement: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for ExtractLane and "
+ "ExtractPenultimateElement");
unsigned Offset =
getOpcode() == VPInstruction::ExtractPenultimateElement ? 2 : 1;
Value *Res;
@@ -1014,36 +1044,30 @@ Value *VPInstruction::generate(VPTransformState &State) {
Res->setName(Name);
return Res;
}
- case VPInstruction::LogicalAnd: {
- Value *A = State.get(getOperand(0));
- Value *B = State.get(getOperand(1));
- return Builder.CreateLogicalAnd(A, B, Name);
- }
- case VPInstruction::LogicalOr: {
- Value *A = State.get(getOperand(0));
- Value *B = State.get(getOperand(1));
- return Builder.CreateLogicalOr(A, B, Name);
- }
case VPInstruction::PtrAdd: {
- assert((State.VF.isScalar() || vputils::onlyFirstLaneUsed(this)) &&
- "can only generate first lane for PtrAdd");
+ assert(GenerateSingleScalar && "Can only generate first lane for PtrAdd");
Value *Ptr = State.get(getOperand(0), VPLane(0));
Value *Addend = State.get(getOperand(1), VPLane(0));
return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags());
}
case VPInstruction::WidePtrAdd: {
+ assert(!GenerateSingleScalar &&
+ "Cannot generate scalar value for WidePtrAdd");
Value *Ptr =
State.get(getOperand(0), vputils::isSingleScalar(getOperand(0)));
Value *Addend = State.get(getOperand(1));
return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags());
}
case VPInstruction::AnyOf: {
+ assert(GenerateSingleScalar && "Can only generate first lane for AnyOf");
Value *Res = State.get(getOperand(0));
for (VPValue *Op : drop_begin(operands()))
Res = Builder.CreateOr(Res, State.get(Op));
return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res);
}
case VPInstruction::ExtractLane: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for ExtractLane");
assert(getNumOperands() != 2 && "ExtractLane from single source should be "
"simplified to ExtractElement.");
Value *LaneToExtract = State.get(getOperand(0), true);
@@ -1071,6 +1095,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Res;
}
case VPInstruction::FirstActiveLane: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for FirstActiveLane");
Type *Ty = this->getScalarType();
if (getNumOperands() == 1) {
Value *Mask = State.get(getOperand(0));
@@ -1107,10 +1133,15 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Res;
}
case VPInstruction::ResumeForEpilogue:
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for ResumeForEpilogue");
return State.get(getOperand(0), true);
case VPInstruction::Reverse:
+ assert(!GenerateSingleScalar && "Cannot generate scalar value for Reverse");
return Builder.CreateVectorReverse(State.get(getOperand(0)), "reverse");
case VPInstruction::ExtractLastActive: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for ExtractLastActive");
Value *Result = State.get(getOperand(0), /*NeedsSingleScalar=*/true);
for (unsigned Idx = 1; Idx < getNumOperands(); Idx += 2) {
Value *Data = State.get(getOperand(Idx));
@@ -1128,6 +1159,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
case VPInstruction::ExtractVectorForPart: {
+ assert(!GenerateSingleScalar &&
+ "Cannot generate scalar value for ExtractVectorForPart");
Value *Src = State.get(getOperand(0));
Type *DstTy = VectorType::get(getScalarType(), State.VF);
uint64_t Part = cast<VPConstantInt>(getOperand(1))->getZExtValue();
@@ -1139,9 +1172,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
DstTy, Src, Builder.getInt64(State.VF.getKnownMinValue() * Part), Name);
}
case VPInstruction::StepVector:
+ assert(!GenerateSingleScalar &&
+ "Cannot generate scalar value for StepVector");
return State.Builder.CreateStepVector(
VectorType::get(getScalarType(), State.VF));
case VPInstruction::Intrinsic: {
+ assert(GenerateSingleScalar &&
+ "Can only generate first lane for Intrinsic");
SmallVector<Value *, 2> Args;
for (VPValue *Op : drop_end(operands()))
Args.push_back(State.get(Op, /*NeedsSingleScalar=*/true));
@@ -1648,20 +1685,15 @@ void VPInstruction::execute(VPTransformState &State) {
assert(hasRequiredFlagsForOpcode(getOpcode(), getScalarType()) &&
"Opcode requires specific flags to be set");
State.Builder.setFastMathFlags(getFastMathFlagsOrNone());
- Value *GeneratedValue = generate(State);
+ bool GenerateSingleScalar = State.VF.isScalar() || doesGenerateSingleScalar();
+ Value *GeneratedValue = generate(State, GenerateSingleScalar);
if (!hasResult())
return;
assert(GeneratedValue && "generate must produce a value");
- bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
- (vputils::onlyFirstLaneUsed(this) ||
- isVectorToScalar() || isSingleScalar());
- assert((((GeneratedValue->getType()->isVectorTy() ||
- GeneratedValue->getType()->isStructTy()) ==
- !GeneratesPerFirstLaneOnly) ||
- State.VF.isScalar()) &&
+ assert(((GeneratedValue->getType()->isVectorTy() ||
+ GeneratedValue->getType()->isStructTy()) == !GenerateSingleScalar) &&
"scalar value but not only first lane defined");
- State.set(this, GeneratedValue,
- /*IsScalar*/ GeneratesPerFirstLaneOnly);
+ State.set(this, GeneratedValue, GenerateSingleScalar);
if (getOpcode() == VPInstruction::ResumeForEpilogue ||
getOpcode() == Instruction::Freeze) {
// FIXME: This is a workaround to enable reliable updates of the scalar loop
More information about the llvm-commits
mailing list