[llvm] c45e3a2 - [VPlan] Make VPIRFlags verification type aware, fix remaining holes. (#213081)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 06:15:23 PDT 2026
Author: Florian Hahn
Date: 2026-08-21T14:15:18+01:00
New Revision: c45e3a23d46e809cd27b4154f2ed0fb9b38ff9ee
URL: https://github.com/llvm/llvm-project/commit/c45e3a23d46e809cd27b4154f2ed0fb9b38ff9ee
DIFF: https://github.com/llvm/llvm-project/commit/c45e3a23d46e809cd27b4154f2ed0fb9b38ff9ee.diff
LOG: [VPlan] Make VPIRFlags verification type aware, fix remaining holes. (#213081)
Update hasRequiredFlagsForOpcode to also take the result type and use it
for flag verification for opcodes that can have flags depending on type
(like Phi, Select and Call).
Each of those sites is covered by existing tests, which assert without
the corresponding fix. The generated IR is unchanged, as the missing
flags are all empty by default.
Update all places we create selects and phis to properly pass through
the proper flags. VPBuilder helpers are extended to take flags as
std::optional, to make it easy to get the default flags automatically if
needed. It also ensures the callers have to pass in correct flags, if
they explicitly pass flags.
Currently depends on https://github.com/llvm/llvm-project/pull/212973.
PR: https://github.com/llvm/llvm-project/pull/213081
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index cca1d4dbc5720..f578a87c9d9cc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -28,6 +28,7 @@
#include "llvm/ADT/SmallSet.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/InstructionCost.h"
+#include <optional>
namespace {
class GeneratedRTChecks;
@@ -299,13 +300,18 @@ class VPBuilder {
return createNaryOp(VPInstruction::LogicalOr, {LHS, RHS}, DL, Name);
}
+ /// Create a select of \p TrueVal and \p FalseVal based on \p Cond, using the
+ /// default flags for the result type, unless \p Flags is set.
VPInstruction *createSelect(VPValue *Cond, VPValue *TrueVal,
VPValue *FalseVal,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "",
- const VPIRFlags &Flags = {}) {
- return tryInsertInstruction(new VPInstruction(
- Instruction::Select, {Cond, TrueVal, FalseVal}, Flags, {}, DL, Name));
+ std::optional<VPIRFlags> Flags = std::nullopt) {
+ return tryInsertInstruction(
+ new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
+ Flags.value_or(VPIRFlags::getDefaultFlags(
+ Instruction::Select, TrueVal->getScalarType())),
+ {}, DL, Name));
}
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
@@ -359,12 +365,18 @@ class VPBuilder {
GEPNoWrapFlags::none(), {}, DL, Name));
}
+ /// Create a phi with \p IncomingValues, using the default flags for the
+ /// result type, unless \p Flags is set.
VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues,
DebugLoc DL = DebugLoc::getUnknown(),
- const Twine &Name = "", const VPIRFlags &Flags = {},
+ const Twine &Name = "",
+ std::optional<VPIRFlags> Flags = std::nullopt,
Type *ResultTy = nullptr) {
- return tryInsertInstruction(
- new VPPhi(IncomingValues, Flags, DL, Name, ResultTy));
+ Type *ScalarTy = ResultTy ? ResultTy : IncomingValues[0]->getScalarType();
+ return tryInsertInstruction(new VPPhi(
+ IncomingValues,
+ Flags.value_or(VPIRFlags::getDefaultFlags(Instruction::PHI, ScalarTy)),
+ DL, Name, ResultTy));
}
VPWidenPHIRecipe *createWidenPhi(ArrayRef<VPValue *> IncomingValues,
@@ -411,18 +423,11 @@ class VPBuilder {
VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
Type *ResultTy, DebugLoc DL,
+ std::optional<VPIRFlags> Flags = std::nullopt,
const VPIRMetadata &Metadata = {}) {
return tryInsertInstruction(new VPInstructionWithType(
- Opcode, Op, ResultTy, VPIRFlags::getDefaultFlags(Opcode), Metadata,
- DL));
- }
-
- VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
- Type *ResultTy, DebugLoc DL,
- const VPIRFlags &Flags,
- const VPIRMetadata &Metadata = {}) {
- return tryInsertInstruction(
- new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));
+ Opcode, Op, ResultTy,
+ Flags.value_or(VPIRFlags::getDefaultFlags(Opcode)), Metadata, DL));
}
/// Create a scalar call to the intrinsic \p IntrinsicID with \p Operands, and
@@ -465,7 +470,7 @@ class VPBuilder {
return createScalarCast(CastOp, Op, ResultTy, DL);
}
- VPValue *createScalarFreeze(VPValue *Op, Type *ResultTy, DebugLoc DL) {
+ VPValue *createScalarFreeze(VPValue *Op, DebugLoc DL) {
return tryInsertInstruction(
new VPInstruction(Instruction::Freeze, Op, {}, {}, DL));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 93c9657f66270..deabeb97183c2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1106,8 +1106,10 @@ class VPIRFlags {
/// Returns true if the set flags are valid for \p Opcode.
LLVM_ABI_FOR_TEST bool flagsValidForOpcode(unsigned Opcode) const;
- /// Returns true if \p Opcode has its required flags set.
- LLVM_ABI_FOR_TEST bool hasRequiredFlagsForOpcode(unsigned Opcode) const;
+ /// Returns true if \p Opcode with scalar result type \p ResultTy has its
+ /// required flags set.
+ LLVM_ABI_FOR_TEST bool hasRequiredFlagsForOpcode(unsigned Opcode,
+ Type *ResultTy) const;
#endif
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1843,7 +1845,7 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
VPIRMetadata(Metadata), Opcode(Opcode) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
- assert(hasRequiredFlagsForOpcode(Opcode) &&
+ assert(hasRequiredFlagsForOpcode(Opcode, getScalarType()) &&
"Opcode requires specific flags to be set");
}
@@ -1903,7 +1905,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPIRMetadata(Metadata), Opcode(Opcode) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
- assert(hasRequiredFlagsForOpcode(Opcode) &&
+ assert(hasRequiredFlagsForOpcode(Opcode, ResultTy) &&
"Opcode requires specific flags to be set");
setUnderlyingValue(CI);
}
@@ -2980,6 +2982,8 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
return getMask(I)->getScalarType()->isIntegerTy(1);
}) &&
"masks must be a bool");
+ assert(hasRequiredFlagsForOpcode(Instruction::PHI, getScalarType()) &&
+ "blends require the flags of the phi they replace");
setUnderlyingValue(Phi);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a81feb7077651..ad963fe9ff158 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1363,7 +1363,7 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
if (isa<VPIRValue>(V))
continue;
VPValue *TailVal = Plan.getPoison(V->getScalarType());
- VPIRFlags Flags;
+ std::optional<VPIRFlags> Flags;
assert(llvm::count_if(Users, IsaPred<VPReductionPHIRecipe>) <= 1 &&
"Value used by more than two reduction phis?");
auto *RedIt = find_if(Users, IsaPred<VPReductionPHIRecipe>);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 0db4c9dc3e5ef..36a1129f5cefe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -1003,8 +1003,8 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
// TODO: Only freeze the required pointer (not both src and sink).
if (Check.NeedsFreeze) {
- Src = Builder.createScalarFreeze(Src, AddrType, DebugLoc::getUnknown());
- Sink = Builder.createScalarFreeze(Sink, AddrType, DebugLoc::getUnknown());
+ Src = Builder.createScalarFreeze(Src, DebugLoc::getUnknown());
+ Sink = Builder.createScalarFreeze(Sink, DebugLoc::getUnknown());
}
// TODO: Generate loop_dependence_raw_mask when there's a read-after-write
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b5524948c1bd9..306a4a259affe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -616,7 +616,7 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
- assert(hasRequiredFlagsForOpcode(getOpcode()) &&
+ assert(hasRequiredFlagsForOpcode(getOpcode(), getScalarType()) &&
"Opcode requires specific flags to be set");
assert((getNumOperandsForOpcode() == -1u ||
getNumOperandsForOpcode() == getNumOperands() ||
@@ -1585,7 +1585,7 @@ void VPInstruction::execute(VPTransformState &State) {
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
- assert(hasRequiredFlagsForOpcode(getOpcode()) &&
+ assert(hasRequiredFlagsForOpcode(getOpcode(), getScalarType()) &&
"Opcode requires specific flags to be set");
State.Builder.setFastMathFlags(getFastMathFlagsOrNone());
Value *GeneratedValue = generate(State);
@@ -2557,8 +2557,11 @@ VPIRFlags VPIRFlags::getDefaultFlags(unsigned Opcode, Type *ResultTy) {
case Instruction::FPTrunc:
return FastMathFlags();
case Instruction::Select:
- // Selects only have fast-math flags if they produce a floating-point value.
- if (ResultTy && FPMathOperator::isSupportedFloatingPointType(ResultTy))
+ case Instruction::PHI:
+ case Instruction::Call:
+ // Selects, phis and calls only have fast-math flags if they have a
+ // supported floating-point result type.
+ if (FPMathOperator::isSupportedFloatingPointType(ResultTy))
return FastMathFlags();
return VPIRFlags();
case Instruction::ICmp:
@@ -2612,7 +2615,8 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
llvm_unreachable("Unknown OperationType enum");
}
-bool VPIRFlags::hasRequiredFlagsForOpcode(unsigned Opcode) const {
+bool VPIRFlags::hasRequiredFlagsForOpcode(unsigned Opcode,
+ Type *ResultTy) const {
// Handle opcodes without default flags.
if (Opcode == Instruction::ICmp)
return OpType == OperationType::Cmp;
@@ -2621,7 +2625,7 @@ bool VPIRFlags::hasRequiredFlagsForOpcode(unsigned Opcode) const {
if (Opcode == VPInstruction::ComputeReductionResult)
return OpType == OperationType::ReductionOp;
- OperationType Required = getDefaultFlags(Opcode).OpType;
+ OperationType Required = getDefaultFlags(Opcode, ResultTy).OpType;
return Required == OperationType::Other || Required == OpType;
}
#endif
More information about the llvm-commits
mailing list