[llvm] [VPlan] Make VPIRFlags verification type aware, fix remaining holes. (PR #213081)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 02:16:43 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/213081
>From 94ae2d60b462c81f48d41e7a17f6ccd4acf539e1 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 30 Jul 2026 13:59:05 +0100
Subject: [PATCH 1/2] [VPlan] Make VPIRFlags verification type aware, fix
remaining holes.
Update hasRequiredFlagsForOpcode to also take the result type and use it
for flag verification for opcodes that can have flags depending on type
(like Phi, Select and Call).
Each of those sites is covered by existing tests, which assert without
the corresponding fix. The generated IR is unchanged, as the missing
flags are all empty by default.
Update all places we create selects and phis to properly pass through
the proper flags. VPBuilder helpers are extended to take flags as
std::optional, to make it easy to get the default flags automatically if
needed. It also ensures the callers have to pass in correct flags, if
they explicitly pass flags.
---
.../Vectorize/LoopVectorizationPlanner.h | 24 ++++++++++++++-----
llvm/lib/Transforms/Vectorize/VPlan.h | 10 +++++---
.../Vectorize/VPlanConstruction.cpp | 2 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 ++++++++-----
4 files changed, 36 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d488607a0c7dc..eeef3258696fd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -28,6 +28,7 @@
#include "llvm/ADT/SmallSet.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/InstructionCost.h"
+#include <optional>
namespace {
class GeneratedRTChecks;
@@ -299,13 +300,18 @@ class VPBuilder {
return createNaryOp(VPInstruction::LogicalOr, {LHS, RHS}, DL, Name);
}
+ /// Create a select of \p TrueVal and \p FalseVal based on \p Cond, using the
+ /// default flags for the result type, unless \p Flags is set.
VPInstruction *createSelect(VPValue *Cond, VPValue *TrueVal,
VPValue *FalseVal,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "",
- const VPIRFlags &Flags = {}) {
- return tryInsertInstruction(new VPInstruction(
- Instruction::Select, {Cond, TrueVal, FalseVal}, Flags, {}, DL, Name));
+ std::optional<VPIRFlags> Flags = std::nullopt) {
+ return tryInsertInstruction(
+ new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
+ Flags.value_or(VPIRFlags::getDefaultFlags(
+ Instruction::Select, TrueVal->getScalarType())),
+ {}, DL, Name));
}
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
@@ -359,12 +365,18 @@ class VPBuilder {
GEPNoWrapFlags::none(), {}, DL, Name));
}
+ /// Create a phi with \p IncomingValues, using the default flags for the
+ /// result type, unless \p Flags is set.
VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues,
DebugLoc DL = DebugLoc::getUnknown(),
- const Twine &Name = "", const VPIRFlags &Flags = {},
+ const Twine &Name = "",
+ std::optional<VPIRFlags> Flags = std::nullopt,
Type *ResultTy = nullptr) {
- return tryInsertInstruction(
- new VPPhi(IncomingValues, Flags, DL, Name, ResultTy));
+ Type *ScalarTy = ResultTy ? ResultTy : IncomingValues[0]->getScalarType();
+ return tryInsertInstruction(new VPPhi(
+ IncomingValues,
+ Flags.value_or(VPIRFlags::getDefaultFlags(Instruction::PHI, ScalarTy)),
+ DL, Name, ResultTy));
}
VPWidenPHIRecipe *createWidenPhi(ArrayRef<VPValue *> IncomingValues,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 4bb18e46c64ad..dc51ee423bf0d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1107,8 +1107,10 @@ class VPIRFlags {
/// Returns true if the set flags are valid for \p Opcode.
LLVM_ABI_FOR_TEST bool flagsValidForOpcode(unsigned Opcode) const;
- /// Returns true if \p Opcode has its required flags set.
- LLVM_ABI_FOR_TEST bool hasRequiredFlagsForOpcode(unsigned Opcode) const;
+ /// Returns true if \p Opcode with scalar result type \p ResultTy has its
+ /// required flags set.
+ LLVM_ABI_FOR_TEST bool hasRequiredFlagsForOpcode(unsigned Opcode,
+ Type *ResultTy) const;
#endif
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1899,7 +1901,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPIRMetadata(Metadata), Opcode(Opcode) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
- assert(hasRequiredFlagsForOpcode(Opcode) &&
+ assert(hasRequiredFlagsForOpcode(Opcode, ResultTy) &&
"Opcode requires specific flags to be set");
setUnderlyingValue(CI);
}
@@ -2976,6 +2978,8 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
return getMask(I)->getScalarType()->isIntegerTy(1);
}) &&
"masks must be a bool");
+ assert(hasRequiredFlagsForOpcode(Instruction::PHI, getScalarType()) &&
+ "blends require the flags of the phi they replace");
setUnderlyingValue(Phi);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4c09e10fe6ea0..1f88c6b37de9e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1403,7 +1403,7 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
if (isa<VPIRValue>(V))
continue;
VPValue *TailVal = Plan.getPoison(V->getScalarType());
- VPIRFlags Flags;
+ std::optional<VPIRFlags> Flags;
assert(llvm::count_if(Users, IsaPred<VPReductionPHIRecipe>) <= 1 &&
"Value used by more than two reduction phis?");
auto *RedIt = find_if(Users, IsaPred<VPReductionPHIRecipe>);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b5524948c1bd9..4641ffb0aea1f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -616,7 +616,7 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
- assert(hasRequiredFlagsForOpcode(getOpcode()) &&
+ assert(hasRequiredFlagsForOpcode(getOpcode(), getScalarType()) &&
"Opcode requires specific flags to be set");
assert((getNumOperandsForOpcode() == -1u ||
getNumOperandsForOpcode() == getNumOperands() ||
@@ -1585,7 +1585,7 @@ void VPInstruction::execute(VPTransformState &State) {
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
- assert(hasRequiredFlagsForOpcode(getOpcode()) &&
+ assert(hasRequiredFlagsForOpcode(getOpcode(), getScalarType()) &&
"Opcode requires specific flags to be set");
State.Builder.setFastMathFlags(getFastMathFlagsOrNone());
Value *GeneratedValue = generate(State);
@@ -2557,8 +2557,11 @@ VPIRFlags VPIRFlags::getDefaultFlags(unsigned Opcode, Type *ResultTy) {
case Instruction::FPTrunc:
return FastMathFlags();
case Instruction::Select:
- // Selects only have fast-math flags if they produce a floating-point value.
- if (ResultTy && FPMathOperator::isSupportedFloatingPointType(ResultTy))
+ case Instruction::PHI:
+ case Instruction::Call:
+ // Selects, phis and calls only have fast-math flags if they produce a
+ // floating-point value.
+ if (FPMathOperator::isSupportedFloatingPointType(ResultTy))
return FastMathFlags();
return VPIRFlags();
case Instruction::ICmp:
@@ -2612,7 +2615,8 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
llvm_unreachable("Unknown OperationType enum");
}
-bool VPIRFlags::hasRequiredFlagsForOpcode(unsigned Opcode) const {
+bool VPIRFlags::hasRequiredFlagsForOpcode(unsigned Opcode,
+ Type *ResultTy) const {
// Handle opcodes without default flags.
if (Opcode == Instruction::ICmp)
return OpType == OperationType::Cmp;
@@ -2621,7 +2625,7 @@ bool VPIRFlags::hasRequiredFlagsForOpcode(unsigned Opcode) const {
if (Opcode == VPInstruction::ComputeReductionResult)
return OpType == OperationType::ReductionOp;
- OperationType Required = getDefaultFlags(Opcode).OpType;
+ OperationType Required = getDefaultFlags(Opcode, ResultTy).OpType;
return Required == OperationType::Other || Required == OpType;
}
#endif
>From ad8ce61ec6fe0fd6e6e28be72c97ad8c24b9722a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 18 Aug 2026 10:11:55 +0100
Subject: [PATCH 2/2] !fixup use std::optional<VPIRFlags> in a few more places
---
.../Vectorize/LoopVectorizationPlanner.h | 15 ++++-----------
llvm/lib/Transforms/Vectorize/VPlanLowering.cpp | 4 ++--
2 files changed, 6 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index eeef3258696fd..526f98c6e8550 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -416,18 +416,11 @@ class VPBuilder {
VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
Type *ResultTy, DebugLoc DL,
+ std::optional<VPIRFlags> Flags = std::nullopt,
const VPIRMetadata &Metadata = {}) {
return tryInsertInstruction(new VPInstructionWithType(
- Opcode, Op, ResultTy, VPIRFlags::getDefaultFlags(Opcode), Metadata,
- DL));
- }
-
- VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
- Type *ResultTy, DebugLoc DL,
- const VPIRFlags &Flags,
- const VPIRMetadata &Metadata = {}) {
- return tryInsertInstruction(
- new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));
+ Opcode, Op, ResultTy,
+ Flags.value_or(VPIRFlags::getDefaultFlags(Opcode)), Metadata, DL));
}
/// Create a scalar call to the intrinsic \p IntrinsicID with \p Operands, and
@@ -470,7 +463,7 @@ class VPBuilder {
return createScalarCast(CastOp, Op, ResultTy, DL);
}
- VPValue *createScalarFreeze(VPValue *Op, Type *ResultTy, DebugLoc DL) {
+ VPValue *createScalarFreeze(VPValue *Op, DebugLoc DL) {
return tryInsertInstruction(
new VPInstruction(Instruction::Freeze, Op, {}, {}, DL));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index d78a6e8c68681..430eba133f80a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -1005,8 +1005,8 @@ VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
// TODO: Only freeze the required pointer (not both src and sink).
if (Check.NeedsFreeze) {
- Src = Builder.createScalarFreeze(Src, AddrType, DebugLoc::getUnknown());
- Sink = Builder.createScalarFreeze(Sink, AddrType, DebugLoc::getUnknown());
+ Src = Builder.createScalarFreeze(Src, DebugLoc::getUnknown());
+ Sink = Builder.createScalarFreeze(Sink, DebugLoc::getUnknown());
}
// TODO: Generate loop_dependence_raw_mask when there's a read-after-write
More information about the llvm-commits
mailing list