[llvm] [LV] Remove DataAndControlFlowWithoutRuntimeCheck. NFC (PR #183762)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 2 04:34:47 PST 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/183762
>From 17bb026c41ef2dafb524f2d45d1a87c64d080dfb Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 27 Feb 2026 20:04:38 +0800
Subject: [PATCH 1/4] [LV] Remove DataAndControlFlowWithoutRuntimeCheck. NFC
Stacked on #183729
After #144963 and #183292 we never emit the runtime check, so DataAndControlFlowWithoutRuntimeCheck is equivalent to DataAndControlFlow.
With that we only need to store one tail folding style instead of two, because we don't need to distinguish whether or not the IV update overflows (to a non-zero value)
---
.../llvm/Analysis/TargetTransformInfo.h | 12 +---
.../llvm/Analysis/TargetTransformInfoImpl.h | 3 +-
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 5 +-
llvm/lib/Analysis/TargetTransformInfo.cpp | 5 +-
.../AArch64/AArch64TargetTransformInfo.h | 11 +---
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 3 +-
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 3 +-
.../Target/RISCV/RISCVTargetTransformInfo.h | 3 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 46 +++++----------
.../Transforms/Vectorize/VPlanTransforms.cpp | 56 ++++---------------
.../Transforms/Vectorize/VPlanTransforms.h | 9 +--
11 files changed, 40 insertions(+), 116 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 18ae6a005d972..476fa057410f8 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -251,10 +251,6 @@ enum class TailFoldingStyle {
/// active.lane.mask to calculate the mask for the next iteration. If the
/// increment overflows, the mask is no longer correct.
DataAndControlFlow,
- /// Use predicate to control both data and control flow, but modify
- /// the trip count so that a runtime overflow check can be avoided
- /// and such that the scalar epilogue loop can always be removed.
- DataAndControlFlowWithoutRuntimeCheck,
/// Use predicated EVL instructions for tail-folding.
/// Indicates that VP intrinsics should be used.
DataWithEVL,
@@ -757,13 +753,7 @@ class TargetTransformInfo {
LLVM_ABI bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const;
/// Query the target what the preferred style of tail folding is.
- /// \param IVUpdateMayOverflow Tells whether it is known if the IV update
- /// may (or will never) overflow for the suggested VF/UF in the given loop.
- /// Targets can use this information to select a more optimal tail folding
- /// style. The value conservatively defaults to true, such that no assumptions
- /// are made on overflow.
- LLVM_ABI TailFoldingStyle
- getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) const;
+ LLVM_ABI TailFoldingStyle getPreferredTailFoldingStyle() const;
// Parameters that control the loop peeling transformation
struct PeelingPreferences {
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index e062b70be6b59..6e5d7d308be21 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -278,8 +278,7 @@ class TargetTransformInfoImplBase {
return false;
}
- virtual TailFoldingStyle
- getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
+ virtual TailFoldingStyle getPreferredTailFoldingStyle() const {
return TailFoldingStyle::DataWithoutLaneMask;
}
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 6dcb6f0062a08..1ccc6deb9cebd 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -802,9 +802,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return BaseT::preferPredicateOverEpilogue(TFI);
}
- TailFoldingStyle
- getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) const override {
- return BaseT::getPreferredTailFoldingStyle(IVUpdateMayOverflow);
+ TailFoldingStyle getPreferredTailFoldingStyle() const override {
+ return BaseT::getPreferredTailFoldingStyle();
}
std::optional<Instruction *>
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 0f97edc424d7e..957f428d9c09c 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -386,9 +386,8 @@ bool TargetTransformInfo::preferPredicateOverEpilogue(
return TTIImpl->preferPredicateOverEpilogue(TFI);
}
-TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle(
- bool IVUpdateMayOverflow) const {
- return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow);
+TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle() const {
+ return TTIImpl->getPreferredTailFoldingStyle();
}
std::optional<Instruction *>
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index f247e9e49e23f..c7716020da13c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -464,14 +464,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
return ST->hasSVE() ? 5 : 0;
}
- TailFoldingStyle
- getPreferredTailFoldingStyle(bool IVUpdateMayOverflow) const override {
- if (ST->hasSVE())
- return IVUpdateMayOverflow
- ? TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck
- : TailFoldingStyle::DataAndControlFlow;
-
- return TailFoldingStyle::DataWithoutLaneMask;
+ TailFoldingStyle getPreferredTailFoldingStyle() const override {
+ return ST->hasSVE() ? TailFoldingStyle::DataAndControlFlow
+ : TailFoldingStyle::DataWithoutLaneMask;
}
bool preferFixedOverScalableIfEqualCost(bool IsEpilogue) const override;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index a7766d91af3f2..2a8ca56b17a92 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -2670,8 +2670,7 @@ bool ARMTTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
*LVL->getDominatorTree());
}
-TailFoldingStyle
-ARMTTIImpl::getPreferredTailFoldingStyle(bool IVUpdateMayOverflow) const {
+TailFoldingStyle ARMTTIImpl::getPreferredTailFoldingStyle() const {
if (!ST->hasMVEIntegerOps() || !EnableTailPredication)
return TailFoldingStyle::DataWithoutLaneMask;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 94804152d96ec..75a32d81a44aa 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -432,8 +432,7 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE) const override;
- TailFoldingStyle
- getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) const override;
+ TailFoldingStyle getPreferredTailFoldingStyle() const override;
void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
TTI::PeelingPreferences &PP) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 424f9fe52c59e..97a364d741314 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -122,8 +122,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const override {
return ST->hasVInstructions();
}
- TailFoldingStyle
- getPreferredTailFoldingStyle(bool IVUpdateMayOverflow) const override {
+ TailFoldingStyle getPreferredTailFoldingStyle() const override {
return ST->hasVInstructions() ? TailFoldingStyle::DataWithEVL
: TailFoldingStyle::None;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0debab4a2a0ee..c6e0247c1c9ea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -243,9 +243,6 @@ static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control",
"Create lane mask using active.lane.mask intrinsic, and use "
"it for both data and control flow"),
- clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck,
- "data-and-control-without-rt-check",
- "Similar to data-and-control, but remove the runtime check"),
clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl",
"Use predicated EVL instructions for tail folding. If EVL "
"is unsupported, fallback to data-without-lane-mask.")));
@@ -1327,11 +1324,10 @@ class LoopVectorizationCostModel {
}
/// Returns the TailFoldingStyle that is best for the current loop.
- TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
+ TailFoldingStyle getTailFoldingStyle() const {
if (!ChosenTailFoldingStyle)
return TailFoldingStyle::None;
- return IVUpdateMayOverflow ? ChosenTailFoldingStyle->first
- : ChosenTailFoldingStyle->second;
+ return *ChosenTailFoldingStyle;
}
/// Selects and saves TailFoldingStyle for 2 options - if IV update may
@@ -1341,20 +1337,16 @@ class LoopVectorizationCostModel {
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC) {
assert(!ChosenTailFoldingStyle && "Tail folding must not be selected yet.");
if (!Legal->canFoldTailByMasking()) {
- ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
+ ChosenTailFoldingStyle = TailFoldingStyle::None;
return;
}
// Default to TTI preference, but allow command line override.
- ChosenTailFoldingStyle = {
- TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/true),
- TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false)};
+ ChosenTailFoldingStyle = TTI.getPreferredTailFoldingStyle();
if (ForceTailFoldingStyle.getNumOccurrences())
- ChosenTailFoldingStyle = {ForceTailFoldingStyle.getValue(),
- ForceTailFoldingStyle.getValue()};
+ ChosenTailFoldingStyle = ForceTailFoldingStyle.getValue();
- if (ChosenTailFoldingStyle->first != TailFoldingStyle::DataWithEVL &&
- ChosenTailFoldingStyle->second != TailFoldingStyle::DataWithEVL)
+ if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
return;
// Override EVL styles if needed.
// FIXME: Investigate opportunity for fixed vector factor.
@@ -1366,10 +1358,9 @@ class LoopVectorizationCostModel {
// if it's allowed, or DataWithoutLaneMask otherwise.
if (ScalarEpilogueStatus == CM_ScalarEpilogueAllowed ||
ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate)
- ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
+ ChosenTailFoldingStyle = TailFoldingStyle::None;
else
- ChosenTailFoldingStyle = {TailFoldingStyle::DataWithoutLaneMask,
- TailFoldingStyle::DataWithoutLaneMask};
+ ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
LLVM_DEBUG(
dbgs() << "LV: Preference for VP intrinsics indicated. Will "
@@ -1381,8 +1372,6 @@ class LoopVectorizationCostModel {
/// Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailByMasking() const {
- // TODO: check if it is possible to check for None style independent of
- // IVUpdateMayOverflow flag in getTailFoldingStyle.
return getTailFoldingStyle() != TailFoldingStyle::None;
}
@@ -1393,8 +1382,7 @@ class LoopVectorizationCostModel {
return false;
TailFoldingStyle TF = getTailFoldingStyle();
- return TF == TailFoldingStyle::DataAndControlFlow ||
- TF == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
+ return TF == TailFoldingStyle::DataAndControlFlow;
}
/// Return maximum safe number of elements to be processed per vector
@@ -1608,8 +1596,7 @@ class LoopVectorizationCostModel {
/// Control finally chosen tail folding style. The first element is used if
/// the IV update may overflow, the second element - if it does not.
- std::optional<std::pair<TailFoldingStyle, TailFoldingStyle>>
- ChosenTailFoldingStyle;
+ std::optional<TailFoldingStyle> ChosenTailFoldingStyle;
/// true if scalable vectorization is supported and enabled.
std::optional<bool> IsScalableVectorizationAllowed;
@@ -2098,13 +2085,11 @@ class GeneratedRTChecks {
static bool useActiveLaneMask(TailFoldingStyle Style) {
return Style == TailFoldingStyle::Data ||
- Style == TailFoldingStyle::DataAndControlFlow ||
- Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
+ Style == TailFoldingStyle::DataAndControlFlow;
}
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style) {
- return Style == TailFoldingStyle::DataAndControlFlow ||
- Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
+ return Style == TailFoldingStyle::DataAndControlFlow;
}
// Return true if \p OuterLp is an outer loop annotated with hints for explicit
@@ -8194,7 +8179,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
for (ElementCount VF : Range)
IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&CM, VF);
- TailFoldingStyle Style = CM.getTailFoldingStyle(IVUpdateMayOverflow);
+ TailFoldingStyle Style = CM.getTailFoldingStyle();
// Use NUW for the induction increment if we proved that it won't overflow in
// the vector loop or when not folding the tail. In the later case, we know
// that the canonical induction increment will not overflow as the vector trip
@@ -8406,10 +8391,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// TODO: Move checks to VPlanTransforms::addActiveLaneMask once
// TailFoldingStyle is visible there.
bool ForControlFlow = useActiveLaneMaskForControlFlow(Style);
- bool WithoutRuntimeCheck =
- Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
- VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
- WithoutRuntimeCheck);
+ VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow);
}
VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues, PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3c2396626b5d7..4206dc1143b7c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2825,33 +2825,23 @@ void VPlanTransforms::optimize(VPlan &Plan) {
// dropped from the canonical IV increment. Return the created
// VPActiveLaneMaskPHIRecipe.
//
-// The function uses the following definitions:
-//
-// %TripCount = DataWithControlFlowWithoutRuntimeCheck ?
-// calculate-trip-count-minus-VF (original TC) : original TC
-// %IncrementValue = DataWithControlFlowWithoutRuntimeCheck ?
-// CanonicalIVPhi : CanonicalIVIncrement
-// %StartV is the canonical induction start value.
-//
// The function adds the following recipes:
//
// vector.ph:
-// %TripCount = calculate-trip-count-minus-VF (original TC)
-// [if DataWithControlFlowWithoutRuntimeCheck]
-// %EntryInc = canonical-iv-increment-for-part %StartV
-// %EntryALM = active-lane-mask %EntryInc, %TripCount
+// %EntryInc = canonical-iv-increment-for-part CanonicalIVStart
+// %EntryALM = active-lane-mask %EntryInc, TC
//
// vector.body:
// ...
// %P = active-lane-mask-phi [ %EntryALM, %vector.ph ], [ %ALM, %vector.body ]
// ...
-// %InLoopInc = canonical-iv-increment-for-part %IncrementValue
-// %ALM = active-lane-mask %InLoopInc, TripCount
+// %InLoopInc = canonical-iv-increment-for-part CanonicalIVIncrement
+// %ALM = active-lane-mask %InLoopInc, TC
// %Negated = Not %ALM
// branch-on-cond %Negated
//
-static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
- VPlan &Plan, bool DataAndControlFlowWithoutRuntimeCheck) {
+static VPActiveLaneMaskPHIRecipe *
+addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
auto *CanonicalIVPHI = TopRegion->getCanonicalIV();
@@ -2871,24 +2861,8 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
// Create the ActiveLaneMask instruction using the correct start values.
VPValue *TC = Plan.getTripCount();
- VPValue *VFxUF = &Plan.getVFxUF();
VPValue *VF = &Plan.getVF();
- VPValue *TripCount, *IncrementValue;
- if (!DataAndControlFlowWithoutRuntimeCheck) {
- // When the loop is guarded by a runtime overflow check for the loop
- // induction variable increment by VF, we can increment the value before
- // the get.active.lane mask and use the unmodified tripcount.
- IncrementValue = CanonicalIVIncrement;
- TripCount = TC;
- } else {
- // When avoiding a runtime check, the active.lane.mask inside the loop
- // uses a modified trip count and the induction variable increment is
- // done after the active.lane.mask intrinsic is called.
- IncrementValue = CanonicalIVPHI;
- TripCount = Builder.createNaryOp(VPInstruction::CalculateTripCountMinusVF,
- {TC, VFxUF}, DL);
- }
auto *EntryIncrement = Builder.createOverflowingOp(
VPInstruction::CanonicalIVIncrementForPart, {StartV, VF}, {false, false},
DL, "index.part.next");
@@ -2912,10 +2886,10 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
Builder.setInsertPoint(OriginalTerminator);
auto *InLoopIncrement = Builder.createOverflowingOp(
VPInstruction::CanonicalIVIncrementForPart,
- {IncrementValue, &Plan.getVF()}, {false, false}, DL);
+ {CanonicalIVIncrement, &Plan.getVF()}, {false, false}, DL);
auto *ALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
- {InLoopIncrement, TripCount, ALMMultiplier},
- DL, "active.lane.mask.next");
+ {InLoopIncrement, TC, ALMMultiplier}, DL,
+ "active.lane.mask.next");
LaneMaskPhi->addOperand(ALM);
// Replace the original terminator with BranchOnCond. We have to invert the
@@ -2926,14 +2900,8 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
return LaneMaskPhi;
}
-void VPlanTransforms::addActiveLaneMask(
- VPlan &Plan, bool UseActiveLaneMaskForControlFlow,
- bool DataAndControlFlowWithoutRuntimeCheck) {
- assert((!DataAndControlFlowWithoutRuntimeCheck ||
- UseActiveLaneMaskForControlFlow) &&
- "DataAndControlFlowWithoutRuntimeCheck implies "
- "UseActiveLaneMaskForControlFlow");
-
+void VPlanTransforms::addActiveLaneMask(VPlan &Plan,
+ bool UseActiveLaneMaskForControlFlow) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
auto *FoundWidenCanonicalIVUser = find_if(
LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
@@ -2944,7 +2912,7 @@ void VPlanTransforms::addActiveLaneMask(
cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
VPSingleDefRecipe *LaneMask;
if (UseActiveLaneMaskForControlFlow) {
- LaneMask = addVPLaneMaskPhiAndUpdateExitBranch(Plan, false);
+ LaneMask = addVPLaneMaskPhiAndUpdateExitBranch(Plan);
} else {
VPBuilder B = VPBuilder::getToInsertAfter(WideCanonicalIV);
VPValue *ALMMultiplier =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 0dce486cb1c2c..16f7ae2daeb5e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -253,14 +253,9 @@ struct VPlanTransforms {
/// Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an
/// (active-lane-mask recipe, wide canonical IV, trip-count). If \p
/// UseActiveLaneMaskForControlFlow is true, introduce an
- /// VPActiveLaneMaskPHIRecipe. If \p DataAndControlFlowWithoutRuntimeCheck is
- /// true, no minimum-iteration runtime check will be created (during skeleton
- /// creation) and instead it is handled using active-lane-mask. \p
- /// DataAndControlFlowWithoutRuntimeCheck implies \p
- /// UseActiveLaneMaskForControlFlow.
+ /// VPActiveLaneMaskPHIRecipe.
static void addActiveLaneMask(VPlan &Plan,
- bool UseActiveLaneMaskForControlFlow,
- bool DataAndControlFlowWithoutRuntimeCheck);
+ bool UseActiveLaneMaskForControlFlow);
/// Insert truncates and extends for any truncated recipe. Redundant casts
/// will be folded later.
>From 31f4c5baf98c55a9632a08eb2e7b57e64aaeaf66 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 2 Mar 2026 20:24:46 +0800
Subject: [PATCH 2/4] Address review comments, remove some outdated comments
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 137c3b5025978..4a84b458c9cec 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1325,13 +1325,10 @@ class LoopVectorizationCostModel {
/// Returns the TailFoldingStyle that is best for the current loop.
TailFoldingStyle getTailFoldingStyle() const {
- if (!ChosenTailFoldingStyle)
- return TailFoldingStyle::None;
- return *ChosenTailFoldingStyle;
+ return ChosenTailFoldingStyle ? *ChosenTailFoldingStyle : TailFoldingStyle::None;
}
- /// Selects and saves TailFoldingStyle for 2 options - if IV update may
- /// overflow or not.
+ /// Selects and saves TailFoldingStyle.
/// \param IsScalableVF true if scalable vector factors enabled.
/// \param UserIC User specific interleave count.
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC) {
@@ -1381,8 +1378,7 @@ class LoopVectorizationCostModel {
if (!EnableWideActiveLaneMask)
return false;
- TailFoldingStyle TF = getTailFoldingStyle();
- return TF == TailFoldingStyle::DataAndControlFlow;
+ return getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
}
/// Return maximum safe number of elements to be processed per vector
@@ -1594,8 +1590,7 @@ class LoopVectorizationCostModel {
/// iterations to execute in the scalar loop.
ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
- /// Control finally chosen tail folding style. The first element is used if
- /// the IV update may overflow, the second element - if it does not.
+ /// Control finally chosen tail folding style.
std::optional<TailFoldingStyle> ChosenTailFoldingStyle;
/// true if scalable vectorization is supported and enabled.
>From 960d8298cf0427376ddcc77054ed4aabb6a84199 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 2 Mar 2026 20:26:28 +0800
Subject: [PATCH 3/4] setTailFoldingStyles -> setTailFoldingStyle
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4a84b458c9cec..5cc676f9eccf6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1325,13 +1325,14 @@ class LoopVectorizationCostModel {
/// Returns the TailFoldingStyle that is best for the current loop.
TailFoldingStyle getTailFoldingStyle() const {
- return ChosenTailFoldingStyle ? *ChosenTailFoldingStyle : TailFoldingStyle::None;
+ return ChosenTailFoldingStyle ? *ChosenTailFoldingStyle
+ : TailFoldingStyle::None;
}
/// Selects and saves TailFoldingStyle.
/// \param IsScalableVF true if scalable vector factors enabled.
/// \param UserIC User specific interleave count.
- void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC) {
+ void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC) {
assert(!ChosenTailFoldingStyle && "Tail folding must not be selected yet.");
if (!Legal->canFoldTailByMasking()) {
ChosenTailFoldingStyle = TailFoldingStyle::None;
@@ -3707,7 +3708,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// by masking.
// FIXME: look for a smaller MaxVF that does divide TC rather than masking.
bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
- setTailFoldingStyles(ContainsScalableVF, UserIC);
+ setTailFoldingStyle(ContainsScalableVF, UserIC);
if (foldTailByMasking()) {
if (foldTailWithEVL()) {
LLVM_DEBUG(
>From d13671bd4c8306c7916e3ebeace959f404a8be82 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 2 Mar 2026 20:34:29 +0800
Subject: [PATCH 4/4] Remove std::optional
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5cc676f9eccf6..36c8c0560c9eb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1325,15 +1325,15 @@ class LoopVectorizationCostModel {
/// Returns the TailFoldingStyle that is best for the current loop.
TailFoldingStyle getTailFoldingStyle() const {
- return ChosenTailFoldingStyle ? *ChosenTailFoldingStyle
- : TailFoldingStyle::None;
+ return ChosenTailFoldingStyle;
}
/// Selects and saves TailFoldingStyle.
/// \param IsScalableVF true if scalable vector factors enabled.
/// \param UserIC User specific interleave count.
void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC) {
- assert(!ChosenTailFoldingStyle && "Tail folding must not be selected yet.");
+ assert(ChosenTailFoldingStyle == TailFoldingStyle::None &&
+ "Tail folding must not be selected yet.");
if (!Legal->canFoldTailByMasking()) {
ChosenTailFoldingStyle = TailFoldingStyle::None;
return;
@@ -1592,7 +1592,7 @@ class LoopVectorizationCostModel {
ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
/// Control finally chosen tail folding style.
- std::optional<TailFoldingStyle> ChosenTailFoldingStyle;
+ TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
/// true if scalable vectorization is supported and enabled.
std::optional<bool> IsScalableVectorizationAllowed;
More information about the llvm-commits
mailing list