[llvm] 6a5375f - [VPlan] Plumb recurrence FMFs through VPReductionPHIRecipe via VPIRFlags. NFC (#181694)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 19 03:23:53 PST 2026
Author: Luke Lau
Date: 2026-02-19T11:23:47Z
New Revision: 6a5375fbce1992cb58a31a8c94d59ecb35c752a9
URL: https://github.com/llvm/llvm-project/commit/6a5375fbce1992cb58a31a8c94d59ecb35c752a9
DIFF: https://github.com/llvm/llvm-project/commit/6a5375fbce1992cb58a31a8c94d59ecb35c752a9.diff
LOG: [VPlan] Plumb recurrence FMFs through VPReductionPHIRecipe via VPIRFlags. NFC (#181694)
In order to be able to create selects for reduction phis through tail
folding in foldTailByMasking (#176143), make VPReductionPHIRecipe an
instance of VPIRFlags and plumb the FMFs from the original RdxDesc.
This allows us to remove more uses of the RecurrenceDescriptor in
addReductionResultComputation, which should help untie it from
LoopVectorizationLegality.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 01433fe4c4ba7..6299e8c2dbd32 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8501,11 +8501,8 @@ void LoopVectorizationPlanner::addReductionResultComputation(
if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
(!RR || !RR->isPartialReduction())) {
VPValue *Cond = vputils::findHeaderMask(*Plan);
- VPIRFlags Flags = PhiTy->isFloatingPointTy()
- ? VPIRFlags(RdxDesc.getFastMathFlags())
- : VPIRFlags();
NewExitingVPV =
- Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", Flags);
+ Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", *PhiR);
OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
using namespace VPlanPatternMatch;
return match(
@@ -8552,12 +8549,8 @@ void LoopVectorizationPlanner::addReductionResultComputation(
Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
{Start, NewVal, NewExitingVPV}, ExitDL);
} else {
- FastMathFlags FMFs =
- RecurrenceDescriptor::isFloatingPointRecurrenceKind(RecurrenceKind)
- ? RdxDesc.getFastMathFlags()
- : FastMathFlags();
VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
- FMFs);
+ PhiR->getFastMathFlags());
FinalReductionResult =
Builder.createNaryOp(VPInstruction::ComputeReductionResult,
{NewExitingVPV}, Flags, ExitDL);
@@ -8643,20 +8636,18 @@ void LoopVectorizationPlanner::addReductionResultComputation(
continue;
}
- RecurKind RK = RdxDesc.getRecurrenceKind();
+ RecurKind RK = PhiR->getRecurrenceKind();
if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) &&
!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
!RecurrenceDescriptor::isMinMaxRecurrenceKind(RK) &&
!RecurrenceDescriptor::isFindLastRecurrenceKind(RK))) {
VPBuilder PHBuilder(Plan->getVectorPreheader());
VPValue *Iden = Plan->getOrAddLiveIn(
- getRecurrenceIdentity(RK, PhiTy, RdxDesc.getFastMathFlags()));
+ getRecurrenceIdentity(RK, PhiTy, PhiR->getFastMathFlags()));
auto *ScaleFactorVPV = Plan->getConstantInt(32, 1);
VPValue *StartV = PHBuilder.createNaryOp(
VPInstruction::ReductionStartVector,
- {PhiR->getStartValue(), Iden, ScaleFactorVPV},
- PhiTy->isFloatingPointTy() ? RdxDesc.getFastMathFlags()
- : FastMathFlags());
+ {PhiR->getStartValue(), Iden, ScaleFactorVPV}, *PhiR);
PhiR->setOperand(0, StartV);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 94b7b0079f84e..a0c23df0b3c38 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2615,7 +2615,7 @@ inline ReductionStyle getReductionStyle(bool InLoop, bool Ordered,
/// A recipe for handling reduction phis. The start value is the first operand
/// of the recipe and the incoming value from the backedge is the second
/// operand.
-class VPReductionPHIRecipe : public VPHeaderPHIRecipe {
+class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPIRFlags {
/// The recurrence kind of the reduction.
const RecurKind Kind;
@@ -2631,9 +2631,10 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe {
/// Create a new VPReductionPHIRecipe for the reduction \p Phi.
VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
VPValue &BackedgeValue, ReductionStyle Style,
+ const VPIRFlags &Flags,
bool HasUsesOutsideReductionChain = false)
: VPHeaderPHIRecipe(VPRecipeBase::VPReductionPHISC, Phi, &Start),
- Kind(Kind), Style(Style),
+ VPIRFlags(Flags), Kind(Kind), Style(Style),
HasUsesOutsideReductionChain(HasUsesOutsideReductionChain) {
addOperand(&BackedgeValue);
}
@@ -2643,7 +2644,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe {
VPReductionPHIRecipe *clone() override {
return new VPReductionPHIRecipe(
dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
- *getOperand(0), *getBackedgeValue(), Style,
+ *getOperand(0), *getBackedgeValue(), Style, *this,
HasUsesOutsideReductionChain);
}
@@ -4011,10 +4012,9 @@ class LLVM_ABI_FOR_TEST VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
~VPScalarIVStepsRecipe() override = default;
VPScalarIVStepsRecipe *clone() override {
- return new VPScalarIVStepsRecipe(
- getOperand(0), getOperand(1), getOperand(2), InductionOpcode,
- hasFastMathFlags() ? getFastMathFlags() : FastMathFlags(),
- getDebugLoc());
+ return new VPScalarIVStepsRecipe(getOperand(0), getOperand(1),
+ getOperand(2), InductionOpcode,
+ getFastMathFlags(), getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPScalarIVStepsSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 969c302dccfc7..1af7392b904da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -728,6 +728,8 @@ void VPlanTransforms::createHeaderPhiRecipes(
Phi, RdxDesc.getRecurrenceKind(), *Start, *BackedgeValue,
getReductionStyle(InLoopReductions.contains(Phi), UseOrderedReductions,
ScaleFactor),
+ Phi->getType()->isFloatingPointTy() ? RdxDesc.getFastMathFlags()
+ : VPIRFlags(),
RdxDesc.hasUsesOutsideReductionChain());
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d31545ebe720d..33cb1509565d5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -368,8 +368,11 @@ void VPIRFlags::intersectFlags(const VPIRFlags &Other) {
FastMathFlags VPIRFlags::getFastMathFlags() const {
assert((OpType == OperationType::FPMathOp || OpType == OperationType::FCmp ||
- OpType == OperationType::ReductionOp) &&
+ OpType == OperationType::ReductionOp ||
+ OpType == OperationType::Other) &&
"recipe doesn't have fast math flags");
+ if (OpType == OperationType::Other)
+ return FastMathFlags();
const FastMathFlagsTy &F = getFMFsRef();
FastMathFlags Res;
Res.setAllowReassoc(F.AllowReassoc);
@@ -585,9 +588,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0)));
Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
- FastMathFlags FMFs =
- hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
- return Builder.CreateSelectFMF(Cond, Op1, Op2, FMFs, Name);
+ return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlags(), Name);
}
case VPInstruction::ActiveLaneMask: {
// Get first lane of vector induction variable.
@@ -752,8 +753,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
RdxParts[Part] = State.get(getOperand(Part), IsInLoop);
IRBuilderBase::FastMathFlagGuard FMFG(Builder);
- if (hasFastMathFlags())
- Builder.setFastMathFlags(getFastMathFlags());
+ Builder.setFastMathFlags(getFastMathFlags());
// Reduce multiple operands into one.
Value *ReducedPartRdx = RdxParts[0];
@@ -1946,10 +1946,8 @@ static InstructionCost getCostForIntrinsics(Intrinsic::ID ID,
}
// TODO: Rework TTI interface to avoid reliance on underlying IntrinsicInst.
- FastMathFlags FMF =
- R.hasFastMathFlags() ? R.getFastMathFlags() : FastMathFlags();
IntrinsicCostAttributes CostAttrs(
- ID, RetTy, Arguments, ParamTys, FMF,
+ ID, RetTy, Arguments, ParamTys, R.getFastMathFlags(),
dyn_cast_or_null<IntrinsicInst>(R.getUnderlyingValue()),
InstructionCost::getInvalid(), &Ctx.TLI);
return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);
@@ -2541,8 +2539,7 @@ void VPDerivedIVRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
void VPScalarIVStepsRecipe::execute(VPTransformState &State) {
// Fast-math-flags propagate from the original induction instruction.
IRBuilder<>::FastMathFlagGuard FMFG(State.Builder);
- if (hasFastMathFlags())
- State.Builder.setFastMathFlags(getFastMathFlags());
+ State.Builder.setFastMathFlags(getFastMathFlags());
/// Compute scalar induction steps. \p ScalarIV is the scalar induction
/// variable on which to base the steps, \p Step is the size of the step.
@@ -4620,7 +4617,8 @@ void VPReductionPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN-REDUCTION-PHI ";
printAsOperand(O, SlotTracker);
- O << " = phi ";
+ O << " = phi";
+ printFlags(O);
printOperands(O, SlotTracker);
if (getVFScaleFactor() > 1)
O << " (VF scaled by 1/" << getVFScaleFactor() << ")";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bb1a91ec8c963..cf501e99a1942 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5757,7 +5757,8 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
auto *NewPhiR = new VPReductionPHIRecipe(
cast<PHINode>(PhiR->getUnderlyingInstr()), RecurKind::FindIV, *Sentinel,
- *BackedgeVal, RdxUnordered{1}, PhiR->hasUsesOutsideReductionChain());
+ *BackedgeVal, RdxUnordered{1}, {},
+ PhiR->hasUsesOutsideReductionChain());
NewPhiR->insertBefore(PhiR);
PhiR->replaceAllUsesWith(NewPhiR);
PhiR->eraseFromParent();
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
index e7bf014fdbd80..cd95f5cb99507 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
@@ -22,7 +22,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi fast vp<[[VP3]]>, ir<%red.next>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -95,7 +95,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi fast vp<[[VP3]]>, ir<%red.next>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -170,7 +170,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi vp<[[VP3]]>, ir<%muladd>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi nnan ninf nsz vp<[[VP3]]>, ir<%muladd>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
More information about the llvm-commits
mailing list