[llvm] [VPlan] Propagate FastMathFlags from phis to blends (PR #180226)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 8 20:19:00 PST 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/180226
>From 56a0954ae903bf87aef6cab389403e6f08c946eb Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sat, 7 Feb 2026 00:19:35 +0800
Subject: [PATCH 1/5] Precommit test
---
.../LoopVectorize/phi-with-fastflags.ll | 46 +++++++++++++++++++
1 file changed, 46 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
diff --git a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
new file mode 100644
index 0000000000000..4f4c1f3d0a12b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S < %s | FileCheck %s
+
+define void @f(ptr noalias %p, i1 %c) {
+; CHECK-LABEL: define void @f(
+; CHECK-SAME: ptr noalias [[P:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: store <4 x float> [[PREDPHI]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr float, ptr %p, i32 %iv
+ %x = load float, ptr %gep
+ br i1 %c, label %then, label %latch
+
+then:
+ br label %latch
+
+latch:
+ %phi = phi fast float [ %x, %loop ], [ 0.0, %then ]
+ store float %phi, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
>From a081ed378c0c502b150ab78a90e630bc3e3d7f1e Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sat, 7 Feb 2026 00:21:50 +0800
Subject: [PATCH 2/5] [VPlan] Propagate FastMathFlags from phis to blends
If a phi has fast math flags, we can propagate it to the widened select. To do this, this patch makes VPPhi and VPBlendRecipe subclasses of VPRecipeWithIRFlags, and propagates it through PlainCFGBuilder and VPPredicator.
Alive2 proofs for some of the FMFs (it looks like it can't reason about the full "fast" set yet)
nnan: https://alive2.llvm.org/ce/z/f0bRd4
nsz: https://alive2.llvm.org/ce/z/u9P96T
The actual motivation for this to eventually be able to move the special casing for tail folding in LoopVectorizationPlanner::addReductionResultComputation into the CFG in #176143, which requires passing through FMFs.
---
.../Vectorize/LoopVectorizationPlanner.h | 12 +++++-----
.../Transforms/Vectorize/LoopVectorize.cpp | 13 +++++------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 20 ++++++++++-------
.../Vectorize/VPlanConstruction.cpp | 10 +++++----
.../Transforms/Vectorize/VPlanPredicator.cpp | 2 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 13 +++++++----
.../Transforms/Vectorize/VPlanTransforms.cpp | 22 +++++++++----------
.../LoopVectorize/phi-with-fastflags.ll | 2 +-
.../Transforms/Vectorize/VPlanTest.cpp | 2 +-
.../Vectorize/VPlanVerifierTest.cpp | 4 ++--
11 files changed, 56 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 38ede9fff3650..15583454beffe 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -235,13 +235,11 @@ class VPBuilder {
VPInstruction *
createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
+ const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "",
std::optional<FastMathFlags> FMFs = std::nullopt) {
- if (!FMFs)
- return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}, DL,
- Name);
return tryInsertInstruction(new VPInstruction(
- Instruction::Select, {Cond, TrueVal, FalseVal}, *FMFs, {}, DL, Name));
+ Instruction::Select, {Cond, TrueVal, FalseVal}, Flags, {}, DL, Name));
}
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
@@ -291,9 +289,11 @@ class VPBuilder {
GEPNoWrapFlags::none(), {}, DL, Name));
}
- VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues, DebugLoc DL,
+ VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues,
+ const VPIRFlags &Flags = {},
+ DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
- return tryInsertInstruction(new VPPhi(IncomingValues, DL, Name));
+ return tryInsertInstruction(new VPPhi(IncomingValues, Flags, DL, Name));
}
VPValue *createElementCount(Type *Ty, ElementCount EC) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index db2c84d78fb15..ede05bfabb62a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7922,7 +7922,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
VPValue *One = Plan.getConstantInt(I->getType(), 1u);
auto *SafeRHS =
- Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
+ Builder.createSelect(Mask, Ops[1], One, {}, VPI->getDebugLoc());
Ops[1] = SafeRHS;
return new VPWidenRecipe(*I, Ops, *VPI, *VPI, VPI->getDebugLoc());
}
@@ -8482,12 +8482,11 @@ void LoopVectorizationPlanner::addReductionResultComputation(
if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
(!RR || !RR->isPartialReduction())) {
VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
- std::optional<FastMathFlags> FMFs =
- PhiTy->isFloatingPointTy()
- ? std::make_optional(RdxDesc.getFastMathFlags())
- : std::nullopt;
+ VPIRFlags Flags = PhiTy->isFloatingPointTy()
+ ? VPIRFlags(RdxDesc.getFastMathFlags())
+ : VPIRFlags();
NewExitingVPV =
- Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", FMFs);
+ Builder.createSelect(Cond, OrigExitingVPV, PhiR, Flags, {}, "");
OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
using namespace VPlanPatternMatch;
return match(
@@ -9082,7 +9081,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
ResumePhi = ScalarPHBuilder.createScalarPhi(
{VectorTC,
MainPlan.getVectorLoopRegion()->getCanonicalIV()->getStartValue()},
- {}, "vec.epilog.resume.val");
+ {}, {}, "vec.epilog.resume.val");
} else {
ResumePhi = cast<VPPhi>(&*ResumePhiIter);
if (MainScalarPH->begin() == MainScalarPH->end())
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 9b22c399d7acf..3bc23728276e6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -849,7 +849,7 @@ void VPRegionBlock::dissolveToCFGLoop() {
assert(this == getPlan()->getVectorLoopRegion() &&
"Canonical IV must be in the entry of the top-level loop region");
auto *ScalarR = VPBuilder(CanIV).createScalarPhi(
- {CanIV->getStartValue(), CanIV->getBackedgeValue()},
+ {CanIV->getStartValue(), CanIV->getBackedgeValue()}, {},
CanIV->getDebugLoc(), "index");
CanIV->replaceAllUsesWith(ScalarR);
CanIV->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d064a0c66a081..986132b48d973 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1046,7 +1046,8 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
: VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
static inline bool classof(const VPRecipeBase *R) {
- return R->getVPRecipeID() == VPRecipeBase::VPInstructionSC ||
+ return R->getVPRecipeID() == VPRecipeBase::VPBlendSC ||
+ R->getVPRecipeID() == VPRecipeBase::VPInstructionSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenCallSC ||
@@ -1510,8 +1511,9 @@ class VPPhiAccessors {
};
struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors {
- VPPhi(ArrayRef<VPValue *> Operands, DebugLoc DL, const Twine &Name = "")
- : VPInstruction(Instruction::PHI, Operands, {}, {}, DL, Name) {}
+ VPPhi(ArrayRef<VPValue *> Operands, const VPIRFlags &Flags, DebugLoc DL,
+ const Twine &Name = "")
+ : VPInstruction(Instruction::PHI, Operands, Flags, {}, DL, Name) {}
static inline bool classof(const VPUser *U) {
auto *VPI = dyn_cast<VPInstruction>(U);
@@ -1529,7 +1531,7 @@ struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors {
}
VPPhi *clone() override {
- auto *PhiR = new VPPhi(operands(), getDebugLoc(), getName());
+ auto *PhiR = new VPPhi(operands(), *this, getDebugLoc(), getName());
PhiR->setUnderlyingValue(getUnderlyingValue());
return PhiR;
}
@@ -2640,20 +2642,22 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
/// instructions.
-class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPSingleDefRecipe {
+class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
public:
/// The blend operation is a User of the incoming values and of their
/// respective masks, ordered [I0, M0, I1, M1, I2, M2, ...]. Note that M0 can
/// be omitted (implied by passing an odd number of operands) in which case
/// all other incoming values are merged into it.
- VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands, DebugLoc DL)
- : VPSingleDefRecipe(VPRecipeBase::VPBlendSC, Operands, Phi, DL) {
+ VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags, DebugLoc DL)
+ : VPRecipeWithIRFlags(VPRecipeBase::VPBlendSC, Operands, Flags, DL) {
assert(Operands.size() >= 2 && "Expected at least two operands!");
+ setUnderlyingValue(Phi);
}
VPBlendRecipe *clone() override {
return new VPBlendRecipe(cast_or_null<PHINode>(getUnderlyingValue()),
- operands(), getDebugLoc());
+ operands(), *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPBlendSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index eea9952243cd3..e5e1e58314fc4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -221,7 +221,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// Phi node's operands may not have been visited at this point. We create
// an empty VPInstruction that we will fix once the whole plain CFG has
// been built.
- NewR = VPIRBuilder.createScalarPhi({}, Phi->getDebugLoc(), "vec.phi");
+ NewR =
+ VPIRBuilder.createScalarPhi({}, *Phi, Phi->getDebugLoc(), "vec.phi");
NewR->setUnderlyingValue(Phi);
if (isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent()))) {
// Header phis need to be fixed after the VPBB for the latch has been
@@ -565,8 +566,9 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
for (const auto &[PhiR, ScalarPhiR] : zip_equal(
drop_begin(HeaderVPBB->phis()), Plan.getScalarHeader()->phis())) {
auto *VectorPhiR = cast<VPPhi>(&PhiR);
- auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
- {VectorPhiR, VectorPhiR->getOperand(0)}, VectorPhiR->getDebugLoc());
+ auto *ResumePhiR =
+ ScalarPHBuilder.createScalarPhi({VectorPhiR, VectorPhiR->getOperand(0)},
+ *VectorPhiR, VectorPhiR->getDebugLoc());
cast<VPIRPhi>(&ScalarPhiR)->addOperand(ResumePhiR);
}
}
@@ -1408,7 +1410,7 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
// Replace select for data.
VPValue *DataSelect =
- Builder.createSelect(AnyOf, Op1, Op2, SelectR->getDebugLoc());
+ Builder.createSelect(AnyOf, Op1, Op2, {}, SelectR->getDebugLoc());
SelectR->replaceAllUsesWith(DataSelect);
SelectR->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index e797cb091b3c2..64426483745e9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -259,7 +259,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue());
auto *Blend =
- new VPBlendRecipe(IRPhi, OperandsWithMask, PhiR->getDebugLoc());
+ new VPBlendRecipe(IRPhi, OperandsWithMask, *PhiR, PhiR->getDebugLoc());
Builder.insert(Blend);
PhiR->replaceAllUsesWith(Blend);
PhiR->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1d0c3d7a61f94..7e38f402ad1b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -578,6 +578,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0)));
Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
+ IRBuilderBase::FastMathFlagGuard FMFG(Builder);
+ if (hasFastMathFlags())
+ Builder.setFastMathFlags(getFastMathFlags());
return Builder.CreateSelect(Cond, Op1, Op2, Name);
}
case VPInstruction::ActiveLaneMask: {
@@ -1613,7 +1616,8 @@ void VPPhi::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";
printAsOperand(O, SlotTracker);
- O << " = phi ";
+ O << " = phi";
+ printFlags(O);
printPhiOperands(O, SlotTracker);
}
#endif
@@ -2123,7 +2127,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
Opcode == Instruction::FNeg || Opcode == Instruction::FDiv ||
Opcode == Instruction::FRem || Opcode == Instruction::FPExt ||
Opcode == Instruction::FPTrunc || Opcode == Instruction::Select ||
- Opcode == VPInstruction::WideIVStep ||
+ Opcode == Instruction::PHI || Opcode == VPInstruction::WideIVStep ||
Opcode == VPInstruction::ReductionStartVector;
case OperationType::FCmp:
return Opcode == Instruction::FCmp;
@@ -2733,19 +2737,20 @@ void VPBlendRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "BLEND ";
printAsOperand(O, SlotTracker);
O << " =";
+ printFlags(O);
if (getNumIncomingValues() == 1) {
// Not a User of any mask: not really blending, this is a
// single-predecessor phi.
- O << " ";
getIncomingValue(0)->printAsOperand(O, SlotTracker);
} else {
for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
- O << " ";
getIncomingValue(I)->printAsOperand(O, SlotTracker);
if (I == 0 && isNormalized())
continue;
O << "/";
getMask(I)->printAsOperand(O, SlotTracker);
+ if (I < E - 1)
+ O << " ";
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2479cb6243b48..aa457ae3cc916 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1792,7 +1792,7 @@ static void simplifyBlends(VPlan &Plan) {
auto *NewBlend =
new VPBlendRecipe(cast_or_null<PHINode>(Blend->getUnderlyingValue()),
- OperandsWithMask, Blend->getDebugLoc());
+ OperandsWithMask, *Blend, Blend->getDebugLoc());
NewBlend->insertBefore(&R);
VPValue *DeadMask = Blend->getMask(StartIndex);
@@ -3162,7 +3162,7 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
Builder.setInsertPoint(Header, Header->getFirstNonPhi());
VPValue *PrevEVL = Builder.createScalarPhi(
- {MaxEVL, &EVL}, DebugLoc::getUnknown(), "prev.evl");
+ {MaxEVL, &EVL}, {}, DebugLoc::getUnknown(), "prev.evl");
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
@@ -3275,14 +3275,14 @@ void VPlanTransforms::addExplicitVectorLength(
// Create the AVL (application vector length), starting from TC -> 0 in steps
// of EVL.
VPPhi *AVLPhi = Builder.createScalarPhi(
- {Plan.getTripCount()}, DebugLoc::getCompilerGenerated(), "avl");
+ {Plan.getTripCount()}, {}, DebugLoc::getCompilerGenerated(), "avl");
VPValue *AVL = AVLPhi;
if (MaxSafeElements) {
// Support for MaxSafeDist for correct loop emission.
VPValue *AVLSafe = Plan.getConstantInt(CanIVTy, *MaxSafeElements);
VPValue *Cmp = Builder.createICmp(ICmpInst::ICMP_ULT, AVL, AVLSafe);
- AVL = Builder.createSelect(Cmp, AVL, AVLSafe, DebugLoc::getUnknown(),
+ AVL = Builder.createSelect(Cmp, AVL, AVLSafe, {}, DebugLoc::getUnknown(),
"safe_avl");
}
auto *VPEVL = Builder.createNaryOp(VPInstruction::ExplicitVectorLength, AVL,
@@ -3340,9 +3340,9 @@ void VPlanTransforms::canonicalizeEVLLoops(VPlan &Plan) {
VPValue *EVLIncrement = EVLPhi->getBackedgeValue();
// Convert EVLPhi to concrete recipe.
- auto *ScalarR =
- VPBuilder(EVLPhi).createScalarPhi({EVLPhi->getStartValue(), EVLIncrement},
- EVLPhi->getDebugLoc(), "evl.based.iv");
+ auto *ScalarR = VPBuilder(EVLPhi).createScalarPhi(
+ {EVLPhi->getStartValue(), EVLIncrement}, {}, EVLPhi->getDebugLoc(),
+ "evl.based.iv");
EVLPhi->replaceAllUsesWith(ScalarR);
EVLPhi->eraseFromParent();
@@ -3817,7 +3817,7 @@ static void expandVPWidenPointerInduction(VPWidenPointerInductionRecipe *R,
DebugLoc DL = R->getDebugLoc();
// Build a scalar pointer phi.
- VPPhi *ScalarPtrPhi = Builder.createScalarPhi(Start, DL, "pointer.phi");
+ VPPhi *ScalarPtrPhi = Builder.createScalarPhi(Start, {}, DL, "pointer.phi");
// Create actual address geps that use the pointer phi as base and a
// vectorized version of the step value (<step*0, ..., step*N>) as offset.
@@ -3940,7 +3940,7 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I)
Select = Builder.createSelect(Blend->getMask(I),
Blend->getIncomingValue(I), Select,
- R.getDebugLoc(), "predphi");
+ *Blend, R.getDebugLoc(), "predphi");
Blend->replaceAllUsesWith(Select);
ToRemove.push_back(Blend);
}
@@ -4708,7 +4708,7 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
for (unsigned I = 1; I < Group.size(); ++I) {
VPValue *Mask = Group[I]->getMask();
VPValue *Value = Group[I]->getOperand(0);
- SelectedValue = Builder.createSelect(Mask, Value, SelectedValue,
+ SelectedValue = Builder.createSelect(Mask, Value, SelectedValue, {},
Group[I]->getDebugLoc());
}
@@ -5554,7 +5554,7 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
auto *Cmp =
MiddleBuilder.createICmp(CmpInst::ICMP_NE, ReducedIV, Sentinel, ExitDL);
VPInstruction *NewRdxResult = MiddleBuilder.createSelect(
- Cmp, ReducedIV, PhiR->getStartValue(), ExitDL);
+ Cmp, ReducedIV, PhiR->getStartValue(), {}, ExitDL);
RdxResult->replaceAllUsesWith(NewRdxResult);
RdxResult->eraseFromParent();
diff --git a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
index 4f4c1f3d0a12b..2ce2c932de2cb 100644
--- a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
+++ b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
@@ -12,7 +12,7 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select fast i1 [[C]], <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
; CHECK-NEXT: store <4 x float> [[PREDPHI]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 52634abcfb670..5f33f0593e05f 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1147,7 +1147,7 @@ TEST_F(VPRecipeTest, CastVPBlendRecipeToVPUser) {
Args.push_back(I1);
Args.push_back(I2);
Args.push_back(M2);
- VPBlendRecipe Recipe(Phi, Args, {});
+ VPBlendRecipe Recipe(Phi, Args, {}, {});
checkVPRecipeCastImpl<VPBlendRecipe, VPUser>(&Recipe);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index eeae72920854b..f07ee94105ff8 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -113,7 +113,7 @@ TEST_F(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
VPInstruction *BranchOnCond =
new VPInstruction(VPInstruction::BranchOnCond, {CanIV});
- auto *Blend = new VPBlendRecipe(Phi, {DefI, Plan.getTrue()}, {});
+ auto *Blend = new VPBlendRecipe(Phi, {DefI, Plan.getTrue()}, {}, {});
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
@@ -166,7 +166,7 @@ TEST_F(VPVerifierTest, VPPhiIncomingValueDoesntDominateIncomingBlock) {
VPInstruction *DefI =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
- VPPhi *Phi = new VPPhi({DefI}, {});
+ VPPhi *Phi = new VPPhi({DefI}, {}, {});
VPBB2->appendRecipe(Phi);
VPBB2->appendRecipe(DefI);
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
>From c0c9f55a7fa3cc1623d255649e7b8f06cf5ce76b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sat, 7 Feb 2026 00:31:53 +0800
Subject: [PATCH 3/5] Don't propagate flags to resume phis yet, need test
coverage
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index e5e1e58314fc4..0246d3d7b4f54 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -566,9 +566,8 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
for (const auto &[PhiR, ScalarPhiR] : zip_equal(
drop_begin(HeaderVPBB->phis()), Plan.getScalarHeader()->phis())) {
auto *VectorPhiR = cast<VPPhi>(&PhiR);
- auto *ResumePhiR =
- ScalarPHBuilder.createScalarPhi({VectorPhiR, VectorPhiR->getOperand(0)},
- *VectorPhiR, VectorPhiR->getDebugLoc());
+ auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
+ {VectorPhiR, VectorPhiR->getOperand(0)}, {}, VectorPhiR->getDebugLoc());
cast<VPIRPhi>(&ScalarPhiR)->addOperand(ResumePhiR);
}
}
>From 5c14ec291b82ad0335e66aa7fbfa0bb1e588dda2 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sat, 7 Feb 2026 01:16:01 +0800
Subject: [PATCH 4/5] Address comments
* Move Flags to end of builder arguments
* Remove unused old FMF arguments (woops)
* Add VPlan test
* Fix opcode ordering in flagsValidForOpcode
---
.../Vectorize/LoopVectorizationPlanner.h | 13 ++--
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +-
.../Vectorize/VPlanConstruction.cpp | 6 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 20 ++---
.../LoopVectorize/phi-with-fastflags-vplan.ll | 74 +++++++++++++++++++
7 files changed, 100 insertions(+), 26 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/phi-with-fastflags-vplan.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 15583454beffe..0b8796f646ae3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -233,11 +233,11 @@ class VPBuilder {
return createNaryOp(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name);
}
- VPInstruction *
- createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
- const VPIRFlags &Flags = {},
- DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "",
- std::optional<FastMathFlags> FMFs = std::nullopt) {
+ VPInstruction *createSelect(VPValue *Cond, VPValue *TrueVal,
+ VPValue *FalseVal,
+ DebugLoc DL = DebugLoc::getUnknown(),
+ const Twine &Name = "",
+ const VPIRFlags &Flags = {}) {
return tryInsertInstruction(new VPInstruction(
Instruction::Select, {Cond, TrueVal, FalseVal}, Flags, {}, DL, Name));
}
@@ -290,9 +290,8 @@ class VPBuilder {
}
VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues,
- const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown(),
- const Twine &Name = "") {
+ const Twine &Name = "", const VPIRFlags &Flags = {}) {
return tryInsertInstruction(new VPPhi(IncomingValues, Flags, DL, Name));
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ede05bfabb62a..4116005ecc314 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7922,7 +7922,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
VPValue *One = Plan.getConstantInt(I->getType(), 1u);
auto *SafeRHS =
- Builder.createSelect(Mask, Ops[1], One, {}, VPI->getDebugLoc());
+ Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
Ops[1] = SafeRHS;
return new VPWidenRecipe(*I, Ops, *VPI, *VPI, VPI->getDebugLoc());
}
@@ -8486,7 +8486,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
? VPIRFlags(RdxDesc.getFastMathFlags())
: VPIRFlags();
NewExitingVPV =
- Builder.createSelect(Cond, OrigExitingVPV, PhiR, Flags, {}, "");
+ Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", Flags);
OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
using namespace VPlanPatternMatch;
return match(
@@ -9081,7 +9081,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
ResumePhi = ScalarPHBuilder.createScalarPhi(
{VectorTC,
MainPlan.getVectorLoopRegion()->getCanonicalIV()->getStartValue()},
- {}, {}, "vec.epilog.resume.val");
+ {}, "vec.epilog.resume.val");
} else {
ResumePhi = cast<VPPhi>(&*ResumePhiIter);
if (MainScalarPH->begin() == MainScalarPH->end())
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 3bc23728276e6..9b22c399d7acf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -849,7 +849,7 @@ void VPRegionBlock::dissolveToCFGLoop() {
assert(this == getPlan()->getVectorLoopRegion() &&
"Canonical IV must be in the entry of the top-level loop region");
auto *ScalarR = VPBuilder(CanIV).createScalarPhi(
- {CanIV->getStartValue(), CanIV->getBackedgeValue()}, {},
+ {CanIV->getStartValue(), CanIV->getBackedgeValue()},
CanIV->getDebugLoc(), "index");
CanIV->replaceAllUsesWith(ScalarR);
CanIV->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 0246d3d7b4f54..e37ce32988ada 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -222,7 +222,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// an empty VPInstruction that we will fix once the whole plain CFG has
// been built.
NewR =
- VPIRBuilder.createScalarPhi({}, *Phi, Phi->getDebugLoc(), "vec.phi");
+ VPIRBuilder.createScalarPhi({}, Phi->getDebugLoc(), "vec.phi", *Phi);
NewR->setUnderlyingValue(Phi);
if (isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent()))) {
// Header phis need to be fixed after the VPBB for the latch has been
@@ -567,7 +567,7 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
drop_begin(HeaderVPBB->phis()), Plan.getScalarHeader()->phis())) {
auto *VectorPhiR = cast<VPPhi>(&PhiR);
auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
- {VectorPhiR, VectorPhiR->getOperand(0)}, {}, VectorPhiR->getDebugLoc());
+ {VectorPhiR, VectorPhiR->getOperand(0)}, VectorPhiR->getDebugLoc());
cast<VPIRPhi>(&ScalarPhiR)->addOperand(ResumePhiR);
}
}
@@ -1409,7 +1409,7 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
// Replace select for data.
VPValue *DataSelect =
- Builder.createSelect(AnyOf, Op1, Op2, {}, SelectR->getDebugLoc());
+ Builder.createSelect(AnyOf, Op1, Op2, SelectR->getDebugLoc());
SelectR->replaceAllUsesWith(DataSelect);
SelectR->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7e38f402ad1b6..4c68ab75c11cf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2126,8 +2126,9 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
Opcode == Instruction::FMul || Opcode == Instruction::FSub ||
Opcode == Instruction::FNeg || Opcode == Instruction::FDiv ||
Opcode == Instruction::FRem || Opcode == Instruction::FPExt ||
- Opcode == Instruction::FPTrunc || Opcode == Instruction::Select ||
- Opcode == Instruction::PHI || Opcode == VPInstruction::WideIVStep ||
+ Opcode == Instruction::FPTrunc || Opcode == Instruction::PHI ||
+ Opcode == Instruction::Select ||
+ Opcode == VPInstruction::WideIVStep ||
Opcode == VPInstruction::ReductionStartVector;
case OperationType::FCmp:
return Opcode == Instruction::FCmp;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index aa457ae3cc916..b0ff25ab869eb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3162,7 +3162,7 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
Builder.setInsertPoint(Header, Header->getFirstNonPhi());
VPValue *PrevEVL = Builder.createScalarPhi(
- {MaxEVL, &EVL}, {}, DebugLoc::getUnknown(), "prev.evl");
+ {MaxEVL, &EVL}, DebugLoc::getUnknown(), "prev.evl");
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
@@ -3275,14 +3275,14 @@ void VPlanTransforms::addExplicitVectorLength(
// Create the AVL (application vector length), starting from TC -> 0 in steps
// of EVL.
VPPhi *AVLPhi = Builder.createScalarPhi(
- {Plan.getTripCount()}, {}, DebugLoc::getCompilerGenerated(), "avl");
+ {Plan.getTripCount()}, DebugLoc::getCompilerGenerated(), "avl");
VPValue *AVL = AVLPhi;
if (MaxSafeElements) {
// Support for MaxSafeDist for correct loop emission.
VPValue *AVLSafe = Plan.getConstantInt(CanIVTy, *MaxSafeElements);
VPValue *Cmp = Builder.createICmp(ICmpInst::ICMP_ULT, AVL, AVLSafe);
- AVL = Builder.createSelect(Cmp, AVL, AVLSafe, {}, DebugLoc::getUnknown(),
+ AVL = Builder.createSelect(Cmp, AVL, AVLSafe, DebugLoc::getUnknown(),
"safe_avl");
}
auto *VPEVL = Builder.createNaryOp(VPInstruction::ExplicitVectorLength, AVL,
@@ -3340,9 +3340,9 @@ void VPlanTransforms::canonicalizeEVLLoops(VPlan &Plan) {
VPValue *EVLIncrement = EVLPhi->getBackedgeValue();
// Convert EVLPhi to concrete recipe.
- auto *ScalarR = VPBuilder(EVLPhi).createScalarPhi(
- {EVLPhi->getStartValue(), EVLIncrement}, {}, EVLPhi->getDebugLoc(),
- "evl.based.iv");
+ auto *ScalarR =
+ VPBuilder(EVLPhi).createScalarPhi({EVLPhi->getStartValue(), EVLIncrement},
+ EVLPhi->getDebugLoc(), "evl.based.iv");
EVLPhi->replaceAllUsesWith(ScalarR);
EVLPhi->eraseFromParent();
@@ -3817,7 +3817,7 @@ static void expandVPWidenPointerInduction(VPWidenPointerInductionRecipe *R,
DebugLoc DL = R->getDebugLoc();
// Build a scalar pointer phi.
- VPPhi *ScalarPtrPhi = Builder.createScalarPhi(Start, {}, DL, "pointer.phi");
+ VPPhi *ScalarPtrPhi = Builder.createScalarPhi(Start, DL, "pointer.phi");
// Create actual address geps that use the pointer phi as base and a
// vectorized version of the step value (<step*0, ..., step*N>) as offset.
@@ -3940,7 +3940,7 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I)
Select = Builder.createSelect(Blend->getMask(I),
Blend->getIncomingValue(I), Select,
- *Blend, R.getDebugLoc(), "predphi");
+ R.getDebugLoc(), "predphi", *Blend);
Blend->replaceAllUsesWith(Select);
ToRemove.push_back(Blend);
}
@@ -4708,7 +4708,7 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
for (unsigned I = 1; I < Group.size(); ++I) {
VPValue *Mask = Group[I]->getMask();
VPValue *Value = Group[I]->getOperand(0);
- SelectedValue = Builder.createSelect(Mask, Value, SelectedValue, {},
+ SelectedValue = Builder.createSelect(Mask, Value, SelectedValue,
Group[I]->getDebugLoc());
}
@@ -5554,7 +5554,7 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
auto *Cmp =
MiddleBuilder.createICmp(CmpInst::ICMP_NE, ReducedIV, Sentinel, ExitDL);
VPInstruction *NewRdxResult = MiddleBuilder.createSelect(
- Cmp, ReducedIV, PhiR->getStartValue(), {}, ExitDL);
+ Cmp, ReducedIV, PhiR->getStartValue(), ExitDL);
RdxResult->replaceAllUsesWith(NewRdxResult);
RdxResult->eraseFromParent();
diff --git a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags-vplan.ll b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags-vplan.ll
new file mode 100644
index 0000000000000..e52a339c7f211
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags-vplan.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -debug-only=loop-vectorize -disable-output -S %s 2>&1 | FileCheck %s
+
+define void @f(ptr noalias %p, i1 %c) {
+; CHECK-LABEL: 'f'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%p>, vp<[[VP4]]>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN ir<%x> = load vp<[[VP5]]>
+; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x>ir<0.000000e+00>/ir<%c>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep>
+; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%phi>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1024>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %gep = getelementptr float, ptr %p, i32 %iv
+; CHECK-NEXT: IR %x = load float, ptr %gep, align 4
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr float, ptr %p, i32 %iv
+ %x = load float, ptr %gep
+ br i1 %c, label %then, label %latch
+
+then:
+ br label %latch
+
+latch:
+ %phi = phi fast float [ %x, %loop ], [ 0.0, %then ]
+ store float %phi, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %done = icmp eq i32 %iv.next, 1024
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
>From f937e5cab499d29d9bbeb0556626ac3ab872a6a2 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 9 Feb 2026 12:18:33 +0800
Subject: [PATCH 5/5] Address comments, fix blend printing
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 11 +++++------
.../LoopVectorize/phi-with-fastflags-vplan.ll | 2 +-
2 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4c68ab75c11cf..b9858b0de4910 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -578,10 +578,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0)));
Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
- IRBuilderBase::FastMathFlagGuard FMFG(Builder);
- if (hasFastMathFlags())
- Builder.setFastMathFlags(getFastMathFlags());
- return Builder.CreateSelect(Cond, Op1, Op2, Name);
+ FastMathFlags FMFs =
+ hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
+ return Builder.CreateSelectFMF(Cond, Op1, Op2, FMFs, Name);
}
case VPInstruction::ActiveLaneMask: {
// Get first lane of vector induction variable.
@@ -2745,13 +2744,13 @@ void VPBlendRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
getIncomingValue(0)->printAsOperand(O, SlotTracker);
} else {
for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
+ if (I != 0)
+ O << " ";
getIncomingValue(I)->printAsOperand(O, SlotTracker);
if (I == 0 && isNormalized())
continue;
O << "/";
getMask(I)->printAsOperand(O, SlotTracker);
- if (I < E - 1)
- O << " ";
}
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags-vplan.ll b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags-vplan.ll
index e52a339c7f211..f685bf806929c 100644
--- a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags-vplan.ll
@@ -22,7 +22,7 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%p>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep>
; CHECK-NEXT: WIDEN ir<%x> = load vp<[[VP5]]>
-; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x>ir<0.000000e+00>/ir<%c>
+; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x> ir<0.000000e+00>/ir<%c>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep>
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%phi>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
More information about the llvm-commits
mailing list