[llvm] [VPlan] Use VPInstruction for VPScalarPHIRecipe. (NFCI) (PR #129767)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 12 03:56:55 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/129767
>From b05ac8042bbed4d7e96fb328d9dd66b6720395ad Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 4 Mar 2025 11:41:55 +0000
Subject: [PATCH 1/5] [VPlan] Use VPInstruction for VPScalarPHIRecipe. (NFCI)
Now that all phi nodes manage their incoming blocks through the
VPlan-predecessors, there should be no need for having a dedicate
recipe, it should be sufficient to allow PHI opcodes in VPInstruction.
Follow-ups will also migrate VPWidenPHIRecipe and possibly others,
building on top of https://github.com/llvm/llvm-project/pull/129388.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 8 ++-
llvm/lib/Transforms/Vectorize/VPlan.h | 50 +++----------------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 20 +++++---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 45 ++++++++---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 11 ++--
.../Transforms/Vectorize/VPlanVerifier.cpp | 5 +-
.../LoopVectorize/AArch64/vplan-printing.ll | 2 +-
.../RISCV/riscv-vector-reverse.ll | 4 +-
.../RISCV/vplan-vp-select-intrinsics.ll | 4 +-
.../LoopVectorize/vplan-predicate-switch.ll | 2 +-
10 files changed, 63 insertions(+), 88 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 563784e4af924..b6e9cf5c64133 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1074,9 +1074,15 @@ void VPlan::execute(VPTransformState *State) {
Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
continue;
}
+ if (auto *VPI = dyn_cast<VPInstruction>(&R)) {
+ Value *Phi = State->get(VPI, true);
+ Value *Val = State->get(VPI->getOperand(1), true);
+ cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
+ continue;
+ }
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
- bool NeedsScalar = isa<VPScalarPHIRecipe>(PhiR) ||
+ bool NeedsScalar =
(isa<VPReductionPHIRecipe>(PhiR) &&
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
Value *Phi = State->get(PhiR, NeedsScalar);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b1288c42b20f2..9bd0b1907e36f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -441,9 +441,7 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
bool mayHaveSideEffects() const;
/// Returns true for PHI-like recipes.
- bool isPhi() const {
- return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
- }
+ bool isPhi() const;
/// Returns true if the recipe may read from memory.
bool mayReadFromMemory() const;
@@ -1887,45 +1885,6 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe,
#endif
};
-/// Recipe to generate a scalar PHI. Used to generate code for recipes that
-/// produce scalar header phis, including VPCanonicalIVPHIRecipe and
-/// VPEVLBasedIVPHIRecipe.
-class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
- std::string Name;
-
-public:
- VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
- StringRef Name)
- : VPHeaderPHIRecipe(VPDef::VPScalarPHISC, nullptr, Start, DL),
- Name(Name.str()) {
- addOperand(BackedgeValue);
- }
-
- ~VPScalarPHIRecipe() override = default;
-
- VPScalarPHIRecipe *clone() override {
- llvm_unreachable("cloning not implemented yet");
- }
-
- VP_CLASSOF_IMPL(VPDef::VPScalarPHISC)
-
- /// Generate the phi/select nodes.
- void execute(VPTransformState &State) override;
-
- /// Returns true if the recipe only uses the first lane of operand \p Op.
- bool onlyFirstLaneUsed(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-};
-
/// A recipe for widened phis. Incoming values are operands of the recipe and
/// their operand index corresponds to the incoming predecessor block. If the
/// recipe is placed in an entry block to a (non-replicate) region, it must have
@@ -3004,6 +2963,13 @@ class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
return 0;
}
+ /// Returns true if the recipe only uses the first lane of operand \p Op.
+ bool onlyFirstLaneUsed(const VPValue *Op) const override {
+ assert(is_contained(operands(), Op) &&
+ "Op must be an operand of the recipe");
+ return true;
+ }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 6f6875f0e5e0e..6bdb284b0677e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -71,6 +71,10 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
}
case VPInstruction::ExplicitVectorLength:
return Type::getIntNTy(Ctx, 32);
+ case Instruction::PHI:
+ // Avoid inferring the type for other operands, as this may lead to infinite
+ // recursions for cycles.
+ return inferScalarType(R->getOperand(0));
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::Not:
case VPInstruction::ResumePhi:
@@ -236,14 +240,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
- VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe,
- VPScalarPHIRecipe>([this](const auto *R) {
- // Handle header phi recipes, except VPWidenIntOrFpInduction
- // which needs special handling due it being possibly truncated.
- // TODO: consider inferring/caching type of siblings, e.g.,
- // backedge value, here and in cases below.
- return inferScalarType(R->getStartValue());
- })
+ VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
+ [this](const auto *R) {
+ // Handle header phi recipes, except VPWidenIntOrFpInduction
+ // which needs special handling due it being possibly truncated.
+ // TODO: consider inferring/caching type of siblings, e.g.,
+ // backedge value, here and in cases below.
+ return inferScalarType(R->getStartValue());
+ })
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
[](const auto *R) { return R->getScalarType(); })
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d154d54c37862..62fc16fe9d6f1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -277,6 +277,11 @@ InstructionCost VPRecipeBase::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
llvm_unreachable("subclasses should implement computeCost");
}
+bool VPRecipeBase::isPhi() const {
+ return (getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC) ||
+ (isa<VPInstruction>(this) &&
+ cast<VPInstruction>(this)->getOpcode() == Instruction::PHI);
+}
InstructionCost
VPPartialReductionRecipe::computeCost(ElementCount VF,
@@ -418,6 +423,7 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
if (isSingleScalar() || isVectorToScalar())
return true;
switch (Opcode) {
+ case Instruction::PHI:
case Instruction::ICmp:
case Instruction::Select:
case VPInstruction::BranchOnCond:
@@ -458,6 +464,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
switch (getOpcode()) {
+ case Instruction::PHI: {
+ BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
+ Value *Start = State.get(getOperand(0), VPLane(0));
+ PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
+ Phi->addIncoming(Start, VectorPH);
+ return Phi;
+ }
case VPInstruction::Not: {
Value *A = State.get(getOperand(0));
return Builder.CreateNot(A, Name);
@@ -838,6 +851,8 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
switch (getOpcode()) {
default:
return false;
+ case Instruction::PHI:
+ return true;
case Instruction::ICmp:
case Instruction::Select:
case Instruction::Or:
@@ -3283,11 +3298,12 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
PHINode *NewPointerPhi = nullptr;
if (CurrentPart == 0) {
- auto *IVR = cast<VPHeaderPHIRecipe>(&getParent()
- ->getPlan()
- ->getVectorLoopRegion()
- ->getEntryBasicBlock()
- ->front());
+ auto *IVR = getParent()
+ ->getPlan()
+ ->getVectorLoopRegion()
+ ->getEntryBasicBlock()
+ ->front()
+ .getVPSingleValue();
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",
CanonicalIV->getIterator());
@@ -3666,22 +3682,3 @@ void VPEVLBasedIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
printOperands(O, SlotTracker);
}
#endif
-
-void VPScalarPHIRecipe::execute(VPTransformState &State) {
- BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
- Value *Start = State.get(getStartValue(), VPLane(0));
- PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
- Phi->addIncoming(Start, VectorPH);
- Phi->setDebugLoc(getDebugLoc());
- State.set(this, Phi, /*IsScalar=*/true);
-}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPScalarPHIRecipe::print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "SCALAR-PHI ";
- printAsOperand(O, SlotTracker);
- O << " = phi ";
- printOperands(O, SlotTracker);
-}
-#endif
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7646350ca0ed2..e1cb5fb48bf9c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1738,7 +1738,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
// Create a scalar phi to track the previous EVL if fixed-order recurrence is
// contained.
- VPScalarPHIRecipe *PrevEVL = nullptr;
+ VPInstruction *PrevEVL = nullptr;
bool ContainsFORs =
any_of(Header->phis(), IsaPred<VPFirstOrderRecurrencePHIRecipe>);
if (ContainsFORs) {
@@ -1753,7 +1753,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
VFSize > 32 ? Instruction::Trunc : Instruction::ZExt, MaxEVL,
Type::getInt32Ty(Ctx), DebugLoc());
}
- PrevEVL = new VPScalarPHIRecipe(MaxEVL, &EVL, DebugLoc(), "prev.evl");
+ PrevEVL = new VPInstruction(Instruction::PHI, {MaxEVL, &EVL}, DebugLoc(),
+ "prev.evl");
PrevEVL->insertBefore(*Header, Header->getFirstNonPhi());
}
@@ -2080,9 +2081,9 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
StringRef Name =
isa<VPCanonicalIVPHIRecipe>(PhiR) ? "index" : "evl.based.iv";
- auto *ScalarR =
- new VPScalarPHIRecipe(PhiR->getStartValue(), PhiR->getBackedgeValue(),
- PhiR->getDebugLoc(), Name);
+ auto *ScalarR = new VPInstruction(
+ Instruction::PHI, {PhiR->getStartValue(), PhiR->getBackedgeValue()},
+ PhiR->getDebugLoc(), Name);
ScalarR->insertBefore(PhiR);
PhiR->replaceAllUsesWith(ScalarR);
PhiR->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 1b3b69ea6a13d..f23b1429c1480 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -143,12 +143,13 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
})
.Case<VPWidenStoreEVLRecipe, VPReductionEVLRecipe>(
[&](const VPRecipeBase *S) { return VerifyEVLUse(*S, 2); })
- .Case<VPWidenLoadEVLRecipe, VPReverseVectorPointerRecipe,
- VPScalarPHIRecipe>(
+ .Case<VPWidenLoadEVLRecipe, VPReverseVectorPointerRecipe>(
[&](const VPRecipeBase *R) { return VerifyEVLUse(*R, 1); })
.Case<VPScalarCastRecipe>(
[&](const VPScalarCastRecipe *S) { return VerifyEVLUse(*S, 0); })
.Case<VPInstruction>([&](const VPInstruction *I) {
+ if (I->getOpcode() == Instruction::PHI)
+ return VerifyEVLUse(*I, I->getNumOperands() - 1);
if (I->getOpcode() != Instruction::Add) {
errs() << "EVL is used as an operand in non-VPInstruction::Add\n";
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
index a880bea2c52d1..e49192adb11c4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
@@ -86,7 +86,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: SCALAR-PHI vp<[[EP_IV:%.+]]> = phi ir<0>, vp<%index.next>
+; CHECK-NEXT: EMIT vp<[[EP_IV:%.+]]> = phi ir<0>, vp<%index.next>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi ir<0>, ir<%add> (VF scaled by 1/4)
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[EP_IV]]>, ir<1>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 4e862bf2f7480..ffc4cfa61f134 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -193,7 +193,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
+; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
@@ -442,7 +442,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
+; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
index 67be80393d829..dfc2fffdad2bb 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -32,8 +32,8 @@
; IF-EVL: <x1> vector loop: {
; IF-EVL-NEXT: vector.body:
- ; IF-EVL-NEXT: SCALAR-PHI vp<[[IV:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT_EXIT:%.+]]>
- ; IF-EVL-NEXT: SCALAR-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEX:%.+]]>
+ ; IF-EVL-NEXT: EMIT vp<[[IV:%.+]]> = phi ir<0>, vp<[[IV_NEXT_EXIT:%.+]]>
+ ; IF-EVL-NEXT: EMIT vp<[[EVL_PHI:%.+]]> = phi ir<0>, vp<[[IV_NEX:%.+]]>
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
index 2cc8aea82ca52..56ba4ccbe99e0 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
@@ -19,7 +19,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
+; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK-NEXT: EMIT vp<[[PTR:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]>
; CHECK-NEXT: vp<[[WIDE_PTR:%.+]]> = vector-pointer vp<[[PTR]]>
>From af491b481d5473f43325ec58cbe6bb4531bc9d71 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 11 Mar 2025 21:32:23 +0000
Subject: [PATCH 2/5] !fixup address latest comments, thanks!
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 14 +++++---------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | 4 ++--
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +++--
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 8 +++++---
.../vplan-vp-intrinsics-fixed-order-recurrence.ll | 2 +-
5 files changed, 16 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index e3886d90057de..0ffcf81f3032b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1004,19 +1004,15 @@ void VPlan::execute(VPTransformState *State) {
Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
continue;
}
- if (auto *VPI = dyn_cast<VPInstruction>(&R)) {
- Value *Phi = State->get(VPI, true);
- Value *Val = State->get(VPI->getOperand(1), true);
- cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
- continue;
- }
- auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
- bool NeedsScalar =
+ auto *PhiR = cast<VPSingleDefRecipe>(&R);
+ // VPInstructions currently model scalar Phis only.
+ bool NeedsScalar = isa<VPInstruction>(PhiR) ||
(isa<VPReductionPHIRecipe>(PhiR) &&
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
Value *Phi = State->get(PhiR, NeedsScalar);
- Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
+ // VPHeaderPHIRecipe supports getBackedgeValue() but VPInstruction does not.
+ Value *Val = State->get(PhiR->getOperand(1), NeedsScalar);
cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 6bdb284b0677e..9b0720760df40 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -72,8 +72,8 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case VPInstruction::ExplicitVectorLength:
return Type::getIntNTy(Ctx, 32);
case Instruction::PHI:
- // Avoid inferring the type for other operands, as this may lead to infinite
- // recursions for cycles.
+ // Infer the type of first operand only, as other operands of header phi's
+ // may lead to infinite recursion.
return inferScalarType(R->getOperand(0));
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::Not:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 55369a3a7cd77..a25bb818aa6f3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -422,8 +422,8 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
if (isSingleScalar() || isVectorToScalar())
return true;
switch (Opcode) {
- case Instruction::PHI:
case Instruction::ICmp:
+ case Instruction::PHI:
case Instruction::Select:
case VPInstruction::BranchOnCond:
case VPInstruction::BranchOnCount:
@@ -464,6 +464,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
switch (getOpcode()) {
case Instruction::PHI: {
+ // At the moment, VPInstructions with PHI opcodes are only user for scalar header phis.
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
Value *Start = State.get(getOperand(0), VPLane(0));
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
@@ -784,7 +785,7 @@ bool VPInstruction::isVectorToScalar() const {
}
bool VPInstruction::isSingleScalar() const {
- return getOpcode() == VPInstruction::ResumePhi;
+ return getOpcode() == VPInstruction::ResumePhi || getOpcode() == Instruction::PHI;
}
#if !defined(NDEBUG)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index f23b1429c1480..bc3364c186168 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -77,7 +77,7 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) {
if (isa<VPActiveLaneMaskPHIRecipe>(RecipeI))
NumActiveLaneMaskPhiRecipes++;
- if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe, VPWidenPHIRecipe>(*RecipeI)) {
+ if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe, VPWidenPHIRecipe>(*RecipeI) && !isa<VPInstruction>(*RecipeI) && cast<VPInstruction>(RecipeI)->getOpcode() == Instruction::PHI) {
errs() << "Found non-header PHI recipe in header VPBB";
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
errs() << ": ";
@@ -149,7 +149,7 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
[&](const VPScalarCastRecipe *S) { return VerifyEVLUse(*S, 0); })
.Case<VPInstruction>([&](const VPInstruction *I) {
if (I->getOpcode() == Instruction::PHI)
- return VerifyEVLUse(*I, I->getNumOperands() - 1);
+ return VerifyEVLUse(*I, 1);
if (I->getOpcode() != Instruction::Add) {
errs() << "EVL is used as an operand in non-VPInstruction::Add\n";
return false;
@@ -209,7 +209,9 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (!UI ||
isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI) ||
(isa<VPIRInstruction>(UI) &&
- isa<PHINode>(cast<VPIRInstruction>(UI)->getInstruction())))
+ isa<PHINode>(cast<VPIRInstruction>(UI)->getInstruction())) ||
+ (isa<VPInstruction>(UI) && cast<VPInstruction>(UI)->getOpcode() == Instruction::PHI)
+ )
continue;
// If the user is in the same block, check it comes after R in the
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
index 0bcfe13832ae7..3fed9e4956107 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
@@ -27,7 +27,7 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<[[FOR_PHI:%.+]]> = phi ir<33>, ir<[[LD:%.+]]>
-; IF-EVL-NEXT: SCALAR-PHI vp<[[PREV_EVL:%.+]]> = phi vp<[[VF32]]>, vp<[[EVL:%.+]]>
+; IF-EVL-NEXT: EMIT vp<[[PREV_EVL:%.+]]> = phi vp<[[VF32]]>, vp<[[EVL:%.+]]>
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%TC>, vp<[[EVL_PHI]]>
; IF-EVL-NEXT: EMIT vp<[[EVL]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>
>From 0b65489bd1dceec6241ad5385a09ebb4501b878f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 12 Mar 2025 10:53:07 +0000
Subject: [PATCH 3/5] !fixup remove unneeded code, add assert.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 7 -------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 19 +++++++++++--------
.../Transforms/LoopVectorize/discriminator.ll | 2 +-
3 files changed, 12 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 91f91f70a4b70..f78eb84b0c445 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2955,13 +2955,6 @@ class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
return 0;
}
- /// Returns true if the recipe only uses the first lane of operand \p Op.
- bool onlyFirstLaneUsed(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4129a9668438c..602a599d0e750 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -463,14 +463,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
switch (getOpcode()) {
- case Instruction::PHI: {
- // At the moment, VPInstructions with PHI opcodes are only user for scalar header phis.
- BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
- Value *Start = State.get(getOperand(0), VPLane(0));
- PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
- Phi->addIncoming(Start, VectorPH);
- return Phi;
- }
case VPInstruction::Not: {
Value *A = State.get(getOperand(0));
return Builder.CreateNot(A, Name);
@@ -481,6 +473,17 @@ Value *VPInstruction::generate(VPTransformState &State) {
Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
return Builder.CreateCmp(getPredicate(), A, B, Name);
}
+ case Instruction::PHI: {
+ assert(getParent() ==
+ getParent()->getPlan()->getVectorLoopRegion()->getEntry() &&
+ "VPInstructions with PHI opcodes must be used for header phis only "
+ "at the moment");
+ BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
+ Value *Start = State.get(getOperand(0), VPLane(0));
+ PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
+ Phi->addIncoming(Start, VectorPH);
+ return Phi;
+ }
case Instruction::Select: {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
Value *Cond = State.get(getOperand(0), OnlyFirstLaneUsed);
diff --git a/llvm/test/Transforms/LoopVectorize/discriminator.ll b/llvm/test/Transforms/LoopVectorize/discriminator.ll
index 5e56dcf0c434f..b66a70b9768c4 100644
--- a/llvm/test/Transforms/LoopVectorize/discriminator.ll
+++ b/llvm/test/Transforms/LoopVectorize/discriminator.ll
@@ -49,8 +49,8 @@ define void @_Z3foov() local_unnamed_addr #0 !dbg !6 {
;LOOPUNROLL_5: discriminator: 21
; When unrolling after loop vectorize, both vec_body and remainder loop
; are unrolled.
-;LOOPVEC_UNROLL: discriminator: 9
;LOOPVEC_UNROLL: discriminator: 385
+;LOOPVEC_UNROLL: discriminator: 9
;DBG_VALUE: ![[DBG]] = {{.*}}, scope: ![[TOP]]
; Pseudo probe should not have duplication factor assigned.
;PSEUDO_PROBE: ![[TOP:[0-9]*]] = distinct !DISubprogram(name: "foo"
>From 963bc78dcdbd06fd92211a3d07e85d6e7ae295e9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 12 Mar 2025 10:54:38 +0000
Subject: [PATCH 4/5] !fixup fix formatting
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 ++-
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 8 +++++---
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 602a599d0e750..ab053885ec85a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -788,7 +788,8 @@ bool VPInstruction::isVectorToScalar() const {
}
bool VPInstruction::isSingleScalar() const {
- return getOpcode() == VPInstruction::ResumePhi || getOpcode() == Instruction::PHI;
+ return getOpcode() == VPInstruction::ResumePhi ||
+ getOpcode() == Instruction::PHI;
}
#if !defined(NDEBUG)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index bc3364c186168..13fb8a61d9afa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -77,7 +77,9 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) {
if (isa<VPActiveLaneMaskPHIRecipe>(RecipeI))
NumActiveLaneMaskPhiRecipes++;
- if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe, VPWidenPHIRecipe>(*RecipeI) && !isa<VPInstruction>(*RecipeI) && cast<VPInstruction>(RecipeI)->getOpcode() == Instruction::PHI) {
+ if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe, VPWidenPHIRecipe>(*RecipeI) &&
+ !isa<VPInstruction>(*RecipeI) &&
+ cast<VPInstruction>(RecipeI)->getOpcode() == Instruction::PHI) {
errs() << "Found non-header PHI recipe in header VPBB";
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
errs() << ": ";
@@ -210,8 +212,8 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI) ||
(isa<VPIRInstruction>(UI) &&
isa<PHINode>(cast<VPIRInstruction>(UI)->getInstruction())) ||
- (isa<VPInstruction>(UI) && cast<VPInstruction>(UI)->getOpcode() == Instruction::PHI)
- )
+ (isa<VPInstruction>(UI) &&
+ cast<VPInstruction>(UI)->getOpcode() == Instruction::PHI))
continue;
// If the user is in the same block, check it comes after R in the
>From b279a740721752f19dcd8c535607d2c0b8c59d18 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 12 Mar 2025 10:56:34 +0000
Subject: [PATCH 5/5] !fixup add newline
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ab053885ec85a..09cc31b80f139 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -276,6 +276,7 @@ InstructionCost VPRecipeBase::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
llvm_unreachable("subclasses should implement computeCost");
}
+
bool VPRecipeBase::isPhi() const {
return (getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC) ||
(isa<VPInstruction>(this) &&
More information about the llvm-commits
mailing list