[llvm] [VPlan] Split simplifyRecipes into simplifyRecipes and combineRecipes (PR #221924)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 02:02:08 PDT 2026
https://github.com/lukel97 created https://github.com/llvm/llvm-project/pull/221924
Bringing it in line with InstSimplify and InstCombine, split up simplifyRecipe into a function that modifies and creates recipes (combineRecipe), and one that is analysis-only (simplifyRecipe).
This allows us to avoid adding simplficiation only folds to the worklist in #213899.
The funclet.ll test no longer erases the constant-folded intrinsic call because vputils::isDeadRecipe returns false.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From 7f291c23a33102c899b2f958e59bb225cfdcc144 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 8 Sep 2026 16:08:46 +0800
Subject: [PATCH] [VPlan] Split simplifyRecipes into simplifyRecipes and
combineRecipes
Bringing it in line with InstSimplify and InstCombine, split up simplifyRecipe into a function that modifies and creates recipes (combineRecipe), and one that is analysis-only (simplifyRecipe).
This allows us to avoid adding simplficiation only folds to the worklist in #213899.
The funclet.ll test no longer erases the constant-folded intrinsic call because vputils::isDeadRecipe returns false.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 429 +++++++++---------
.../Transforms/Vectorize/VPlanTransforms.h | 2 +-
.../LoopVectorize/VPlan/constant-fold.ll | 2 +-
.../VPlan/vplan-print-before-after-all.ll | 10 +-
.../Transforms/LoopVectorize/X86/funclet.ll | 15 +-
.../LoopVectorize/vplan-print-before-after.ll | 8 +-
7 files changed, 246 insertions(+), 228 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e027dfa834759..10a58f4d40da7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5719,7 +5719,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVF, BestUF, PSE);
RUN_VPLAN_PASS(VPlanTransforms::optimizeForVFAndUF, BestVPlan, BestVF, BestUF,
PSE);
- RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
+ RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
// Check if scalar epilogue is required, before simplifying constant branches.
const bool RequiresScalarEpilogue = BestVPlan.requiresScalarEpilogue();
if (EpilogueVecKind == EpilogueVectorizationKind::None)
@@ -5776,13 +5776,13 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::expandSCEVsToVPInstructions, BestVPlan,
*PSE.getSE());
RUN_VPLAN_PASS(VPlanTransforms::cse, BestVPlan);
- RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
+ RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
// Removing branches and incoming values may expose additional simplification
// opportunities.
if (RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
/*OnlyLatches=*/EpilogueVecKind !=
EpilogueVectorizationKind::None))
- RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
+ RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::simplifyKnownEVL, BestVPlan, BestVF, PSE);
// 0. Generate SCEV-dependent code in the entry, including TripCount, before
@@ -6387,7 +6387,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
if (const LoopAccessInfo *LAI = Legal->getLAI())
RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
LAI->getSymbolicStrides(), VPDT);
- RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
// Create recipes for header phis. For outer loops, reductions, recurrences
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 25a78bc13644f..2de2fc24567a6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1170,14 +1170,12 @@ static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
}
/// Try to simplify logical and bitwise recipes in \p Def.
-static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def,
- VPBuilder &Builder,
- bool CanCreateNewRecipe) {
+static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
// Simplify (X && Y) | (X && !Y) -> X.
// TODO: Split up into simpler, modular combines: (X && Y) | (X && Z) into X
// && (Y | Z) and (X | !X) into true. This requires queuing newly created
// recipes to be visited during simplification.
- VPValue *X, *Y, *Z;
+ VPValue *X, *Y;
if (match(Def,
m_c_BinaryOr(m_LogicalAnd(m_VPValue(X), m_VPValue(Y)),
m_LogicalAnd(m_Deferred(X), m_Not(m_Deferred(Y))))))
@@ -1211,26 +1209,11 @@ static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def,
if (match(Def, m_c_LogicalAnd(m_VPValue(X), m_True())))
return X;
- // (x && y) | (x && z) -> x && (y | z)
- if (CanCreateNewRecipe &&
- match(Def, m_c_BinaryOr(m_LogicalAnd(m_VPValue(X), m_VPValue(Y)),
- m_LogicalAnd(m_Deferred(X), m_VPValue(Z)))) &&
- // Simplify only if one of the operands has one use to avoid creating an
- // extra recipe.
- (!Def->getOperand(0)->hasMoreThanOneUniqueUser() ||
- !Def->getOperand(1)->hasMoreThanOneUniqueUser()))
- return Builder.createLogicalAnd(X, Builder.createOr(Y, Z));
-
// x && (x && y) -> x && y
if (match(Def, m_LogicalAnd(m_VPValue(X),
m_LogicalAnd(m_Deferred(X), m_VPValue()))))
return Def->getOperand(1);
- // x && (y && x) -> x && y
- if (match(Def, m_LogicalAnd(m_VPValue(X),
- m_LogicalAnd(m_VPValue(Y), m_Deferred(X)))))
- return Builder.createLogicalAnd(X, Y);
-
// x && !x -> 0
if (match(Def, m_LogicalAnd(m_VPValue(X), m_Not(m_Deferred(X)))))
return Plan.getFalse();
@@ -1238,52 +1221,11 @@ static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def,
if (match(Def, m_Select(m_VPValue(), m_VPValue(X), m_Deferred(X))))
return X;
- // (x && y) | !x -> !x || y
- if (CanCreateNewRecipe &&
- match(Def,
- m_c_BinaryOr(m_OneUse(m_LogicalAnd(m_VPValue(X), m_VPValue(Y))),
- m_VPValue(Z, m_Not(m_Deferred(X))))))
- return Builder.createLogicalOr(Z, Y);
-
- // select c, false, true -> not c
- VPValue *C;
- if (CanCreateNewRecipe &&
- match(Def, m_Select(m_VPValue(C), m_False(), m_True())))
- return Builder.createNot(C);
-
- // select !c, x, y -> select c, y, x
- if (match(Def, m_Select(m_Not(m_VPValue(C)), m_VPValue(X), m_VPValue(Y)))) {
- Def->setOperand(0, C);
- Def->setOperand(1, Y);
- Def->setOperand(2, X);
- return Def;
- }
-
- // select x, (i1 y | z), y -> y | (x && z)
- if (CanCreateNewRecipe &&
- match(Def, m_Select(m_VPValue(X),
- m_OneUse(m_c_BinaryOr(m_VPValue(Y), m_VPValue(Z))),
- m_Deferred(Y))) &&
- Y->getScalarType()->isIntegerTy(1))
- return Builder.createOr(Y, Builder.createLogicalAnd(X, Z));
-
- // select %M0, (select %M1, %X, %Y), %Y -> select (%M0 && %M1), %X, %Y
- VPValue *Mask0, *Mask1;
- if (CanCreateNewRecipe &&
- match(Def,
- m_SelectLike(m_VPValue(Mask0),
- m_OneUse(m_SelectLike(m_VPValue(Mask1), m_VPValue(X),
- m_VPValue(Y))),
- m_Deferred(Y))))
- return Builder.createSelect(Builder.createLogicalAnd(Mask0, Mask1), X, Y,
- Def->getDebugLoc());
-
return nullptr;
}
-/// Try to simplify VPSingleDefRecipe \p Def. Returns a new recipe if it should
-/// be replaced, or the existing recipe if it was modified. Returns nullptr if
-/// nothing was simplified.
+/// Return a simpler value for VPSingleDefRecipe \p Def if possible. This must
+/// not create or modify recipes.
static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
// Simplification of live-in IR values for SingleDef recipes using
// InstSimplifyFolder.
@@ -1298,6 +1240,140 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
return Op;
}
+ if (VPValue *V = simplifyLogicalRecipe(Plan, Def))
+ return V;
+
+ VPValue *A, *B;
+
+ if (match(Def, m_c_Add(m_VPValue(A), m_ZeroInt())))
+ return A;
+
+ if (match(Def, m_c_Mul(m_VPValue(A), m_One())))
+ return A;
+
+ if (match(Def, m_c_Mul(m_VPValue(), m_ZeroInt())))
+ return Plan.getZero(Def->getScalarType());
+
+ // A bitcast to the same type is a no-op.
+ if (match(Def, m_BitCast(m_VPValue(A))) &&
+ Def->getScalarType() == A->getScalarType())
+ return A;
+
+ if (match(Def, m_Trunc(m_ZExtOrSExt(m_VPValue(A)))))
+ if (Def->getScalarType() == A->getScalarType())
+ return A;
+
+ if (match(Def, m_Not(m_Not(m_VPValue(A)))))
+ return A;
+
+ // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0.
+ if ((match(Def, m_DerivedIV(m_ZeroInt(), m_VPValue(A), m_One())) ||
+ match(Def, m_DerivedIV(m_ZeroInt(), m_VPValue(A, m_ZeroInt()),
+ m_VPValue()))) &&
+ A->getScalarType() == Def->getScalarType())
+ return A;
+
+ // Simplify MaskedCond with no block mask to its single operand.
+ if (match(Def, m_VPInstruction<VPInstruction::MaskedCond>()) &&
+ !cast<VPInstruction>(Def)->isMasked())
+ return Def->getOperand(0);
+
+ // Look through ExtractLastLane.
+ if (match(Def, m_ExtractLastLane(m_VPValue(A)))) {
+ if (match(A, m_BuildVector())) {
+ auto *BuildVector = cast<VPInstruction>(A);
+ return BuildVector->getOperand(BuildVector->getNumOperands() - 1);
+ }
+
+ if (match(A, m_Broadcast(m_VPValue(B))))
+ return B;
+
+ if (isa<VPInstruction, VPReplicateRecipe>(A) && vputils::isSingleScalar(A))
+ return A;
+
+ if (Plan.hasScalarVFOnly())
+ return A;
+ }
+
+ // Look through ExtractPenultimateElement (BuildVector ....).
+ if (match(Def, m_ExtractPenultimateElement(m_BuildVector()))) {
+ auto *BuildVector = cast<VPInstruction>(Def->getOperand(0));
+ return BuildVector->getOperand(BuildVector->getNumOperands() - 2);
+ }
+
+ uint64_t Idx;
+ if (match(Def, m_ExtractElement(m_BuildVector(), m_ConstantInt(Idx)))) {
+ auto *BuildVector = cast<VPInstruction>(Def->getOperand(0));
+ return BuildVector->getOperand(Idx);
+ }
+
+ if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Def)) {
+ if (Def->getNumOperands() == 1) {
+ return Def->getOperand(0);
+ }
+ if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Def)) {
+ if (all_equal(Phi->incoming_values()))
+ return Phi->getOperand(0);
+ }
+ return nullptr;
+ }
+
+ VPIRValue *IRV;
+ if (Def->getNumOperands() == 1 &&
+ match(Def, m_ComputeReductionResult(m_VPIRValue(IRV))))
+ return IRV;
+
+ if (match(Def, m_VPInstruction<VPInstruction::WideIVStep>(m_VPValue(A),
+ m_One())) &&
+ A->getScalarType() == Def->getScalarType())
+ return A;
+
+ // Some simplifications can only be applied after unrolling. Perform them
+ // below.
+ if (!Plan.isUnrolled())
+ return nullptr;
+
+ // After unrolling, extract-lane may be used to extract values from multiple
+ // scalar sources. Only simplify when extracting from a single scalar source.
+ VPValue *LaneToExtract;
+ if (match(Def, m_ExtractLane(m_VPValue(LaneToExtract), m_VPValue(A)))) {
+ // Simplify extract-lane(%lane_num, %scalar_val) -> %scalar_val.
+ if (vputils::isSingleScalar(A))
+ return A;
+
+ // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
+ // scalar canonical IV.
+ VPWidenIntOrFpInductionRecipe *WidenIV;
+ if (match(LaneToExtract, m_ZeroInt()) &&
+ match(A, m_CanonicalWidenIV(WidenIV)))
+ return WidenIV->getRegion()->getCanonicalIV();
+ }
+
+ // Simplify unrolled VectorPointer without offset, or with zero offset, to
+ // just the pointer operand.
+ if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Def))
+ if (!VPR->getVFxPart() || match(VPR->getVFxPart(), m_ZeroInt()))
+ return VPR->getOperand(0);
+
+ // VPScalarIVSteps after unrolling can be replaced by their start value, if
+ // the start index is zero and only the first lane 0 is demanded.
+ if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Def))
+ if (!Steps->getStartIndex() && vputils::onlyFirstLaneUsed(Steps))
+ return Steps->getOperand(0);
+
+ if (Plan.getConcreteUF() == 1 && match(Def, m_ExtractLastPart(m_VPValue(A))))
+ return A;
+
+ return nullptr;
+}
+
+/// Combine \p Def into a simpler recipe. May modify or create new recipes.
+static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
+ if (auto *V = simplifyRecipe(Plan, Def)) {
+ Def->replaceAllUsesWith(V);
+ return Def;
+ }
+
// Drop the mask of a predicated store masked by the header mask (which is
// guaranteed to be true at least for the first lane) and both the stored
// value and the address are uniform across VF and UF. The header mask is
@@ -1327,54 +1403,87 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
bool CanCreateNewRecipe =
!isa<VPInstruction>(Def) || !Def->getUnderlyingValue();
- VPValue *A, *Z;
+ VPValue *A, *X, *Y, *Z;
- // A bitcast to the same type is a no-op.
- if (match(Def, m_BitCast(m_VPValue(A))) &&
- Def->getScalarType() == A->getScalarType())
- return A;
+ // x && (y && x) -> x && y
+ if (CanCreateNewRecipe &&
+ match(Def, m_LogicalAnd(m_VPValue(X),
+ m_LogicalAnd(m_VPValue(Y), m_Deferred(X)))))
+ return Builder.createLogicalAnd(X, Y);
+
+ // (x && y) | (x && z) -> x && (y | z)
+ if (CanCreateNewRecipe &&
+ match(Def, m_c_BinaryOr(m_LogicalAnd(m_VPValue(X), m_VPValue(Y)),
+ m_LogicalAnd(m_Deferred(X), m_VPValue(Z)))) &&
+ // Simplify only if one of the operands has one use to avoid creating an
+ // extra recipe.
+ (!Def->getOperand(0)->hasMoreThanOneUniqueUser() ||
+ !Def->getOperand(1)->hasMoreThanOneUniqueUser()))
+ return Builder.createLogicalAnd(X, Builder.createOr(Y, Z));
+
+ // (x && y) | !x -> !x || y
+ if (CanCreateNewRecipe &&
+ match(Def,
+ m_c_BinaryOr(m_OneUse(m_LogicalAnd(m_VPValue(X), m_VPValue(Y))),
+ m_VPValue(Z, m_Not(m_Deferred(X))))))
+ return Builder.createLogicalOr(Z, Y);
+
+ // select c, false, true -> not c
+ VPValue *C;
+ if (CanCreateNewRecipe &&
+ match(Def, m_Select(m_VPValue(C), m_False(), m_True())))
+ return Builder.createNot(C);
+
+ // select !c, x, y -> select c, y, x
+ if (match(Def, m_Select(m_Not(m_VPValue(C)), m_VPValue(X), m_VPValue(Y)))) {
+ Def->setOperand(0, C);
+ Def->setOperand(1, Y);
+ Def->setOperand(2, X);
+ return Def;
+ }
+
+ // select x, (i1 y | z), y -> y | (x && z)
+ if (CanCreateNewRecipe &&
+ match(Def, m_Select(m_VPValue(X),
+ m_OneUse(m_c_BinaryOr(m_VPValue(Y), m_VPValue(Z))),
+ m_Deferred(Y))) &&
+ Y->getScalarType()->isIntegerTy(1))
+ return Builder.createOr(Y, Builder.createLogicalAnd(X, Z));
+
+ // select %M0, (select %M1, %X, %Y), %Y -> select (%M0 && %M1), %X, %Y
+ VPValue *Mask0, *Mask1;
+ if (CanCreateNewRecipe &&
+ match(Def,
+ m_SelectLike(m_VPValue(Mask0),
+ m_OneUse(m_SelectLike(m_VPValue(Mask1), m_VPValue(X),
+ m_VPValue(Y))),
+ m_Deferred(Y))))
+ return Builder.createSelect(Builder.createLogicalAnd(Mask0, Mask1), X, Y,
+ Def->getDebugLoc());
if (match(Def, m_Trunc(m_VPValue(Z, m_ZExtOrSExt(m_VPValue(A)))))) {
+ // Don't replace a non-widened cast recipe with a widened cast.
+ if (!isa<VPWidenCastRecipe>(Def))
+ return nullptr;
Type *TruncTy = Def->getScalarType();
Type *ATy = A->getScalarType();
- if (TruncTy == ATy) {
- return A;
- } else {
- // Don't replace a non-widened cast recipe with a widened cast.
- if (!isa<VPWidenCastRecipe>(Def))
- return nullptr;
- if (ATy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) {
-
- unsigned ExtOpcode = match(Z, m_SExt(m_VPValue())) ? Instruction::SExt
- : Instruction::ZExt;
- auto *Ext = Builder.createWidenCast(Instruction::CastOps(ExtOpcode), A,
- TruncTy);
- if (auto *UnderlyingExt = Z->getUnderlyingValue()) {
- // UnderlyingExt has distinct return type, used to retain legacy cost.
- Ext->setUnderlyingValue(UnderlyingExt);
- }
- return Ext;
- } else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
- auto *Trunc = Builder.createWidenCast(Instruction::Trunc, A, TruncTy);
- return Trunc;
+ if (ATy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) {
+
+ unsigned ExtOpcode =
+ match(Z, m_SExt(m_VPValue())) ? Instruction::SExt : Instruction::ZExt;
+ auto *Ext =
+ Builder.createWidenCast(Instruction::CastOps(ExtOpcode), A, TruncTy);
+ if (auto *UnderlyingExt = Z->getUnderlyingValue()) {
+ // UnderlyingExt has distinct return type, used to retain legacy cost.
+ Ext->setUnderlyingValue(UnderlyingExt);
}
+ return Ext;
+ } else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
+ auto *Trunc = Builder.createWidenCast(Instruction::Trunc, A, TruncTy);
+ return Trunc;
}
}
- if (VPValue *V =
- simplifyLogicalRecipe(Plan, Def, Builder, CanCreateNewRecipe))
- return V;
-
- VPValue *X, *Y;
- if (match(Def, m_c_Add(m_VPValue(A), m_ZeroInt())))
- return A;
-
- if (match(Def, m_c_Mul(m_VPValue(A), m_One())))
- return A;
-
- if (match(Def, m_c_Mul(m_VPValue(A), m_ZeroInt())))
- return Plan.getZero(Def->getScalarType());
-
if (CanCreateNewRecipe && match(Def, m_c_Mul(m_VPValue(A), m_AllOnes()))) {
// Preserve nsw from the Mul on the new Sub.
VPIRFlags::WrapFlagsTy NW = {
@@ -1421,9 +1530,6 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
*cast<VPRecipeWithIRFlags>(Def), Def->getDebugLoc());
if (match(Def, m_Not(m_VPValue(A)))) {
- if (match(A, m_Not(m_VPValue(A))))
- return A;
-
// Try to fold Not into compares by adjusting the predicate in-place.
CmpPredicate Pred;
if (match(A, m_Cmp(Pred, m_VPValue(), m_VPValue()))) {
@@ -1481,10 +1587,8 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
if (UnpairedCmp)
NewOps.push_back(UnpairedCmp->getVPSingleValue());
- if (NewOps.size() < Def->getNumOperands()) {
- VPValue *NewAnyOf = Builder.createNaryOp(VPInstruction::AnyOf, NewOps);
- return NewAnyOf;
- }
+ if (NewOps.size() < Def->getNumOperands())
+ return Builder.createNaryOp(VPInstruction::AnyOf, NewOps);
}
// Fold (fcmp uno %X, %X) or (fcmp uno %Y, %Y) -> fcmp uno %X, %Y
@@ -1497,20 +1601,10 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
m_SpecificCmp(CmpInst::FCMP_UNO, m_VPValue(Y), m_Deferred(Y)))))
return Builder.createFCmp(CmpInst::FCMP_UNO, X, Y);
- // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0.
- if ((match(Def, m_DerivedIV(m_ZeroInt(), m_VPValue(A), m_One())) ||
- match(Def, m_DerivedIV(m_ZeroInt(), m_VPValue(A, m_ZeroInt()),
- m_VPValue()))) &&
- A->getScalarType() == Def->getScalarType())
- return A;
-
if (match(Def, m_VPInstruction<VPInstruction::WideIVStep>(m_VPValue(X),
- m_One()))) {
- Type *WideStepTy = Def->getScalarType();
- if (X->getScalarType() != WideStepTy)
- X = Builder.createWidenCast(Instruction::Trunc, X, WideStepTy);
- return X;
- }
+ m_One())) &&
+ X->getScalarType() != Def->getScalarType())
+ return Builder.createWidenCast(Instruction::Trunc, X, Def->getScalarType());
// For i1 vp.merges produced by AnyOf reductions:
// vp.merge true, (or x, y), x, evl -> vp.merge y, true, x, evl
@@ -1523,40 +1617,6 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
return Def;
}
- // Simplify MaskedCond with no block mask to its single operand.
- if (match(Def, m_VPInstruction<VPInstruction::MaskedCond>()) &&
- !cast<VPInstruction>(Def)->isMasked())
- return Def->getOperand(0);
-
- // Look through ExtractLastLane.
- if (match(Def, m_ExtractLastLane(m_VPValue(A)))) {
- if (match(A, m_BuildVector())) {
- auto *BuildVector = cast<VPInstruction>(A);
- return BuildVector->getOperand(BuildVector->getNumOperands() - 1);
- }
-
- if (match(A, m_Broadcast(m_VPValue(X))))
- return X;
-
- if (isa<VPInstruction, VPReplicateRecipe>(A) && vputils::isSingleScalar(A))
- return A;
-
- if (Plan.hasScalarVFOnly())
- return A;
- }
-
- // Look through ExtractPenultimateElement (BuildVector ....).
- if (match(Def, m_ExtractPenultimateElement(m_BuildVector()))) {
- auto *BuildVector = cast<VPInstruction>(Def->getOperand(0));
- return BuildVector->getOperand(BuildVector->getNumOperands() - 2);
- }
-
- uint64_t Idx;
- if (match(Def, m_ExtractElement(m_BuildVector(), m_ConstantInt(Idx)))) {
- auto *BuildVector = cast<VPInstruction>(Def->getOperand(0));
- return BuildVector->getOperand(Idx);
- }
-
if (match(Def, m_BuildVector()) && all_equal(Def->operands()))
return Builder.createNaryOp(VPInstruction::Broadcast, Def->getOperand(0));
@@ -1585,46 +1645,16 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
return Def;
}
- if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Def)) {
- if (Def->getNumOperands() == 1) {
- return Def->getOperand(0);
- }
- if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Def)) {
- if (all_equal(Phi->incoming_values()))
- return Phi->getOperand(0);
- }
- return nullptr;
- }
-
- VPIRValue *IRV;
- if (Def->getNumOperands() == 1 &&
- match(Def, m_ComputeReductionResult(m_VPIRValue(IRV))))
- return IRV;
-
// Some simplifications can only be applied after unrolling. Perform them
// below.
if (!Plan.isUnrolled())
return nullptr;
- // After unrolling, extract-lane may be used to extract values from multiple
- // scalar sources. Only simplify when extracting from a single scalar source.
+ // Simplify extract-lane with single source to extract-element.
VPValue *LaneToExtract;
- if (match(Def, m_ExtractLane(m_VPValue(LaneToExtract), m_VPValue(A)))) {
- // Simplify extract-lane(%lane_num, %scalar_val) -> %scalar_val.
- if (vputils::isSingleScalar(A))
- return A;
-
- // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
- // scalar canonical IV.
- VPWidenIntOrFpInductionRecipe *WidenIV;
- if (match(LaneToExtract, m_ZeroInt()) &&
- match(A, m_CanonicalWidenIV(WidenIV)))
- return WidenIV->getRegion()->getCanonicalIV();
-
- // Simplify extract-lane with single source to extract-element.
+ if (match(Def, m_ExtractLane(m_VPValue(LaneToExtract), m_VPValue(A))))
return Builder.createNaryOp(Instruction::ExtractElement, {A, LaneToExtract},
Def->getDebugLoc());
- }
// Look for cycles where Def is of the form:
// X = phi(0, IVInc) ; used only by IVInc, or by IVInc and Inc = X + Y
@@ -1652,18 +1682,6 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
}
}
- // Simplify unrolled VectorPointer without offset, or with zero offset, to
- // just the pointer operand.
- if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Def))
- if (!VPR->getVFxPart() || match(VPR->getVFxPart(), m_ZeroInt()))
- return VPR->getOperand(0);
-
- // VPScalarIVSteps after unrolling can be replaced by their start value, if
- // the start index is zero and only the first lane 0 is demanded.
- if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Def))
- if (!Steps->getStartIndex() && vputils::onlyFirstLaneUsed(Steps))
- return Steps->getOperand(0);
-
// Simplify redundant ReductionStartVector recipes after unrolling.
VPValue *StartV;
if (match(Def, m_VPInstruction<VPInstruction::ReductionStartVector>(
@@ -1675,19 +1693,16 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
return Def;
}
- if (Plan.getConcreteUF() == 1 && match(Def, m_ExtractLastPart(m_VPValue(A))))
- return A;
-
return nullptr;
}
-void VPlanTransforms::simplifyRecipes(VPlan &Plan) {
+void VPlanTransforms::combineRecipes(VPlan &Plan) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB))
if (auto *Def = dyn_cast<VPSingleDefRecipe>(&R))
- if (VPValue *New = simplifyRecipe(Plan, Def)) {
+ if (VPSingleDefRecipe *New = combineRecipe(Plan, Def)) {
if (New != Def) {
// Replace the recipe with a new one.
Def->replaceAllUsesWith(New);
@@ -1874,7 +1889,7 @@ static void removeCommonBlendMask(VPBlendRecipe *Blend) {
Blend->setMask(I, Blend->getMask(I)->getDefiningRecipe()->getOperand(1));
}
-/// Normalize and simplify VPBlendRecipes. Should be run after simplifyRecipes
+/// Normalize and simplify VPBlendRecipes. Should be run after combineRecipes
/// to make sure the masks are simplified.
static void simplifyBlends(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
@@ -2663,14 +2678,14 @@ void VPlanTransforms::optimize(VPlan &Plan) {
RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan);
RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
- RUN_VPLAN_PASS(simplifyRecipes, Plan);
+ RUN_VPLAN_PASS(combineRecipes, Plan);
RUN_VPLAN_PASS(removeDeadRecipes, Plan);
RUN_VPLAN_PASS(simplifyBlends, Plan);
RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan);
RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan);
RUN_VPLAN_PASS(removeRedundantExpandSCEVRecipes, Plan);
RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
- RUN_VPLAN_PASS(simplifyRecipes, Plan);
+ RUN_VPLAN_PASS(combineRecipes, Plan);
RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
RUN_VPLAN_PASS(simplifyReverses, Plan);
RUN_VPLAN_PASS(removeDeadRecipes, Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index fef0c14415091..6686d887bcf83 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -421,7 +421,7 @@ struct VPlanTransforms {
VFRange &Range);
/// Perform instcombine-like simplifications on recipes in \p Plan.
- static void simplifyRecipes(VPlan &Plan);
+ static void combineRecipes(VPlan &Plan);
/// Cancel out redundant reverses in \p Plan, e.g. reverse(reverse(x)) -> x.
static void simplifyReverses(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll b/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
index 8c905cf63dce8..057e2c71eacd5 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
-; RUN: -vplan-print-after=VPlanTransforms::simplifyRecipes$ -disable-output %s 2>&1 \
+; RUN: -vplan-print-after=VPlanTransforms::combineRecipes$ -disable-output %s 2>&1 \
; RUN: | FileCheck --strict-whitespace %s
; REQUIRES: asserts
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index c2bd046efae87..b5ee488ae5bd5 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -12,7 +12,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printAfterInitialConstruction
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::simplifyRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::combineRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeDeadRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createHeaderPhiRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides at 2
@@ -49,14 +49,14 @@
; CHECK-BEFORE: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeRedundantInductionCasts
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] reassociateHeaderMask
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] combineRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeDeadRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyBlends
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] legalizeAndOptimizeInductions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] narrowToSingleScalarRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeRedundantExpandSCEVRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] reassociateHeaderMask at 2
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes at 2
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] combineRecipes at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeBranchOnConst
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyReverses
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeDeadRecipes at 2
@@ -75,7 +75,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replicateByVF
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeConstantVectorTripCount
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeForVFAndUF
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::simplifyRecipes at 2
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::combineRecipes at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeBranchOnConst at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeDeadRecipes at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::convertToConcreteRecipes
@@ -89,7 +89,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeFactors
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::expandSCEVsToVPInstructions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::cse
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::simplifyRecipes at 3
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::combineRecipes at 3
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeBranchOnConst at 4
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::simplifyKnownEVL
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::expandSCEVs
diff --git a/llvm/test/Transforms/LoopVectorize/X86/funclet.ll b/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
index 615748ebe1e45..7ae6d531484e4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
@@ -82,14 +82,17 @@ define void @constant_foldable() #0 personality ptr @__CxxFrameHandler3 {
; CHECK-NEXT: [[TMP1:%.*]] = catchpad within [[TMP0]] [ptr null, i32 64, ptr null]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[CATCH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[CALL:%.*]] = call double @floor(double 1.000000e+00) #[[ATTR1:[0-9]+]] [ "funclet"(token [[TMP1]]) ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <16 x double> @llvm.floor.v16f64(<16 x double> splat (double 1.000000e+00)) [ "funclet"(token [[TMP1]]) ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[EXIT:.*]], label %[[VECTOR_BODY]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[EXIT:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[CALL_LCSSA:%.*]] = phi double [ [[CALL]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: store double [[CALL_LCSSA]], ptr @sink, align 8
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: store double 1.000000e+00, ptr @sink, align 8
; CHECK-NEXT: catchret from [[TMP1]] to label %[[TRY_CONT:.*]]
; CHECK: [[TRY_CONT]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-print-before-after.ll b/llvm/test/Transforms/LoopVectorize/vplan-print-before-after.ll
index fce501ea014d8..b744fa02b8529 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-print-before-after.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-print-before-after.ll
@@ -1,9 +1,9 @@
; RUN: opt -passes=loop-vectorize -disable-output -force-vector-width=4 < %s \
-; RUN: -vplan-print-before=simplify -vplan-print-before=printFinalVPlan \
+; RUN: -vplan-print-before='combine|simplify' -vplan-print-before=printFinalVPlan \
; RUN: 2>&1 | FileCheck %s -DBEFORE_OR_AFTER=before --check-prefix=CHECK \
; RUN: --implicit-check-not "VPlan before"
; RUN: opt -passes=loop-vectorize -disable-output -force-vector-width=4 < %s \
-; RUN: -vplan-print-after=simplify -vplan-print-after=printFinalVPlan \
+; RUN: -vplan-print-after='combine|simplify' -vplan-print-after=printFinalVPlan \
; RUN: 2>&1 | FileCheck %s -DBEFORE_OR_AFTER=after --check-prefix=CHECK \
; RUN: --implicit-check-not "VPlan after"
@@ -14,11 +14,11 @@
; REQUIRES: asserts
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] combineRecipes
; CHECK-NEXT: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyBlends
; CHECK-NEXT: VPlan 'Initial VPlan for VF={4},UF>=1' {
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] combineRecipes
; CHECK-NEXT: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printFinalVPlan
; CHECK-NEXT: VPlan 'Final VPlan for VF={4},UF={1}' {
More information about the llvm-commits
mailing list