[llvm] 954097d - [VPlan] Use SCEV to check subtract in getOptimizableIVOf.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 23 14:01:34 PDT 2025
Author: Florian Hahn
Date: 2025-08-23T22:00:01+01:00
New Revision: 954097dd61d6f03bef50c4def946a25eab22e835
URL: https://github.com/llvm/llvm-project/commit/954097dd61d6f03bef50c4def946a25eab22e835
DIFF: https://github.com/llvm/llvm-project/commit/954097dd61d6f03bef50c4def946a25eab22e835.diff
LOG: [VPlan] Use SCEV to check subtract in getOptimizableIVOf.
Simplify checks for IV subtractions in getOptimizableIVOf by using SCEV.
This slightly generalizes the patterns we can handle.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 04d777f0d00f7..a669e5938a12d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8908,7 +8908,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
WithoutRuntimeCheck);
}
- VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues);
+ VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues, *PSE.getSE());
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
return Plan;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 0d2e8cf64ac83..e0d63214f135a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -716,7 +716,8 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
/// Check if \p VPV is an untruncated wide induction, either before or after the
/// increment. If so return the header IV (before the increment), otherwise
/// return null.
-static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV) {
+static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV,
+ ScalarEvolution &SE) {
auto *WideIV = dyn_cast<VPWidenInductionRecipe>(VPV);
if (WideIV) {
// VPV itself is a wide induction, separately compute the end value for exit
@@ -753,13 +754,13 @@ static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV) {
// IVStep will be the negated step of the subtraction. Check if Step == -1
// * IVStep.
VPValue *Step;
- if (!match(VPV, m_Sub(m_VPValue(), m_VPValue(Step))) ||
- !Step->isLiveIn() || !IVStep->isLiveIn())
+ if (!match(VPV, m_Sub(m_VPValue(), m_VPValue(Step))))
return false;
- auto *StepCI = dyn_cast<ConstantInt>(Step->getLiveInIRValue());
- auto *IVStepCI = dyn_cast<ConstantInt>(IVStep->getLiveInIRValue());
- return StepCI && IVStepCI &&
- StepCI->getValue() == (-1 * IVStepCI->getValue());
+ const SCEV *IVStepSCEV = vputils::getSCEVExprForVPValue(IVStep, SE);
+ const SCEV *StepSCEV = vputils::getSCEVExprForVPValue(Step, SE);
+ return !isa<SCEVCouldNotCompute>(IVStepSCEV) &&
+ !isa<SCEVCouldNotCompute>(StepSCEV) &&
+ IVStepSCEV == SE.getNegativeSCEV(StepSCEV);
}
default:
return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
@@ -776,7 +777,8 @@ static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV) {
static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
VPTypeAnalysis &TypeInfo,
VPBlockBase *PredVPBB,
- VPValue *Op) {
+ VPValue *Op,
+ ScalarEvolution &SE) {
VPValue *Incoming, *Mask;
if (!match(Op, m_VPInstruction<VPInstruction::ExtractLane>(
m_VPInstruction<VPInstruction::FirstActiveLane>(
@@ -784,7 +786,7 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
m_VPValue(Incoming))))
return nullptr;
- auto *WideIV = getOptimizableIVOf(Incoming);
+ auto *WideIV = getOptimizableIVOf(Incoming, SE);
if (!WideIV)
return nullptr;
@@ -827,15 +829,14 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
/// Attempts to optimize the induction variable exit values for users in the
/// exit block coming from the latch in the original scalar loop.
-static VPValue *
-optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
- VPBlockBase *PredVPBB, VPValue *Op,
- DenseMap<VPValue *, VPValue *> &EndValues) {
+static VPValue *optimizeLatchExitInductionUser(
+ VPlan &Plan, VPTypeAnalysis &TypeInfo, VPBlockBase *PredVPBB, VPValue *Op,
+ DenseMap<VPValue *, VPValue *> &EndValues, ScalarEvolution &SE) {
VPValue *Incoming;
if (!match(Op, m_ExtractLastElement(m_VPValue(Incoming))))
return nullptr;
- auto *WideIV = getOptimizableIVOf(Incoming);
+ auto *WideIV = getOptimizableIVOf(Incoming, SE);
if (!WideIV)
return nullptr;
@@ -874,7 +875,8 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
}
void VPlanTransforms::optimizeInductionExitUsers(
- VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
+ VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues,
+ ScalarEvolution &SE) {
VPBlockBase *MiddleVPBB = Plan.getMiddleBlock();
VPTypeAnalysis TypeInfo(Plan);
for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
@@ -884,11 +886,12 @@ void VPlanTransforms::optimizeInductionExitUsers(
for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
VPValue *Escape = nullptr;
if (PredVPBB == MiddleVPBB)
- Escape = optimizeLatchExitInductionUser(
- Plan, TypeInfo, PredVPBB, ExitIRI->getOperand(Idx), EndValues);
+ Escape = optimizeLatchExitInductionUser(Plan, TypeInfo, PredVPBB,
+ ExitIRI->getOperand(Idx),
+ EndValues, SE);
else
Escape = optimizeEarlyExitInductionUser(Plan, TypeInfo, PredVPBB,
- ExitIRI->getOperand(Idx));
+ ExitIRI->getOperand(Idx), SE);
if (Escape)
ExitIRI->setOperand(Idx, Escape);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index def46b5d6d75c..700b94621d5fb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -247,7 +247,8 @@ struct VPlanTransforms {
/// one step backwards.
static void
optimizeInductionExitUsers(VPlan &Plan,
- DenseMap<VPValue *, VPValue *> &EndValues);
+ DenseMap<VPValue *, VPValue *> &EndValues,
+ ScalarEvolution &SE);
/// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
static void materializeBroadcasts(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
index fea027d6803c6..4b4103e9806b9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
@@ -111,7 +111,6 @@ define i16 @wide_sub_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i16> [[TMP10]], i32 3
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -128,7 +127,7 @@ define i16 @wide_sub_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: exit:
-; CHECK-NEXT: [[SUB_LCSSA:%.*]] = phi i16 [ [[SUB]], [[LOOP]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[SUB_LCSSA:%.*]] = phi i16 [ [[SUB]], [[LOOP]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i16 [[SUB_LCSSA]]
;
entry:
More information about the llvm-commits
mailing list