[llvm] [LV] Check if the VF is scalar by VFRange in `handleUncountableEarlyExit`. (PR #135294)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 16 17:17:40 PDT 2025
https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/135294
>From 1b83ae508b9c9e73a0ff98b04be4af699217bac3 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 10 Apr 2025 17:32:46 -0700
Subject: [PATCH 1/8] [LV] Check if the VF is scalar by VFRange instead of
VPlan in `handleUncountableEarlyExit` NFC.
This patch check if the plan contains scalar VF by VFRange instead of
Plan. Then we can furthur clamp the range after this transformation.
Split by #113903.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 5 +++--
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 4 +++-
3 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a4d546f698d5f..0cec4c5c87f67 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9744,7 +9744,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Legal->getUncountableEarlyExitingBlock()) {
VPlanTransforms::runPass(VPlanTransforms::handleUncountableEarlyExit, *Plan,
*PSE.getSE(), OrigLoop, UncountableExitingBlock,
- RecipeBuilder);
+ RecipeBuilder, Range);
}
DenseMap<VPValue *, VPValue *> IVEndValues;
addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 67a55aa67c978..b49a0c452fa1e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -17,6 +17,7 @@
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
+#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "VPlanVerifier.h"
@@ -2380,7 +2381,7 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
void VPlanTransforms::handleUncountableEarlyExit(
VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop,
- BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder) {
+ BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, VFRange &Range) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
auto *LatchVPBB = cast<VPBasicBlock>(LoopRegion->getExiting());
VPBuilder Builder(LatchVPBB->getTerminator());
@@ -2436,7 +2437,7 @@ void VPlanTransforms::handleUncountableEarlyExit(
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
}
// Add the incoming value from the early exit.
- if (!IncomingFromEarlyExit->isLiveIn() && !Plan.hasScalarVFOnly()) {
+ if (!IncomingFromEarlyExit->isLiveIn() && !Range.Start.isScalar()) {
VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
"first.active.lane");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index c23ff38265670..85649367c3478 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -28,6 +28,7 @@ class PredicatedScalarEvolution;
class TargetLibraryInfo;
class VPBuilder;
class VPRecipeBuilder;
+class VFRange;
extern cl::opt<bool> VerifyEachVPlan;
@@ -174,7 +175,8 @@ struct VPlanTransforms {
static void handleUncountableEarlyExit(VPlan &Plan, ScalarEvolution &SE,
Loop *OrigLoop,
BasicBlock *UncountableExitingBlock,
- VPRecipeBuilder &RecipeBuilder);
+ VPRecipeBuilder &RecipeBuilder,
+ VFRange &Range);
/// Lower abstract recipes to concrete ones, that can be codegen'd.
static void convertToConcreteRecipes(VPlan &Plan);
>From b7eca73760804353e09ed9810c9cbb4e1e0fb1ec Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 10 Apr 2025 18:23:23 -0700
Subject: [PATCH 2/8] !fixup, formatting.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b49a0c452fa1e..c551cfa6ba513 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2381,7 +2381,8 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
void VPlanTransforms::handleUncountableEarlyExit(
VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop,
- BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, VFRange &Range) {
+ BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder,
+ VFRange &Range) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
auto *LatchVPBB = cast<VPBasicBlock>(LoopRegion->getExiting());
VPBuilder Builder(LatchVPBB->getTerminator());
>From 9e96e1d6f56797562fd7fa3152c7e281893097b9 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Fri, 11 Apr 2025 01:26:11 -0700
Subject: [PATCH 3/8] !fixup, add const.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c551cfa6ba513..1cd1f5acda305 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2382,7 +2382,7 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
void VPlanTransforms::handleUncountableEarlyExit(
VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop,
BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder,
- VFRange &Range) {
+ const VFRange &Range) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
auto *LatchVPBB = cast<VPBasicBlock>(LoopRegion->getExiting());
VPBuilder Builder(LatchVPBB->getTerminator());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 85649367c3478..401cd8b7bd4e7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -176,7 +176,7 @@ struct VPlanTransforms {
Loop *OrigLoop,
BasicBlock *UncountableExitingBlock,
VPRecipeBuilder &RecipeBuilder,
- VFRange &Range);
+ const VFRange &Range);
/// Lower abstract recipes to concrete ones, that can be codegen'd.
static void convertToConcreteRecipes(VPlan &Plan);
>From 01625c087a3fc0ed63b47fa8da853b260972293d Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Fri, 11 Apr 2025 02:04:02 -0700
Subject: [PATCH 4/8] !fixup, clamp the range to make sure the plan has
scalar/vector VF only.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 9 +++++++--
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 2 +-
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1cd1f5acda305..59c5dff35b37c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2382,13 +2382,18 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
void VPlanTransforms::handleUncountableEarlyExit(
VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop,
BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder,
- const VFRange &Range) {
+ VFRange &Range) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
auto *LatchVPBB = cast<VPBasicBlock>(LoopRegion->getExiting());
VPBuilder Builder(LatchVPBB->getTerminator());
auto *MiddleVPBB = Plan.getMiddleBlock();
VPValue *IsEarlyExitTaken = nullptr;
+ // Clamp the range that make sure we insert extractElement for incoming value
+ // correctly.
+ bool IsScalarVF = LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) { return VF.isScalar(); }, Range);
+
// Process the uncountable exiting block. Update IsEarlyExitTaken, which
// tracks if the uncountable early exit has been taken. Also split the middle
// block and have it conditionally branch to the early exit block if
@@ -2438,7 +2443,7 @@ void VPlanTransforms::handleUncountableEarlyExit(
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
}
// Add the incoming value from the early exit.
- if (!IncomingFromEarlyExit->isLiveIn() && !Range.Start.isScalar()) {
+ if (!IncomingFromEarlyExit->isLiveIn() && !IsScalarVF) {
VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
"first.active.lane");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 401cd8b7bd4e7..85649367c3478 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -176,7 +176,7 @@ struct VPlanTransforms {
Loop *OrigLoop,
BasicBlock *UncountableExitingBlock,
VPRecipeBuilder &RecipeBuilder,
- const VFRange &Range);
+ VFRange &Range);
/// Lower abstract recipes to concrete ones, that can be codegen'd.
static void convertToConcreteRecipes(VPlan &Plan);
>From 65bf3359885ff630fa5a17b94f4f3ad9df69e672 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Sun, 13 Apr 2025 20:43:16 -0700
Subject: [PATCH 5/8] !fixup, address comment. Only clamp the range when
extract is needed.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 26 ++++++++++---------
1 file changed, 14 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 59c5dff35b37c..e2fcc769566cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2389,11 +2389,6 @@ void VPlanTransforms::handleUncountableEarlyExit(
auto *MiddleVPBB = Plan.getMiddleBlock();
VPValue *IsEarlyExitTaken = nullptr;
- // Clamp the range that make sure we insert extractElement for incoming value
- // correctly.
- bool IsScalarVF = LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) { return VF.isScalar(); }, Range);
-
// Process the uncountable exiting block. Update IsEarlyExitTaken, which
// tracks if the uncountable early exit has been taken. Also split the middle
// block and have it conditionally branch to the early exit block if
@@ -2443,13 +2438,20 @@ void VPlanTransforms::handleUncountableEarlyExit(
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
}
// Add the incoming value from the early exit.
- if (!IncomingFromEarlyExit->isLiveIn() && !IsScalarVF) {
- VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
- VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
- "first.active.lane");
- IncomingFromEarlyExit = EarlyExitB.createNaryOp(
- Instruction::ExtractElement, {IncomingFromEarlyExit, FirstActiveLane},
- nullptr, "early.exit.value");
+ if (!IncomingFromEarlyExit->isLiveIn()) {
+ // Limit range to scalar VF only, if the range contains the scalar VF.
+ bool IsVectorVF = LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) { return VF.isVector(); }, Range);
+
+ if (IsVectorVF) {
+ VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
+ VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
+ "first.active.lane");
+ IncomingFromEarlyExit =
+ EarlyExitB.createNaryOp(Instruction::ExtractElement,
+ {IncomingFromEarlyExit, FirstActiveLane},
+ nullptr, "early.exit.value");
+ }
}
ExitIRI->addOperand(IncomingFromEarlyExit);
}
>From 53c168b9058a1644c2058035dedd7dc515b41c55 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 14 Apr 2025 16:31:20 -0700
Subject: [PATCH 6/8] !fixup, address comment.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e2fcc769566cd..4971bd9fe1eca 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2441,7 +2441,7 @@ void VPlanTransforms::handleUncountableEarlyExit(
if (!IncomingFromEarlyExit->isLiveIn()) {
// Limit range to scalar VF only, if the range contains the scalar VF.
bool IsVectorVF = LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) { return VF.isVector(); }, Range);
+ [](ElementCount VF) { return VF.isVector(); }, Range);
if (IsVectorVF) {
VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
>From ce30391a149cb704185de909010ecd03870ecb03 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 15 Apr 2025 01:06:14 -0700
Subject: [PATCH 7/8] !fixup, address comments.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 25 ++++++++-----------
1 file changed, 11 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4971bd9fe1eca..06d299a2d8f08 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2437,21 +2437,18 @@ void VPlanTransforms::handleUncountableEarlyExit(
ExitIRI->addOperand(IncomingFromLatch);
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
}
+
+ auto IsVector = [](ElementCount VF) { return VF.isVector(); };
// Add the incoming value from the early exit.
- if (!IncomingFromEarlyExit->isLiveIn()) {
- // Limit range to scalar VF only, if the range contains the scalar VF.
- bool IsVectorVF = LoopVectorizationPlanner::getDecisionAndClampRange(
- [](ElementCount VF) { return VF.isVector(); }, Range);
-
- if (IsVectorVF) {
- VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
- VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
- "first.active.lane");
- IncomingFromEarlyExit =
- EarlyExitB.createNaryOp(Instruction::ExtractElement,
- {IncomingFromEarlyExit, FirstActiveLane},
- nullptr, "early.exit.value");
- }
+ if (!IncomingFromEarlyExit->isLiveIn() &&
+ // Limit range to scalar VF only, if the range contains the scalar VF.
+ LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) {
+ VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
+ VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
+ "first.active.lane");
+ IncomingFromEarlyExit = EarlyExitB.createNaryOp(
+ Instruction::ExtractElement, {IncomingFromEarlyExit, FirstActiveLane},
+ nullptr, "early.exit.value");
}
ExitIRI->addOperand(IncomingFromEarlyExit);
}
>From dcaed17c60ad69ab05744499bbb848b59813904f Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Wed, 16 Apr 2025 17:08:44 -0700
Subject: [PATCH 8/8] !fixup, update comments.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 06d299a2d8f08..e6651a505d2a1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2439,9 +2439,11 @@ void VPlanTransforms::handleUncountableEarlyExit(
}
auto IsVector = [](ElementCount VF) { return VF.isVector(); };
- // Add the incoming value from the early exit.
+ // When the VFs are vectors, need to add `extract` to get the incoming value
+ // from early exit. When the range contains scalar VF, limit the range to
+ // scalar VF to prevent mis-compilation for the range containing both scalar
+ // and vector VFs.
if (!IncomingFromEarlyExit->isLiveIn() &&
- // Limit range to scalar VF only, if the range contains the scalar VF.
LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) {
VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
More information about the llvm-commits
mailing list