[llvm] [VPlan] Connect (MemRuntime|SCEV)Check blocks as VPlan transform (NFC). (PR #143879)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 27 02:51:26 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/143879
>From 556dac991b05e560e18a00494b5681e01a34ebc4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 17 Jun 2025 22:51:50 +0100
Subject: [PATCH 1/2] [VPlan] Connect (MemRuntime|SCEV)Check blocks as VPlan
transform (NFC).
Connect SCEV and memory runtime check block directly in VPlan as
VPIRBasicBlocks, removing ILV::emitSCEVChecks and ILV::emitMemRuntimeChecks.
The new logic is currently split across LoopVectorizationPlanner::addRuntimeChecks
which collects a list of {Condition, CheckBlock} pairs and performs some
checks and emits remarks if needed. The list of checks is then added to
VPlan in VPlanTransforms;:connectCheckBlocks.
---
.../Vectorize/LoopVectorizationPlanner.h | 7 +
.../Transforms/Vectorize/LoopVectorize.cpp | 209 +++++++-----------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 3 +-
.../Vectorize/VPlanConstruction.cpp | 39 ++++
.../Transforms/Vectorize/VPlanTransforms.h | 7 +
.../RISCV/riscv-vector-reverse.ll | 82 +++----
6 files changed, 160 insertions(+), 187 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 144f35e10132f..caad61543262b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -28,6 +28,10 @@
#include "llvm/ADT/SmallSet.h"
#include "llvm/Support/InstructionCost.h"
+namespace {
+class GeneratedRTChecks;
+}
+
namespace llvm {
class LoopInfo;
@@ -560,6 +564,9 @@ class LoopVectorizationPlanner {
VPRecipeBuilder &RecipeBuilder,
ElementCount MinVF);
+ /// Add the runtime checks from \p RTChecks to \p VPlan.
+ void addRuntimeChecks(VPlan &Plan, GeneratedRTChecks &RTChecks) const;
+
#ifndef NDEBUG
/// \return The most profitable vectorization factor for the available VPlans
/// and the cost of that VF.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb29e4fc6d232..a4848e9185623 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -399,12 +399,6 @@ static cl::opt<bool> EnableEarlyExitVectorization(
cl::desc(
"Enable vectorization of early exit loops with uncountable exits."));
-// Likelyhood of bypassing the vectorized loop because assumptions about SCEV
-// variables not overflowing do not hold. See `emitSCEVChecks`.
-static constexpr uint32_t SCEVCheckBypassWeights[] = {1, 127};
-// Likelyhood of bypassing the vectorized loop because pointers overlap. See
-// `emitMemRuntimeChecks`.
-static constexpr uint32_t MemCheckBypassWeights[] = {1, 127};
// Likelyhood of bypassing the vectorized loop because there are zero trips left
// after prolog. See `emitIterationCountCheck`.
static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
@@ -544,16 +538,6 @@ class InnerLoopVectorizer {
/// it overflows.
void emitIterationCountCheck(BasicBlock *Bypass);
- /// Emit a bypass check to see if all of the SCEV assumptions we've
- /// had to make are correct. Returns the block containing the checks or
- /// nullptr if no checks have been added.
- BasicBlock *emitSCEVChecks(BasicBlock *Bypass);
-
- /// Emit bypass checks to check any memory assumptions we may have made.
- /// Returns the block containing the checks or nullptr if no checks have been
- /// added.
- BasicBlock *emitMemRuntimeChecks(BasicBlock *Bypass);
-
/// Emit basic blocks (prefixed with \p Prefix) for the iteration check,
/// vector loop preheader, middle block and scalar preheader.
void createVectorLoopSkeleton(StringRef Prefix);
@@ -1790,7 +1774,6 @@ class GeneratedRTChecks {
SCEVExpander MemCheckExp;
bool CostTooHigh = false;
- const bool AddBranchWeights;
Loop *OuterLoop = nullptr;
@@ -1802,11 +1785,10 @@ class GeneratedRTChecks {
public:
GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
LoopInfo *LI, TargetTransformInfo *TTI,
- const DataLayout &DL, bool AddBranchWeights,
- TTI::TargetCostKind CostKind)
+ const DataLayout &DL, TTI::TargetCostKind CostKind)
: DT(DT), LI(LI), TTI(TTI), SCEVExp(*PSE.getSE(), DL, "scev.check"),
- MemCheckExp(*PSE.getSE(), DL, "scev.check"),
- AddBranchWeights(AddBranchWeights), PSE(PSE), CostKind(CostKind) {}
+ MemCheckExp(*PSE.getSE(), DL, "scev.check"), PSE(PSE),
+ CostKind(CostKind) {}
/// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
/// accurately estimate the cost of the runtime checks. The blocks are
@@ -2025,61 +2007,35 @@ class GeneratedRTChecks {
/// Adds the generated SCEVCheckBlock before \p LoopVectorPreHeader and
/// adjusts the branches to branch to the vector preheader or \p Bypass,
/// depending on the generated condition.
- BasicBlock *emitSCEVChecks(BasicBlock *Bypass,
- BasicBlock *LoopVectorPreHeader) {
+ std::pair<Value *, BasicBlock *> emitSCEVChecks() {
using namespace llvm::PatternMatch;
if (!SCEVCheckCond || match(SCEVCheckCond, m_ZeroInt()))
- return nullptr;
+ return {nullptr, nullptr};
- auto *Pred = LoopVectorPreHeader->getSinglePredecessor();
- BranchInst::Create(LoopVectorPreHeader, SCEVCheckBlock);
-
- SCEVCheckBlock->getTerminator()->eraseFromParent();
- SCEVCheckBlock->moveBefore(LoopVectorPreHeader);
- Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
- SCEVCheckBlock);
-
- BranchInst &BI =
- *BranchInst::Create(Bypass, LoopVectorPreHeader, SCEVCheckCond);
- if (AddBranchWeights)
- setBranchWeights(BI, SCEVCheckBypassWeights, /*IsExpected=*/false);
- ReplaceInstWithInst(SCEVCheckBlock->getTerminator(), &BI);
- // Mark the check as used, to prevent it from being removed during cleanup.
+ Value *Cond = SCEVCheckCond;
SCEVCheckCond = nullptr;
AddedAnyChecks = true;
- return SCEVCheckBlock;
+ return {Cond, SCEVCheckBlock};
}
/// Adds the generated MemCheckBlock before \p LoopVectorPreHeader and adjusts
/// the branches to branch to the vector preheader or \p Bypass, depending on
/// the generated condition.
- BasicBlock *emitMemRuntimeChecks(BasicBlock *Bypass,
- BasicBlock *LoopVectorPreHeader) {
+ std::pair<Value *, BasicBlock *> emitMemRuntimeChecks() {
// Check if we generated code that checks in runtime if arrays overlap.
if (!MemRuntimeCheckCond)
- return nullptr;
-
- auto *Pred = LoopVectorPreHeader->getSinglePredecessor();
- Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
- MemCheckBlock);
-
- MemCheckBlock->moveBefore(LoopVectorPreHeader);
-
- BranchInst &BI =
- *BranchInst::Create(Bypass, LoopVectorPreHeader, MemRuntimeCheckCond);
- if (AddBranchWeights) {
- setBranchWeights(BI, MemCheckBypassWeights, /*IsExpected=*/false);
- }
- ReplaceInstWithInst(MemCheckBlock->getTerminator(), &BI);
- MemCheckBlock->getTerminator()->setDebugLoc(
- Pred->getTerminator()->getDebugLoc());
+ return {nullptr, nullptr};
// Mark the check as used, to prevent it from being removed during cleanup.
+ Value *Cond = MemRuntimeCheckCond;
MemRuntimeCheckCond = nullptr;
AddedAnyChecks = true;
- return MemCheckBlock;
+ return {Cond, MemCheckBlock};
}
+ BasicBlock *getSCEVCheckBlock() const { return SCEVCheckBlock; }
+ BasicBlock *getMemCheckBlock() const { return MemCheckBlock; }
+
/// Return true if any runtime checks have been added
bool hasChecks() const { return AddedAnyChecks; }
};
@@ -2466,53 +2422,6 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
"Plan's entry must be TCCCheckBlock");
}
-BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
- BasicBlock *const SCEVCheckBlock =
- RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader);
- if (!SCEVCheckBlock)
- return nullptr;
-
- assert((!Cost->OptForSize ||
- Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
- "Cannot SCEV check stride or overflow when optimizing for size");
-
- introduceCheckBlockInVPlan(SCEVCheckBlock);
- return SCEVCheckBlock;
-}
-
-BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
- BasicBlock *const MemCheckBlock =
- RTChecks.emitMemRuntimeChecks(Bypass, LoopVectorPreHeader);
-
- // Check if we generated code that checks in runtime if arrays overlap. We put
- // the checks into a separate block to make the more common case of few
- // elements faster.
- if (!MemCheckBlock)
- return nullptr;
-
- // VPlan-native path does not do any analysis for runtime checks currently.
- assert((!EnableVPlanNativePath || OrigLoop->begin() == OrigLoop->end()) &&
- "Runtime checks are not supported for outer loops yet");
-
- if (Cost->OptForSize) {
- assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
- "Cannot emit memory checks when optimizing for size, unless forced "
- "to vectorize.");
- ORE->emit([&]() {
- return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
- OrigLoop->getStartLoc(),
- OrigLoop->getHeader())
- << "Code-size may be reduced by not forcing "
- "vectorization, or by source-code modifications "
- "eliminating the need for runtime checks "
- "(e.g., adding 'restrict').";
- });
- }
-
- introduceCheckBlockInVPlan(MemCheckBlock);
- return MemCheckBlock;
-}
-
/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
/// VPBB are moved to the end of the newly created VPIRBasicBlock. VPBB must
/// have a single predecessor, which is rewired to the new VPIRBasicBlock. All
@@ -2629,15 +2538,6 @@ BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
// to the scalar loop.
emitIterationCountCheck(LoopScalarPreHeader);
- // Generate the code to check any assumptions that we've made for SCEV
- // expressions.
- emitSCEVChecks(LoopScalarPreHeader);
-
- // Generate the code that checks in runtime if arrays overlap. We put the
- // checks into a separate block to make the more common case of few elements
- // faster.
- emitMemRuntimeChecks(LoopScalarPreHeader);
-
replaceVPBBWithIRVPBB(Plan.getScalarPreheader(), LoopScalarPreHeader);
return LoopVectorPreHeader;
}
@@ -7333,6 +7233,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
VPlanTransforms::runPass(VPlanTransforms::addBranchWeightToMiddleTerminator,
BestVPlan, BestVF);
+
+ if (!VectorizingEpilogue)
+ addRuntimeChecks(BestVPlan, ILV.RTChecks);
+
+ VPBasicBlock *VectorPH = cast<VPBasicBlock>(BestVPlan.getVectorPreheader());
VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
VPlanTransforms::narrowInterleaveGroups(
@@ -7380,7 +7285,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// 1. Set up the skeleton for vectorization, including vector pre-header and
// middle block. The vector loop is created during VPlan execution.
- VPBasicBlock *VectorPH = cast<VPBasicBlock>(Entry->getSuccessors()[1]);
+ BasicBlock *EntryBB =
+ cast<VPIRBasicBlock>(BestVPlan.getEntry())->getIRBasicBlock();
State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
if (VectorizingEpilogue)
VPlanTransforms::removeDeadRecipes(BestVPlan);
@@ -7404,6 +7310,12 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ILV.getOrCreateVectorTripCount(ILV.LoopVectorPreHeader), State);
replaceVPBBWithIRVPBB(VectorPH, State.CFG.PrevBB);
+ // Move check blocks to their final position.
+ if (BasicBlock *MemCheckBlock = ILV.RTChecks.getMemCheckBlock())
+ MemCheckBlock->moveAfter(EntryBB);
+ if (BasicBlock *SCEVCheckBlock = ILV.RTChecks.getSCEVCheckBlock())
+ SCEVCheckBlock->moveAfter(EntryBB);
+
BestVPlan.execute(&State);
// 2.5 When vectorizing the epilogue, fix reduction resume values from the
@@ -7504,15 +7416,6 @@ BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton() {
emitIterationCountCheck(LoopScalarPreHeader, true);
EPI.EpilogueIterationCountCheck->setName("iter.check");
- // Generate the code to check any assumptions that we've made for SCEV
- // expressions.
- EPI.SCEVSafetyCheck = emitSCEVChecks(LoopScalarPreHeader);
-
- // Generate the code that checks at runtime if arrays overlap. We put the
- // checks into a separate block to make the more common case of few elements
- // faster.
- EPI.MemSafetyCheck = emitMemRuntimeChecks(LoopScalarPreHeader);
-
// Generate the iteration count check for the main loop, *after* the check
// for the epilogue loop, so that the path-length is shorter for the case
// that goes directly through the vector epilogue. The longer-path length for
@@ -7616,6 +7519,13 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
+ BasicBlock *SCEVCheckBlock = RTChecks.getSCEVCheckBlock();
+ if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessorsOrMore(1))
+ EPI.SCEVSafetyCheck = SCEVCheckBlock;
+
+ BasicBlock *MemCheckBlock = RTChecks.getMemCheckBlock();
+ if (MemCheckBlock && MemCheckBlock->hasNPredecessorsOrMore(1))
+ EPI.MemSafetyCheck = MemCheckBlock;
if (EPI.SCEVSafetyCheck)
EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
@@ -9353,6 +9263,47 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
VPlanTransforms::runPass(VPlanTransforms::clearReductionWrapFlags, *Plan);
}
+void LoopVectorizationPlanner::addRuntimeChecks(
+ VPlan &Plan, GeneratedRTChecks &RTChecks) const {
+ SmallVector<std::pair<VPValue *, VPIRBasicBlock *>> Checks;
+ const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.emitSCEVChecks();
+ if (SCEVCheckBlock) {
+ assert((!CM.OptForSize ||
+ CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
+ "Cannot SCEV check stride or overflow when optimizing for size");
+ Checks.emplace_back(Plan.getOrAddLiveIn(SCEVCheckCond),
+ Plan.createVPIRBasicBlock(SCEVCheckBlock));
+ }
+ const auto &[MemCheckCond, MemCheckBlock] = RTChecks.emitMemRuntimeChecks();
+ if (MemCheckBlock) {
+ // VPlan-native path does not do any analysis for runtime checks
+ // currently.
+ assert((!EnableVPlanNativePath || OrigLoop->begin() == OrigLoop->end()) &&
+ "Runtime checks are not supported for outer loops yet");
+
+ if (CM.OptForSize) {
+ assert(
+ CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
+ "Cannot emit memory checks when optimizing for size, unless forced "
+ "to vectorize.");
+ ORE->emit([&]() {
+ return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
+ OrigLoop->getStartLoc(),
+ OrigLoop->getHeader())
+ << "Code-size may be reduced by not forcing "
+ "vectorization, or by source-code modifications "
+ "eliminating the need for runtime checks "
+ "(e.g., adding 'restrict').";
+ });
+ }
+ Checks.emplace_back(Plan.getOrAddLiveIn(MemCheckCond),
+ Plan.createVPIRBasicBlock(MemCheckBlock));
+ }
+ VPlanTransforms::connectCheckBlocks(
+ Plan, Checks,
+ hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()));
+}
+
void VPDerivedIVRecipe::execute(VPTransformState &State) {
assert(!State.Lane && "VPDerivedIVRecipe being replicated.");
@@ -9474,10 +9425,7 @@ static bool processLoopInVPlanNativePath(
VPlan &BestPlan = LVP.getPlanFor(VF.Width);
{
- bool AddBranchWeights =
- hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
- GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(),
- AddBranchWeights, CM.CostKind);
+ GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width,
VF.Width, 1, &CM, BFI, PSI, Checks, BestPlan);
LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
@@ -10116,10 +10064,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (ORE->allowExtraAnalysis(LV_NAME))
LVP.emitInvalidCostRemarks(ORE);
- bool AddBranchWeights =
- hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
- GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(),
- AddBranchWeights, CM.CostKind);
+ GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
if (LVP.hasPlanWithVF(VF.Width)) {
// Select the interleave count.
IC = CM.selectInterleaveCount(LVP.getPlanFor(VF.Width), VF.Width, VF.Cost);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 63ac80698643d..002f4ef2c000b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -490,7 +490,8 @@ void VPBasicBlock::connectToPredecessors(VPTransformState &State) {
unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
assert((TermBr && (!TermBr->getSuccessor(idx) ||
(isa<VPIRBasicBlock>(this) &&
- TermBr->getSuccessor(idx) == NewBB))) &&
+ (TermBr->getSuccessor(idx) == NewBB ||
+ PredVPBlock == getPlan()->getEntry())))) &&
"Trying to reset an existing successor block.");
TermBr->setSuccessor(idx, NewBB);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 593e5063802ba..a55e95ef274b7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -20,6 +20,7 @@
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/IR/MDBuilder.h"
#define DEBUG_TYPE "vplan"
@@ -589,3 +590,41 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->setName("vector loop");
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
+
+// Likelyhood of bypassing the vectorized loop because SCEV assumptions or
+// memory runtime checks.
+static constexpr uint32_t CheckBypassWeights[] = {1, 127};
+
+void VPlanTransforms::connectCheckBlocks(
+ VPlan &Plan, ArrayRef<std::pair<VPValue *, VPIRBasicBlock *>> Checks,
+ bool AddBranchWeights) {
+ VPBlockBase *VectorPH = Plan.getVectorPreheader();
+ VPBlockBase *ScalarPH = Plan.getScalarPreheader();
+ for (const auto &[Cond, CheckBlock] : Checks) {
+ VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor();
+ VPBlockUtils::insertOnEdge(PreVectorPH, VectorPH, CheckBlock);
+ VPBlockUtils::connectBlocks(CheckBlock, ScalarPH);
+ CheckBlock->swapSuccessors();
+
+ // We just connected a new block to the scalar preheader. Update all
+ // VPPhis by adding an incoming value for it, replicating the last value.
+ unsigned NumPredecessors = ScalarPH->getNumPredecessors();
+ for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) {
+ assert(isa<VPPhi>(&R) && "Phi expected to be VPPhi");
+ assert(cast<VPPhi>(&R)->getNumIncoming() == NumPredecessors - 1 &&
+ "must have incoming values for all operands");
+ R.addOperand(R.getOperand(NumPredecessors - 2));
+ }
+
+ VPIRMetadata VPBranchWeights;
+ auto *Term = VPBuilder(CheckBlock)
+ .createNaryOp(VPInstruction::BranchOnCond, {Cond},
+ Plan.getCanonicalIV()->getDebugLoc());
+ if (AddBranchWeights) {
+ MDBuilder MDB(Plan.getScalarHeader()->getIRBasicBlock()->getContext());
+ MDNode *BranchWeights =
+ MDB.createBranchWeights(CheckBypassWeights, /*IsExpected=*/false);
+ Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
+ }
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 40885cd52a127..515f656d4be3c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -74,6 +74,13 @@ struct VPlanTransforms {
/// flat CFG into a hierarchical CFG.
static void createLoopRegions(VPlan &Plan);
+ /// Connect the blocks in \p Checks to \p Plan, using the corresponding
+ /// VPValue as branch condition.
+ static void
+ connectCheckBlocks(VPlan &Plan,
+ ArrayRef<std::pair<VPValue *, VPIRBasicBlock *>> Checks,
+ bool AddBranchWeights);
+
/// Replaces the VPInstructions in \p Plan with corresponding
/// widen recipes. Returns false if any VPInstructions could not be converted
/// to a wide recipe if needed.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index dd8b7d6ea7e42..002fae6827557 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -210,6 +210,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: IR %8 = or i1 %7, %mul.overflow
; CHECK-NEXT: IR %9 = icmp ugt i64 %3, 4294967295
; CHECK-NEXT: IR %10 = or i1 %8, %9
+; CHECK-NEXT: EMIT branch-on-cond ir<%10>
; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vector.memcheck>:
@@ -218,6 +219,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: IR %13 = mul i64 %12, 4
; CHECK-NEXT: IR %14 = sub i64 %B1, %A2
; CHECK-NEXT: IR %diff.check = icmp ult i64 %14, %13
+; CHECK-NEXT: EMIT branch-on-cond ir<%diff.check>
; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vector.ph>:
@@ -227,22 +229,22 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: IR %n.vec = sub i64 %0, %n.mod.vf
; CHECK-NEXT: IR %17 = call i64 @llvm.vscale.i64()
; CHECK-NEXT: IR %18 = mul nuw i64 %17, 4
-; CHECK-NEXT: vp<%1> = DERIVED-IV ir<%0> + ir<%n.vec> * ir<-1>
-; CHECK-NEXT: vp<%2> = DERIVED-IV ir<%n> + ir<%n.vec> * ir<-1>
+; CHECK-NEXT: vp<%3> = DERIVED-IV ir<%0> + ir<%n.vec> * ir<-1>
+; CHECK-NEXT: vp<%4> = DERIVED-IV ir<%n> + ir<%n.vec> * ir<-1>
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<%index.next>, vector.body ]
-; CHECK-NEXT: vp<%3> = DERIVED-IV ir<%n> + vp<%index> * ir<-1>
-; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<%3>, ir<-1>
+; CHECK-NEXT: vp<%5> = DERIVED-IV ir<%n> + vp<%index> * ir<-1>
+; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT: vp<%4> = vector-end-pointer inbounds ir<%arrayidx>, ir<%18>
-; CHECK-NEXT: WIDEN ir<%19> = load vp<%4>
+; CHECK-NEXT: vp<%6> = vector-end-pointer inbounds ir<%arrayidx>, ir<%18>
+; CHECK-NEXT: WIDEN ir<%19> = load vp<%6>
; CHECK-NEXT: WIDEN ir<%add9> = add ir<%19>, ir<1>
; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT: vp<%5> = vector-end-pointer inbounds ir<%arrayidx3>, ir<%18>
-; CHECK-NEXT: WIDEN store vp<%5>, ir<%add9>
+; CHECK-NEXT: vp<%7> = vector-end-pointer inbounds ir<%arrayidx3>, ir<%18>
+; CHECK-NEXT: WIDEN store vp<%7>, ir<%add9>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<%18>.1
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, ir<%n.vec>
; CHECK-NEXT: Successor(s): middle.block, vector.body
@@ -256,8 +258,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<scalar.ph>:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%1>, middle.block ], [ ir<%0>, ir-bb<for.body.preheader> ], [ ir<%0>, ir-bb<vector.scevcheck> ], [ ir<%0>, ir-bb<vector.memcheck> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<%2>, middle.block ], [ ir<%n>, ir-bb<for.body.preheader> ], [ ir<%n>, ir-bb<vector.scevcheck> ], [ ir<%n>, ir-bb<vector.memcheck> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%3>, middle.block ], [ ir<%0>, ir-bb<for.body.preheader> ], [ ir<%0>, ir-bb<vector.scevcheck> ], [ ir<%0>, ir-bb<vector.memcheck> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<%4>, middle.block ], [ ir<%n>, ir-bb<for.body.preheader> ], [ ir<%n>, ir-bb<vector.scevcheck> ], [ ir<%n>, ir-bb<vector.memcheck> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -284,7 +286,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: br i1 %min.iters.check, label %scalar.ph, label %vector.scevcheck
; CHECK-NEXT: LV: vectorizing VPBB: ir-bb<vector.scevcheck> in BB: vector.scevcheck
; CHECK-NEXT: LV: filled BB:
-; CHECK-NEXT: vector.scevcheck: ; preds = %for.body.preheader
+; CHECK-NEXT: vector.scevcheck: ; No predecessors!
; CHECK-NEXT: %3 = add nsw i64 %0, -1
; CHECK-NEXT: %4 = add i32 %n, -1
; CHECK-NEXT: %5 = trunc i64 %3 to i32
@@ -296,25 +298,13 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: %8 = or i1 %7, %mul.overflow
; CHECK-NEXT: %9 = icmp ugt i64 %3, 4294967295
; CHECK-NEXT: %10 = or i1 %8, %9
-; CHECK-NEXT: br i1 %10, label %scalar.ph, label %vector.memcheck
-; CHECK-NEXT: LV: draw edge from for.body.preheader
-; CHECK-NEXT: LV: vectorizing VPBB: ir-bb<vector.memcheck> in BB: vector.memcheck
; CHECK-NEXT: LV: filled BB:
-; CHECK-NEXT: vector.memcheck: ; preds = %vector.scevcheck
+; CHECK-NEXT: vector.memcheck: ; No predecessors!
; CHECK-NEXT: %11 = call i64 @llvm.vscale.i64()
; CHECK-NEXT: %12 = mul nuw i64 %11, 4
; CHECK-NEXT: %13 = mul i64 %12, 4
; CHECK-NEXT: %14 = sub i64 %B1, %A2
; CHECK-NEXT: %diff.check = icmp ult i64 %14, %13
-; CHECK-NEXT: br i1 %diff.check, label %scalar.ph, label %vector.ph
-; CHECK-NEXT: LV: draw edge from vector.scevcheck
-; CHECK-NEXT: LV: vectorizing VPBB: ir-bb<vector.ph> in BB: vector.ph
-; CHECK-NEXT: LV: filled BB:
-; CHECK-NEXT: vector.ph: ; preds = %vector.memcheck
-; CHECK-NEXT: %15 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: %16 = mul nuw i64 %15, 4
-; CHECK-NEXT: %n.mod.vf = urem i64 %0, %16
-; CHECK-NEXT: %n.vec = sub i64 %0, %n.mod.vf
; CHECK-NEXT: %17 = call i64 @llvm.vscale.i64()
; CHECK-NEXT: %18 = mul nuw i64 %17, 4
; CHECK-NEXT: %19 = sub i64 %0, %n.vec
@@ -364,7 +354,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: draw edge from middle.block
; CHECK-NEXT: LV: vectorizing VPBB: ir-bb<scalar.ph> in BB: scalar.ph
; CHECK-NEXT: LV: filled BB:
-; CHECK-NEXT: scalar.ph: ; preds = %vector.memcheck, %vector.scevcheck, %for.body.preheader
+; CHECK-NEXT: scalar.ph: ; preds = %for.body.preheader
; CHECK-NEXT: %bc.resume.val = phi i64 [ %19, %middle.block ], [ %0, %for.body.preheader ], [ %0, %vector.scevcheck ], [ %0, %vector.memcheck ]
; CHECK-NEXT: %bc.resume.val5 = phi i32 [ %20, %middle.block ], [ %n, %for.body.preheader ], [ %n, %vector.scevcheck ], [ %n, %vector.memcheck ]
; CHECK-NEXT: br label %for.body
@@ -619,6 +609,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: IR %8 = or i1 %7, %mul.overflow
; CHECK-NEXT: IR %9 = icmp ugt i64 %3, 4294967295
; CHECK-NEXT: IR %10 = or i1 %8, %9
+; CHECK-NEXT: EMIT branch-on-cond ir<%10>
; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vector.memcheck>:
@@ -627,6 +618,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: IR %13 = mul i64 %12, 4
; CHECK-NEXT: IR %14 = sub i64 %B1, %A2
; CHECK-NEXT: IR %diff.check = icmp ult i64 %14, %13
+; CHECK-NEXT: EMIT branch-on-cond ir<%diff.check>
; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<vector.ph>:
@@ -635,23 +627,20 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: IR %n.mod.vf = urem i64 %0, %16
; CHECK-NEXT: IR %n.vec = sub i64 %0, %n.mod.vf
; CHECK-NEXT: IR %17 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: IR %18 = mul nuw i64 %17, 4
-; CHECK-NEXT: vp<%1> = DERIVED-IV ir<%0> + ir<%n.vec> * ir<-1>
-; CHECK-NEXT: vp<%2> = DERIVED-IV ir<%n> + ir<%n.vec> * ir<-1>
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<%index.next>, vector.body ]
-; CHECK-NEXT: vp<%3> = DERIVED-IV ir<%n> + vp<%index> * ir<-1>
-; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<%3>, ir<-1>
+; CHECK-NEXT: vp<%5> = DERIVED-IV ir<%n> + vp<%index> * ir<-1>
+; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<%5>, ir<-1>
; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
-; CHECK-NEXT: vp<%4> = vector-end-pointer inbounds ir<%arrayidx>, ir<%18>
-; CHECK-NEXT: WIDEN ir<%19> = load vp<%4>
+; CHECK-NEXT: vp<%6> = vector-end-pointer inbounds ir<%arrayidx>, ir<%18>
+; CHECK-NEXT: WIDEN ir<%19> = load vp<%6>
; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%19>, ir<1.000000e+00>
; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
-; CHECK-NEXT: vp<%5> = vector-end-pointer inbounds ir<%arrayidx3>, ir<%18>
-; CHECK-NEXT: WIDEN store vp<%5>, ir<%conv1>
+; CHECK-NEXT: vp<%7> = vector-end-pointer inbounds ir<%arrayidx3>, ir<%18>
+; CHECK-NEXT: WIDEN store vp<%7>, ir<%conv1>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<%18>.1
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, ir<%n.vec>
; CHECK-NEXT: Successor(s): middle.block, vector.body
@@ -665,8 +654,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<scalar.ph>:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%1>, middle.block ], [ ir<%0>, ir-bb<for.body.preheader> ], [ ir<%0>, ir-bb<vector.scevcheck> ], [ ir<%0>, ir-bb<vector.memcheck> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<%2>, middle.block ], [ ir<%n>, ir-bb<for.body.preheader> ], [ ir<%n>, ir-bb<vector.scevcheck> ], [ ir<%n>, ir-bb<vector.memcheck> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<%3>, middle.block ], [ ir<%0>, ir-bb<for.body.preheader> ], [ ir<%0>, ir-bb<vector.scevcheck> ], [ ir<%0>, ir-bb<vector.memcheck> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<%4>, middle.block ], [ ir<%n>, ir-bb<for.body.preheader> ], [ ir<%n>, ir-bb<vector.scevcheck> ], [ ir<%n>, ir-bb<vector.memcheck> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -690,10 +679,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: %1 = call i64 @llvm.vscale.i64()
; CHECK-NEXT: %2 = mul nuw i64 %1, 4
; CHECK-NEXT: %min.iters.check = icmp ult i64 %0, %2
-; CHECK-NEXT: br i1 %min.iters.check, label %scalar.ph, label %vector.scevcheck
-; CHECK-NEXT: LV: vectorizing VPBB: ir-bb<vector.scevcheck> in BB: vector.scevcheck
; CHECK-NEXT: LV: filled BB:
-; CHECK-NEXT: vector.scevcheck: ; preds = %for.body.preheader
+; CHECK-NEXT: vector.scevcheck: ; No predecessors!
; CHECK-NEXT: %3 = add nsw i64 %0, -1
; CHECK-NEXT: %4 = add i32 %n, -1
; CHECK-NEXT: %5 = trunc i64 %3 to i32
@@ -705,21 +692,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: %8 = or i1 %7, %mul.overflow
; CHECK-NEXT: %9 = icmp ugt i64 %3, 4294967295
; CHECK-NEXT: %10 = or i1 %8, %9
-; CHECK-NEXT: br i1 %10, label %scalar.ph, label %vector.memcheck
-; CHECK-NEXT: LV: draw edge from for.body.preheader
-; CHECK-NEXT: LV: vectorizing VPBB: ir-bb<vector.memcheck> in BB: vector.memcheck
-; CHECK-NEXT: LV: filled BB:
-; CHECK-NEXT: vector.memcheck: ; preds = %vector.scevcheck
-; CHECK-NEXT: %11 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: %12 = mul nuw i64 %11, 4
-; CHECK-NEXT: %13 = mul i64 %12, 4
-; CHECK-NEXT: %14 = sub i64 %B1, %A2
-; CHECK-NEXT: %diff.check = icmp ult i64 %14, %13
-; CHECK-NEXT: br i1 %diff.check, label %scalar.ph, label %vector.ph
-; CHECK-NEXT: LV: draw edge from vector.scevcheck
-; CHECK-NEXT: LV: vectorizing VPBB: ir-bb<vector.ph> in BB: vector.ph
; CHECK-NEXT: LV: filled BB:
-; CHECK-NEXT: vector.ph: ; preds = %vector.memcheck
+; CHECK-NEXT: vector.ph: ; No predecessors!
; CHECK-NEXT: %15 = call i64 @llvm.vscale.i64()
; CHECK-NEXT: %16 = mul nuw i64 %15, 4
; CHECK-NEXT: %n.mod.vf = urem i64 %0, %16
@@ -773,7 +747,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: draw edge from middle.block
; CHECK-NEXT: LV: vectorizing VPBB: ir-bb<scalar.ph> in BB: scalar.ph
; CHECK-NEXT: LV: filled BB:
-; CHECK-NEXT: scalar.ph: ; preds = %vector.memcheck, %vector.scevcheck, %for.body.preheader
+; CHECK-NEXT: scalar.ph: ; preds = %for.body.preheader
; CHECK-NEXT: %bc.resume.val = phi i64 [ %19, %middle.block ], [ %0, %for.body.preheader ], [ %0, %vector.scevcheck ], [ %0, %vector.memcheck ]
; CHECK-NEXT: %bc.resume.val5 = phi i32 [ %20, %middle.block ], [ %n, %for.body.preheader ], [ %n, %vector.scevcheck ], [ %n, %vector.memcheck ]
; CHECK-NEXT: br label %for.body
>From 044d73ac383da7cc1db481f8f1902733c1904287 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 27 Jun 2025 10:44:27 +0100
Subject: [PATCH 2/2] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 34 ++++++++++---------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 5 +++
.../Vectorize/VPlanConstruction.cpp | 4 +--
3 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b65a21a0e6a9b..8641b510667ed 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -641,8 +641,6 @@ struct EpilogueLoopVectorizationInfo {
unsigned EpilogueUF = 0;
BasicBlock *MainLoopIterationCountCheck = nullptr;
BasicBlock *EpilogueIterationCountCheck = nullptr;
- BasicBlock *SCEVSafetyCheck = nullptr;
- BasicBlock *MemSafetyCheck = nullptr;
Value *TripCount = nullptr;
Value *VectorTripCount = nullptr;
VPlan &EpiloguePlan;
@@ -7514,18 +7512,21 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
+ // Retrieve blocks with SCEV and memory runtime checks, if they have been
+ // connected to the CFG, otherwise they are unused and will be deleted. Their
+ // terminators and phis using them need adjusting below.
BasicBlock *SCEVCheckBlock = RTChecks.getSCEVCheckBlock();
- if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessorsOrMore(1))
- EPI.SCEVSafetyCheck = SCEVCheckBlock;
-
+ if (SCEVCheckBlock && pred_empty(SCEVCheckBlock))
+ SCEVCheckBlock = nullptr;
BasicBlock *MemCheckBlock = RTChecks.getMemCheckBlock();
- if (MemCheckBlock && MemCheckBlock->hasNPredecessorsOrMore(1))
- EPI.MemSafetyCheck = MemCheckBlock;
- if (EPI.SCEVSafetyCheck)
- EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith(
+ if (MemCheckBlock && pred_empty(MemCheckBlock))
+ MemCheckBlock = nullptr;
+
+ if (SCEVCheckBlock)
+ SCEVCheckBlock->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
- if (EPI.MemSafetyCheck)
- EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith(
+ if (MemCheckBlock)
+ MemCheckBlock->getTerminator()->replaceUsesOfWith(
VecEpilogueIterationCountCheck, LoopScalarPreHeader);
DT->changeImmediateDominator(LoopScalarPreHeader,
@@ -7552,10 +7553,10 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
}))
continue;
Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck);
- if (EPI.SCEVSafetyCheck)
- Phi->removeIncomingValue(EPI.SCEVSafetyCheck);
- if (EPI.MemSafetyCheck)
- Phi->removeIncomingValue(EPI.MemSafetyCheck);
+ if (SCEVCheckBlock)
+ Phi->removeIncomingValue(SCEVCheckBlock);
+ if (MemCheckBlock)
+ Phi->removeIncomingValue(MemCheckBlock);
}
replaceVPBBWithIRVPBB(Plan.getScalarPreheader(), LoopScalarPreHeader);
@@ -9273,7 +9274,7 @@ void LoopVectorizationPlanner::addRuntimeChecks(
if (MemCheckBlock) {
// VPlan-native path does not do any analysis for runtime checks
// currently.
- assert((!EnableVPlanNativePath || OrigLoop->begin() == OrigLoop->end()) &&
+ assert((!EnableVPlanNativePath || OrigLoop->isInnermost()) &&
"Runtime checks are not supported for outer loops yet");
if (CM.OptForSize) {
@@ -9294,6 +9295,7 @@ void LoopVectorizationPlanner::addRuntimeChecks(
Checks.emplace_back(Plan.getOrAddLiveIn(MemCheckCond),
Plan.createVPIRBasicBlock(MemCheckBlock));
}
+
VPlanTransforms::connectCheckBlocks(
Plan, Checks,
hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()));
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 0abd936e28398..3b629b42e2752 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -487,6 +487,11 @@ void VPBasicBlock::connectToPredecessors(VPTransformState &State) {
} else {
// Set each forward successor here when it is created, excluding
// backedges. A backward successor is set when the branch is created.
+ // Branches to VPIRBasicBlocks must have the same successors in VPlan as
+ // in the original IR, except if the predecessors is the entry block. This
+ // enables including SCEV and memory runtime check blocks in VPlan.
+ // TODO: Remove exception by modeling branch in the entry block using
+ // BranchOnCond.
unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
assert((TermBr && (!TermBr->getSuccessor(idx) ||
(isa<VPIRBasicBlock>(this) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a55e95ef274b7..1f594260c4426 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -591,8 +591,8 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
-// Likelyhood of bypassing the vectorized loop because SCEV assumptions or
-// memory runtime checks.
+// Likelyhood of bypassing the vectorized loop due to SCEV or memory runtime
+// checks.
static constexpr uint32_t CheckBypassWeights[] = {1, 127};
void VPlanTransforms::connectCheckBlocks(
More information about the llvm-commits
mailing list