[llvm] [LV] Compute SCEV for memcheck before unlinking (PR #160326)
Igor Kirillov via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 26 07:07:33 PDT 2025
https://github.com/igogo-x86 updated https://github.com/llvm/llvm-project/pull/160326
>From da4ec02c53cc549a22ae53792c8643a33d1a1062 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Tue, 23 Sep 2025 15:27:40 +0000
Subject: [PATCH 1/4] [LV] Compute SCEV for memcheck before unlinking
When generating runtime memory checks for outer loops, we split
blocks and later unlink them, making the memcheck block unreachable.
For instructions in unreachable blocks, ScalarEvolution returns an
unknown/poison SCEV, which is treated as a constant and thus
loop-invariant. The cost model then assumes the check can be hoisted
and underestimates its cost.
Set OuterLoop early in GeneratedRTChecks::create and compute the
SCEV for MemRuntimeCheckCond before unlinking, so getCost() sees the
cached expression rather than a poison constant.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ca092dcfcb492..575f45b051cf6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1807,6 +1807,10 @@ class GeneratedRTChecks {
BasicBlock *LoopHeader = L->getHeader();
BasicBlock *Preheader = L->getLoopPreheader();
+ // Outer loop is used as part of later cost calculations (e.g. to
+ // determine if runtime checks are loop-invariant and can be hoisted).
+ OuterLoop = L->getParentLoop();
+
// Use SplitBlock to create blocks for SCEV & memory runtime checks to
// ensure the blocks are properly added to LoopInfo & DominatorTree. Those
// may be used by SCEVExpander. The blocks will be un-linked from their
@@ -1850,6 +1854,11 @@ class GeneratedRTChecks {
assert(MemRuntimeCheckCond &&
"no RT checks generated although RtPtrChecking "
"claimed checks are required");
+ // Compute SCEV while the block is reachable.
+ // After unlinking, SCEV returns unknown/poison (constant -> invariant),
+ // which makes getCost() wrongly discount hoisted checks.
+ if (OuterLoop)
+ PSE.getSE()->getSCEV(MemRuntimeCheckCond);
}
SCEVExp.eraseDeadInstructions(SCEVCheckCond);
@@ -1889,8 +1898,6 @@ class GeneratedRTChecks {
LI->removeBlock(SCEVCheckBlock);
}
- // Outer loop is used as part of the later cost calculations.
- OuterLoop = L->getParentLoop();
}
InstructionCost getCost() {
>From a14a3a32d64623b802b71bec73c58cac522723b6 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 24 Sep 2025 10:24:44 +0000
Subject: [PATCH 2/4] Fix
---
.../include/llvm/Transforms/Utils/LoopUtils.h | 3 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 15 +++++++-
llvm/lib/Transforms/Utils/LoopVersioning.cpp | 6 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 13 ++-----
.../AArch64/low_trip_memcheck_cost.ll | 38 +++++++++++++++++++
5 files changed, 62 insertions(+), 13 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 2d2355d6be68a..2ff2dda9744c4 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -578,7 +578,8 @@ LLVM_ABI Loop *cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM, LoopInfo *LI,
LLVM_ABI Value *
addRuntimeChecks(Instruction *Loc, Loop *TheLoop,
const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
- SCEVExpander &Expander, bool HoistRuntimeChecks = false);
+ SCEVExpander &Expander, bool HoistRuntimeChecks,
+ bool &AllChecksHoisted);
LLVM_ABI Value *addDiffRuntimeChecks(
Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index b6ba82288aeb4..58e24f07eb913 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2038,18 +2038,23 @@ expandBounds(const SmallVectorImpl<RuntimePointerCheck> &PointerChecks, Loop *L,
Value *llvm::addRuntimeChecks(
Instruction *Loc, Loop *TheLoop,
const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
- SCEVExpander &Exp, bool HoistRuntimeChecks) {
+ SCEVExpander &Exp, bool HoistRuntimeChecks, bool &AllChecksHoisted) {
// TODO: Move noalias annotation code from LoopVersioning here and share with LV if possible.
// TODO: Pass RtPtrChecking instead of PointerChecks and SE separately, if possible
auto ExpandedChecks =
expandBounds(PointerChecks, TheLoop, Loc, Exp, HoistRuntimeChecks);
LLVMContext &Ctx = Loc->getContext();
+ auto *SE = Exp.getSE();
+ auto *OuterLoop = TheLoop->getParentLoop();
+ AllChecksHoisted = HoistRuntimeChecks && OuterLoop;
IRBuilder ChkBuilder(Ctx, InstSimplifyFolder(Loc->getDataLayout()));
ChkBuilder.SetInsertPoint(Loc);
// Our instructions might fold to a constant.
Value *MemoryRuntimeCheck = nullptr;
+ AllChecksHoisted = HoistRuntimeChecks && OuterLoop;
+
for (const auto &[A, B] : ExpandedChecks) {
// Check if two pointers (A and B) conflict where conflict is computed as:
// start(A) <= end(B) && start(B) <= end(A)
@@ -2070,6 +2075,14 @@ Value *llvm::addRuntimeChecks(
// IsConflict = bound0 & bound1
Value *Cmp0 = ChkBuilder.CreateICmpULT(A.Start, B.End, "bound0");
Value *Cmp1 = ChkBuilder.CreateICmpULT(B.Start, A.End, "bound1");
+
+ if (AllChecksHoisted) {
+ AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(A.Start), OuterLoop);
+ AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(B.Start), OuterLoop);
+ AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(A.End), OuterLoop);
+ AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(B.End), OuterLoop);
+ }
+
Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");
if (A.StrideToCheck) {
Value *IsNegativeStride = ChkBuilder.CreateICmpSLT(
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index ec2e6c1ab796b..6ba69b52c9234 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -63,8 +63,10 @@ void LoopVersioning::versionLoop(
SCEVExpander Exp2(*RtPtrChecking.getSE(),
VersionedLoop->getHeader()->getDataLayout(),
"induction");
- MemRuntimeCheck = addRuntimeChecks(RuntimeCheckBB->getTerminator(),
- VersionedLoop, AliasChecks, Exp2);
+ bool AllCheckHoisted;
+ MemRuntimeCheck =
+ addRuntimeChecks(RuntimeCheckBB->getTerminator(), VersionedLoop,
+ AliasChecks, Exp2, false, AllCheckHoisted);
SCEVExpander Exp(*SE, RuntimeCheckBB->getDataLayout(),
"scev.check");
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 575f45b051cf6..b923a1a7de6e0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1763,6 +1763,8 @@ class GeneratedRTChecks {
/// If it is nullptr no memory runtime checks have been generated.
Value *MemRuntimeCheckCond = nullptr;
+ bool AllCheckHoisted = false;
+
DominatorTree *DT;
LoopInfo *LI;
TargetTransformInfo *TTI;
@@ -1849,16 +1851,11 @@ class GeneratedRTChecks {
} else {
MemRuntimeCheckCond = addRuntimeChecks(
MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
- MemCheckExp, VectorizerParams::HoistRuntimeChecks);
+ MemCheckExp, VectorizerParams::HoistRuntimeChecks, AllCheckHoisted);
}
assert(MemRuntimeCheckCond &&
"no RT checks generated although RtPtrChecking "
"claimed checks are required");
- // Compute SCEV while the block is reachable.
- // After unlinking, SCEV returns unknown/poison (constant -> invariant),
- // which makes getCost() wrongly discount hoisted checks.
- if (OuterLoop)
- PSE.getSE()->getSCEV(MemRuntimeCheckCond);
}
SCEVExp.eraseDeadInstructions(SCEVCheckCond);
@@ -1935,13 +1932,11 @@ class GeneratedRTChecks {
// the checks will likely be hoisted out and so the effective cost will
// reduce according to the outer loop trip count.
if (OuterLoop) {
- ScalarEvolution *SE = MemCheckExp.getSE();
// TODO: If profitable, we could refine this further by analysing every
// individual memory check, since there could be a mixture of loop
// variant and invariant checks that mean the final condition is
// variant.
- const SCEV *Cond = SE->getSCEV(MemRuntimeCheckCond);
- if (SE->isLoopInvariant(Cond, OuterLoop)) {
+ if (AllCheckHoisted) {
// It seems reasonable to assume that we can reduce the effective
// cost of the checks even when we know nothing about the trip
// count. Assume that the outer loop executes at least twice.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
index 611b980999bfe..636efe5a94056 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
@@ -250,6 +250,44 @@ outer.exit:
ret void
}
+define void @outer_cannot_hoist(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, ptr nocapture noundef readonly %offsets, i64 noundef %n) {
+; CHECK-LABEL: LV: Checking a loop in 'outer_cannot_hoist'
+; CHECK: Calculating cost of runtime checks:
+; CHECK-NOT: We expect runtime memory checks to be hoisted out of the outer loop.
+; CHECK: Total cost of runtime checks: 7
+; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:8
+entry:
+ br label %outer.loop
+
+outer.loop:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %inner.exit ]
+ %offset.ptr = getelementptr inbounds i64, ptr %offsets, i64 %outer.iv
+ %offset = load i64, ptr %offset.ptr, align 8
+ %0 = mul nsw i64 %offset, %n
+ br label %inner.loop
+
+inner.loop:
+ %iv.inner = phi i64 [ 0, %outer.loop ], [ %iv.inner.next, %inner.loop ]
+ %1 = add nuw nsw i64 %iv.inner, %0
+ %arrayidx.us = getelementptr inbounds i32, ptr %src, i64 %1
+ %2 = load i32, ptr %arrayidx.us, align 4
+ %arrayidx8.us = getelementptr inbounds i32, ptr %dst, i64 %1
+ %3 = load i32, ptr %arrayidx8.us, align 4
+ %add9.us = add nsw i32 %3, %2
+ store i32 %add9.us, ptr %arrayidx8.us, align 4
+ %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+ %inner.exit.cond = icmp eq i64 %iv.inner.next, %n
+ br i1 %inner.exit.cond, label %inner.exit, label %inner.loop
+
+inner.exit:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.exit.cond = icmp eq i64 %outer.iv.next, 3
+ br i1 %outer.exit.cond, label %outer.exit, label %outer.loop
+
+outer.exit:
+ ret void
+}
+
!0 = !{!"branch_weights", i32 10, i32 20}
!1 = !{!"branch_weights", i32 1, i32 -1}
>From 32ed96bb6d6b2daffd105e973456e94f2c4ce12d Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Wed, 24 Sep 2025 10:55:37 +0000
Subject: [PATCH 3/4] Add test, and also the patch didn't work
---
llvm/lib/Transforms/Utils/LoopUtils.cpp | 25 +++++++++++--------
llvm/lib/Transforms/Utils/LoopVersioning.cpp | 4 +--
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +++---
...ive-path-inner-loop-with-runtime-checks.ll | 3 +--
4 files changed, 22 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 58e24f07eb913..948fa7f7e37d4 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2047,14 +2047,12 @@ Value *llvm::addRuntimeChecks(
LLVMContext &Ctx = Loc->getContext();
auto *SE = Exp.getSE();
auto *OuterLoop = TheLoop->getParentLoop();
- AllChecksHoisted = HoistRuntimeChecks && OuterLoop;
+ AllChecksHoisted = HoistRuntimeChecks && OuterLoop != nullptr;
IRBuilder ChkBuilder(Ctx, InstSimplifyFolder(Loc->getDataLayout()));
ChkBuilder.SetInsertPoint(Loc);
// Our instructions might fold to a constant.
Value *MemoryRuntimeCheck = nullptr;
- AllChecksHoisted = HoistRuntimeChecks && OuterLoop;
-
for (const auto &[A, B] : ExpandedChecks) {
// Check if two pointers (A and B) conflict where conflict is computed as:
// start(A) <= end(B) && start(B) <= end(A)
@@ -2076,13 +2074,6 @@ Value *llvm::addRuntimeChecks(
Value *Cmp0 = ChkBuilder.CreateICmpULT(A.Start, B.End, "bound0");
Value *Cmp1 = ChkBuilder.CreateICmpULT(B.Start, A.End, "bound1");
- if (AllChecksHoisted) {
- AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(A.Start), OuterLoop);
- AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(B.Start), OuterLoop);
- AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(A.End), OuterLoop);
- AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(B.End), OuterLoop);
- }
-
Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");
if (A.StrideToCheck) {
Value *IsNegativeStride = ChkBuilder.CreateICmpSLT(
@@ -2096,6 +2087,20 @@ Value *llvm::addRuntimeChecks(
"stride.check");
IsConflict = ChkBuilder.CreateOr(IsConflict, IsNegativeStride);
}
+
+ if (AllChecksHoisted) {
+ AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(A.Start), OuterLoop);
+ AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(B.Start), OuterLoop);
+ AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(A.End), OuterLoop);
+ AllChecksHoisted &= SE->isLoopInvariant(SE->getSCEV(B.End), OuterLoop);
+ if (A.StrideToCheck)
+ AllChecksHoisted &=
+ SE->isLoopInvariant(SE->getSCEV(A.StrideToCheck), OuterLoop);
+ if (B.StrideToCheck)
+ AllChecksHoisted &=
+ SE->isLoopInvariant(SE->getSCEV(B.StrideToCheck), OuterLoop);
+ }
+
if (MemoryRuntimeCheck) {
IsConflict =
ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict, "conflict.rdx");
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index 6ba69b52c9234..a844d7847fa22 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -63,10 +63,10 @@ void LoopVersioning::versionLoop(
SCEVExpander Exp2(*RtPtrChecking.getSE(),
VersionedLoop->getHeader()->getDataLayout(),
"induction");
- bool AllCheckHoisted;
+ bool AllChecksHoisted;
MemRuntimeCheck =
addRuntimeChecks(RuntimeCheckBB->getTerminator(), VersionedLoop,
- AliasChecks, Exp2, false, AllCheckHoisted);
+ AliasChecks, Exp2, false, AllChecksHoisted);
SCEVExpander Exp(*SE, RuntimeCheckBB->getDataLayout(),
"scev.check");
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b923a1a7de6e0..a8b846ab26a34 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1763,7 +1763,7 @@ class GeneratedRTChecks {
/// If it is nullptr no memory runtime checks have been generated.
Value *MemRuntimeCheckCond = nullptr;
- bool AllCheckHoisted = false;
+ bool AllChecksHoisted = false;
DominatorTree *DT;
LoopInfo *LI;
@@ -1851,7 +1851,8 @@ class GeneratedRTChecks {
} else {
MemRuntimeCheckCond = addRuntimeChecks(
MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
- MemCheckExp, VectorizerParams::HoistRuntimeChecks, AllCheckHoisted);
+ MemCheckExp, VectorizerParams::HoistRuntimeChecks,
+ AllChecksHoisted);
}
assert(MemRuntimeCheckCond &&
"no RT checks generated although RtPtrChecking "
@@ -1894,7 +1895,6 @@ class GeneratedRTChecks {
DT->eraseNode(SCEVCheckBlock);
LI->removeBlock(SCEVCheckBlock);
}
-
}
InstructionCost getCost() {
@@ -1936,7 +1936,7 @@ class GeneratedRTChecks {
// individual memory check, since there could be a mixture of loop
// variant and invariant checks that mean the final condition is
// variant.
- if (AllCheckHoisted) {
+ if (AllChecksHoisted) {
// It seems reasonable to assume that we can reduce the effective
// cost of the checks even when we know nothing about the trip
// count. Assume that the outer loop executes at least twice.
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
index 9ace6be64b69a..e7b0db4269da9 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
@@ -20,8 +20,7 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
; CHECK-NEXT: [[SMAX6:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 1000)
; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[SMAX6]], 4
; CHECK-NEXT: [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT: [[SMAX8:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 1000)
-; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[SMAX8]], [[TMP0]]
+; CHECK-NEXT: [[TMP8:%.*]] = sub i64 [[SMAX6]], [[TMP0]]
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
>From 6a482e8688a39e498b922a4bca1b81a960dd5bf9 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Fri, 26 Sep 2025 14:04:57 +0000
Subject: [PATCH 4/4] Address comments
---
llvm/include/llvm/Transforms/Utils/LoopUtils.h | 2 ++
llvm/lib/Transforms/Utils/LoopUtils.cpp | 1 -
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 9 +++++----
3 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 2ff2dda9744c4..7dfa21696201e 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -575,6 +575,8 @@ LLVM_ABI Loop *cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM, LoopInfo *LI,
/// Add code that checks at runtime if the accessed arrays in \p PointerChecks
/// overlap. Returns the final comparator value or NULL if no check is needed.
+/// If \p HoistRuntimeChecks and \p TheLoop has a parent, sets \p
+/// AllChecksHoisted when all checks are outer-loop invariant (hoistable).
LLVM_ABI Value *
addRuntimeChecks(Instruction *Loc, Loop *TheLoop,
const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 948fa7f7e37d4..41b0bb0b2b318 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2073,7 +2073,6 @@ Value *llvm::addRuntimeChecks(
// IsConflict = bound0 & bound1
Value *Cmp0 = ChkBuilder.CreateICmpULT(A.Start, B.End, "bound0");
Value *Cmp1 = ChkBuilder.CreateICmpULT(B.Start, A.End, "bound1");
-
Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");
if (A.StrideToCheck) {
Value *IsNegativeStride = ChkBuilder.CreateICmpSLT(
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a8b846ab26a34..32a528c4328d0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1763,6 +1763,8 @@ class GeneratedRTChecks {
/// If it is nullptr no memory runtime checks have been generated.
Value *MemRuntimeCheckCond = nullptr;
+ /// True if memory checks are outer-loop invariant (hoistable).
+ /// Used to discount check cost for inner loops.
bool AllChecksHoisted = false;
DominatorTree *DT;
@@ -1809,10 +1811,6 @@ class GeneratedRTChecks {
BasicBlock *LoopHeader = L->getHeader();
BasicBlock *Preheader = L->getLoopPreheader();
- // Outer loop is used as part of later cost calculations (e.g. to
- // determine if runtime checks are loop-invariant and can be hoisted).
- OuterLoop = L->getParentLoop();
-
// Use SplitBlock to create blocks for SCEV & memory runtime checks to
// ensure the blocks are properly added to LoopInfo & DominatorTree. Those
// may be used by SCEVExpander. The blocks will be un-linked from their
@@ -1895,6 +1893,9 @@ class GeneratedRTChecks {
DT->eraseNode(SCEVCheckBlock);
LI->removeBlock(SCEVCheckBlock);
}
+
+ // Outer loop is used as part of the later cost calculations.
+ OuterLoop = L->getParentLoop();
}
InstructionCost getCost() {
More information about the llvm-commits
mailing list