[llvm] [SCEVExpander] Expand UDiv avoiding UB when in seq_min/max. (PR #92177)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 15 03:59:14 PDT 2024
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/92177
>From bff4e5adb6a438947898e5852af3367b911b6635 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 14 May 2024 20:36:16 +0100
Subject: [PATCH 1/7] [LV] Don't vectorize if trip count expansion may
introduce UB.
Introduce a utility to check if a SCEV expansion may introduce UB
(couldn't find a similar utility after a quick glance) and use to the
avoid vectorizing when expanding the trip count introduces UB.
Fixes https://github.com/llvm/llvm-project/issues/89958.
!fixup introduce SafeUDivMode to SCEVExpander.
Step
---
.../Utils/ScalarEvolutionExpander.h | 13 +++++++
llvm/lib/Analysis/ScalarEvolution.cpp | 11 ++++--
.../Utils/ScalarEvolutionExpander.cpp | 15 +++++++-
.../trip-count-expansion-may-introduce-ub.ll | 36 +++++++++++++------
4 files changed, 61 insertions(+), 14 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 62c1e15a9a60e1..cb3b7372f7e66e 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -124,6 +124,11 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
/// "expanded" form.
bool LSRMode;
+ /// When true, rewrite any divisors of UDiv expressions that may be 0 to
+ /// umax(Divisor, 1) to avoid introducing UB. If the divisor may be poison,
+ /// freeze it first.
+ bool SafeUDivMode = false;
+
typedef IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> BuilderType;
BuilderType Builder;
@@ -300,6 +305,9 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
/// location and their operands are defined at this location.
bool isSafeToExpandAt(const SCEV *S, const Instruction *InsertionPoint) const;
+ static bool isSafeToExpand(const SCEV *S, bool CanonicalMode,
+ ScalarEvolution &SE);
+
/// Insert code to directly compute the specified SCEV expression into the
/// program. The code is inserted into the specified block.
Value *expandCodeFor(const SCEV *SH, Type *Ty, BasicBlock::iterator I);
@@ -418,6 +426,11 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
BasicBlock::iterator findInsertPointAfter(Instruction *I,
Instruction *MustDominate) const;
+ static const SCEV *rewriteExpressionToRemoveUB(const SCEV *BTC, Loop *L,
+ ScalarEvolution &SE);
+
+ void setSafeUDivMode() { SafeUDivMode = true; }
+
private:
LLVMContext &getContext() const { return SE.getContext(); }
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 1d3443588ce60d..b4d02df02694ca 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -4304,12 +4304,19 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
}
for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
+ bool MayBeUB = SCEVExprContains(Ops[i], [](const SCEV *S) {
+ auto *UDiv = dyn_cast<SCEVUDivExpr>(S);
+ return UDiv && !isa<SCEVConstant>(UDiv->getOperand(1));
+ });
+
+ if (MayBeUB)
+ continue;
// We can replace %x umin_seq %y with %x umin %y if either:
// * %y being poison implies %x is also poison.
// * %x cannot be the saturating value (e.g. zero for umin).
if (::impliesPoison(Ops[i], Ops[i - 1]) ||
- isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, Ops[i - 1],
- SaturationPoint)) {
+ (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, Ops[i - 1],
+ SaturationPoint))) {
SmallVector<const SCEV *> SeqOps = {Ops[i - 1], Ops[i]};
Ops[i - 1] = getMinMaxExpr(
SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Kind),
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 0927a3015818fd..ee7b2f74db6fa9 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -665,6 +665,12 @@ Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
}
Value *RHS = expand(S->getRHS());
+ if (SafeUDivMode && !SE.isKnownNonZero(S->getRHS())) {
+ if (!isa<SCEVConstant>(S->getRHS()))
+ RHS = Builder.CreateFreeze(RHS);
+ RHS = Builder.CreateIntrinsic(RHS->getType(), Intrinsic::umax,
+ {RHS, ConstantInt::get(RHS->getType(), 1)});
+ }
return InsertBinop(Instruction::UDiv, LHS, RHS, SCEV::FlagAnyWrap,
/*IsSafeToHoist*/ SE.isKnownNonZero(S->getRHS()));
}
@@ -1358,11 +1364,13 @@ Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
Value *SCEVExpander::expandMinMaxExpr(const SCEVNAryExpr *S,
Intrinsic::ID IntrinID, Twine Name,
bool IsSequential) {
+ SafeUDivMode = true;
Value *LHS = expand(S->getOperand(S->getNumOperands() - 1));
Type *Ty = LHS->getType();
if (IsSequential)
LHS = Builder.CreateFreeze(LHS);
for (int i = S->getNumOperands() - 2; i >= 0; --i) {
+ SafeUDivMode = i != 0;
Value *RHS = expand(S->getOperand(i));
if (IsSequential && i != 0)
RHS = Builder.CreateFreeze(RHS);
@@ -2315,12 +2323,17 @@ struct SCEVFindUnsafe {
};
} // namespace
-bool SCEVExpander::isSafeToExpand(const SCEV *S) const {
+bool SCEVExpander::isSafeToExpand(const SCEV *S, bool CanonicalMode,
+ ScalarEvolution &SE) {
SCEVFindUnsafe Search(SE, CanonicalMode);
visitAll(S, Search);
return !Search.IsUnsafe;
}
+bool SCEVExpander::isSafeToExpand(const SCEV *S) const {
+ return isSafeToExpand(S, CanonicalMode, SE);
+}
+
bool SCEVExpander::isSafeToExpandAt(const SCEV *S,
const Instruction *InsertionPoint) const {
if (!isSafeToExpand(S))
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index 6fba8ccd590c62..69806880ce46ee 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -463,9 +463,12 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch(ptr %dst, i64 %N
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP9:%.*]] = freeze i64 [[N]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP9]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[TMP10]]
+; CHECK-NEXT: [[TMP8:%.*]] = freeze i64 [[TMP0]]
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
-; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[N]]
-; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP8]], i64 [[SMAX]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -529,7 +532,9 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_different_bounds
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_different_bounds(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 42, [[M]]
+; CHECK-NEXT: [[TMP0:%.*]] = freeze i64 [[M]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 42, [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = freeze i64 [[TMP2]]
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[SMAX]])
@@ -598,9 +603,12 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_frozen_value_in_latch(ptr %dst,
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[FR_N:%.*]] = freeze i64 [[N]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
-; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[FR_N]]
-; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 [[TMP0]])
+; CHECK-NEXT: [[TMP0:%.*]] = freeze i64 [[FR_N]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 42, [[TMP1]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze i64 [[TMP2]]
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP10]], i64 [[SMAX]])
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[UMIN]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -786,12 +794,15 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
-; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT: [[TMP11:%.*]] = freeze i64 [[N]]
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP11]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[TMP12]]
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[N]], [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 42, [[TMP1]]
; CHECK-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP2]], i64 0)
-; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[SMAX1]])
+; CHECK-NEXT: [[TMP10:%.*]] = freeze i64 [[SMAX1]]
+; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP10]], i64 [[SMAX2]])
; CHECK-NEXT: [[TMP3:%.*]] = add nuw i64 [[UMIN]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -1004,9 +1015,12 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch1(ptr %dst, i64 %
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch1(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = freeze i64 [[N]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; CHECK-NEXT: [[TMP9:%.*]] = udiv i64 42, [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze i64 [[TMP9]]
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
-; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[N]]
-; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[SMAX]], i64 [[TMP0]])
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP10]], i64 [[SMAX]])
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[UMIN]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
>From 32833a70445de97edc44403e184fff5232eab8d1 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 27 Sep 2024 21:28:08 +0100
Subject: [PATCH 2/7] !fixup
---
llvm/lib/Analysis/ScalarEvolution.cpp | 5 ++-
.../Utils/ScalarEvolutionExpander.cpp | 6 ++-
.../trip-count-expansion-may-introduce-ub.ll | 44 +++++++++++++++++--
3 files changed, 48 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 9b000bdbfa4c4d..97411272c2e775 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -4304,9 +4304,10 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
}
for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
- bool MayBeUB = SCEVExprContains(Ops[i], [](const SCEV *S) {
+ bool MayBeUB = SCEVExprContains(Ops[i], [this](const SCEV *S) {
auto *UDiv = dyn_cast<SCEVUDivExpr>(S);
- return UDiv && !isa<SCEVConstant>(UDiv->getOperand(1));
+ // The UDiv may be UB if the divisor is poison or zero. Unless the divisor is a non-zero constant, we have to assume the UDiv may be UB.
+ return UDiv && (!isa<SCEVConstant>(UDiv->getOperand(1)) || !isKnownNonZero(UDiv->getOperand(1)));
});
if (MayBeUB)
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index ee7b2f74db6fa9..7d9ebeef51f951 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1364,13 +1364,14 @@ Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
Value *SCEVExpander::expandMinMaxExpr(const SCEVNAryExpr *S,
Intrinsic::ID IntrinID, Twine Name,
bool IsSequential) {
- SafeUDivMode = true;
+ bool PrevSafeMode = SafeUDivMode;
+ SafeUDivMode = IsSequential;
Value *LHS = expand(S->getOperand(S->getNumOperands() - 1));
Type *Ty = LHS->getType();
if (IsSequential)
LHS = Builder.CreateFreeze(LHS);
for (int i = S->getNumOperands() - 2; i >= 0; --i) {
- SafeUDivMode = i != 0;
+ SafeUDivMode = (IsSequential && i != 0) || PrevSafeMode;
Value *RHS = expand(S->getOperand(i));
if (IsSequential && i != 0)
RHS = Builder.CreateFreeze(RHS);
@@ -1385,6 +1386,7 @@ Value *SCEVExpander::expandMinMaxExpr(const SCEVNAryExpr *S,
}
LHS = Sel;
}
+ SafeUDivMode = PrevSafeMode;
return LHS;
}
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index 69806880ce46ee..b006f2d3390b29 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -794,9 +794,7 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP11:%.*]] = freeze i64 [[N]]
-; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP11]], i64 1)
-; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[TMP12]]
+; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[N]]
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[N]], [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 42, [[TMP1]]
; CHECK-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP2]], i64 0)
@@ -1082,6 +1080,46 @@ exit:
ret i64 %p
}
+define i64 @multi_exit_exit_count_with_udiv_by_0_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_exit_count_with_udiv_by_0_in_latch(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
+; CHECK: loop.header:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[D:%.*]] = udiv i64 42, 0
+; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT: ret i64 [[P]]
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 1, ptr %gep
+ %c.0 = icmp slt i64 %iv, %N
+ br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %d = udiv i64 42, 0
+ %c.1 = icmp slt i64 %iv, %d
+ br i1 %c.1, label %loop.header, label %exit
+
+exit:
+ %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+ ret i64 %p
+}
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
>From b0d098cd529809bf46d7a5d397591f4bf7f53e3a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 27 Sep 2024 21:48:55 +0100
Subject: [PATCH 3/7] !fixup incluede missing change
---
llvm/lib/Analysis/ScalarEvolution.cpp | 6 ++++--
llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 97411272c2e775..b34748a6830a3b 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -4306,8 +4306,10 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
bool MayBeUB = SCEVExprContains(Ops[i], [this](const SCEV *S) {
auto *UDiv = dyn_cast<SCEVUDivExpr>(S);
- // The UDiv may be UB if the divisor is poison or zero. Unless the divisor is a non-zero constant, we have to assume the UDiv may be UB.
- return UDiv && (!isa<SCEVConstant>(UDiv->getOperand(1)) || !isKnownNonZero(UDiv->getOperand(1)));
+ // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
+ // is a non-zero constant, we have to assume the UDiv may be UB.
+ return UDiv && (!isa<SCEVConstant>(UDiv->getOperand(1)) ||
+ !isKnownNonZero(UDiv->getOperand(1)));
});
if (MayBeUB)
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 7d9ebeef51f951..1b4089a3ec4fac 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -1365,7 +1365,7 @@ Value *SCEVExpander::expandMinMaxExpr(const SCEVNAryExpr *S,
Intrinsic::ID IntrinID, Twine Name,
bool IsSequential) {
bool PrevSafeMode = SafeUDivMode;
- SafeUDivMode = IsSequential;
+ SafeUDivMode |= IsSequential;
Value *LHS = expand(S->getOperand(S->getNumOperands() - 1));
Type *Ty = LHS->getType();
if (IsSequential)
>From 4ace5249a492b53751da722683ce36d3aec56f78 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 28 Sep 2024 21:13:45 +0100
Subject: [PATCH 4/7] !fixup update test
---
.../LoopVectorize/trip-count-expansion-may-introduce-ub.ll | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index b006f2d3390b29..4ac02a1739bc7e 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -794,7 +794,9 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT: [[TMP11:%.*]] = freeze i64 [[N]]
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP11]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[TMP12]]
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[N]], [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 42, [[TMP1]]
; CHECK-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP2]], i64 0)
>From 70f2ef733191caed7ceb0337dd0a7f5749c9066c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 28 Sep 2024 21:29:15 +0100
Subject: [PATCH 5/7] !fixup address latest comments, thanks!
---
.../llvm/Analysis/ScalarEvolutionExpressions.h | 9 ++++++---
.../llvm/Transforms/Utils/ScalarEvolutionExpander.h | 3 ---
llvm/lib/Analysis/ScalarEvolution.cpp | 4 ++--
.../Transforms/Utils/ScalarEvolutionExpander.cpp | 13 +++++--------
4 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index fd884f2a2f55b0..26e400e69699cd 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -510,9 +510,12 @@ class SCEVUMinExpr : public SCEVMinMaxExpr {
/// This node is the base class for sequential/in-order min/max selections.
/// Note that their fundamental difference from SCEVMinMaxExpr's is that they
-/// are early-returning upon reaching saturation point.
-/// I.e. given `0 umin_seq poison`, the result will be `0`,
-/// while the result of `0 umin poison` is `poison`.
+/// are early-returning
+/// * upon reaching saturation point
+/// I.e. given `0 umin_seq poison`, the result will be `0`,
+/// while the result of `0 umin poison` is `poison`.
+/// * if any operand may trigger UB, e.g. if there is an UDiv operand that may
+/// divide by 0.
class SCEVSequentialMinMaxExpr : public SCEVNAryExpr {
friend class ScalarEvolution;
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index cb3b7372f7e66e..81333f8704e023 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -305,9 +305,6 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
/// location and their operands are defined at this location.
bool isSafeToExpandAt(const SCEV *S, const Instruction *InsertionPoint) const;
- static bool isSafeToExpand(const SCEV *S, bool CanonicalMode,
- ScalarEvolution &SE);
-
/// Insert code to directly compute the specified SCEV expression into the
/// program. The code is inserted into the specified block.
Value *expandCodeFor(const SCEV *SH, Type *Ty, BasicBlock::iterator I);
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 995b6bdbbd71b4..d6c29cb7894cfb 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -4318,8 +4318,8 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
// * %y being poison implies %x is also poison.
// * %x cannot be the saturating value (e.g. zero for umin).
if (::impliesPoison(Ops[i], Ops[i - 1]) ||
- (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, Ops[i - 1],
- SaturationPoint))) {
+ isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, Ops[i - 1],
+ SaturationPoint)) {
SmallVector<const SCEV *> SeqOps = {Ops[i - 1], Ops[i]};
Ops[i - 1] = getMinMaxExpr(
SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Kind),
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 1b4089a3ec4fac..2132845de2232f 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -664,8 +664,10 @@ Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
SCEV::FlagAnyWrap, /*IsSafeToHoist*/ true);
}
- Value *RHS = expand(S->getRHS());
- if (SafeUDivMode && !SE.isKnownNonZero(S->getRHS())) {
+ const SCEV *RHSExpr = S->getRHS();
+ Value *RHS = expand(RHSExpr);
+ if (SafeUDivMode &&
+ (!isa<SCEVConstant>(RHSExpr) || SE.isKnownNonZero(RHSExpr))) {
if (!isa<SCEVConstant>(S->getRHS()))
RHS = Builder.CreateFreeze(RHS);
RHS = Builder.CreateIntrinsic(RHS->getType(), Intrinsic::umax,
@@ -2325,17 +2327,12 @@ struct SCEVFindUnsafe {
};
} // namespace
-bool SCEVExpander::isSafeToExpand(const SCEV *S, bool CanonicalMode,
- ScalarEvolution &SE) {
+bool SCEVExpander::isSafeToExpand(const SCEV *S) const {
SCEVFindUnsafe Search(SE, CanonicalMode);
visitAll(S, Search);
return !Search.IsUnsafe;
}
-bool SCEVExpander::isSafeToExpand(const SCEV *S) const {
- return isSafeToExpand(S, CanonicalMode, SE);
-}
-
bool SCEVExpander::isSafeToExpandAt(const SCEV *S,
const Instruction *InsertionPoint) const {
if (!isSafeToExpand(S))
>From 5b1a7899815ca27fc449905c8fc67111d1c7708c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 2 Oct 2024 11:01:32 +0100
Subject: [PATCH 6/7] !fixup remove unneeded function, add SCEV test.
---
.../Utils/ScalarEvolutionExpander.h | 2 -
.../umin-seq-operand-may-trigger-ub.ll | 107 ++++++++++++++++++
2 files changed, 107 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index b1927bcc381927..1221af983572ac 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -427,8 +427,6 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
static const SCEV *rewriteExpressionToRemoveUB(const SCEV *BTC, Loop *L,
ScalarEvolution &SE);
- void setSafeUDivMode() { SafeUDivMode = true; }
-
private:
LLVMContext &getContext() const { return SE.getContext(); }
diff --git a/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll b/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll
new file mode 100644
index 00000000000000..f2f718d1577c82
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/umin-seq-operand-may-trigger-ub.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<scalar-evolution>' -scalar-evolution-classify-expressions=0 -disable-output %s 2>&1 | FileCheck %s
+
+; The UDiv in the latch may never be executed. The trip count expressions must
+; account for the fact that evaluating the UDiv unconditionally may trigger UB.
+define i64 @multi_exit_exit_count_with_udiv_by_value_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: 'multi_exit_exit_count_with_udiv_by_value_in_latch'
+; CHECK-NEXT: Determining loop execution counts for: @multi_exit_exit_count_with_udiv_by_value_in_latch
+; CHECK-NEXT: Loop %loop.header: <multiple exits> backedge-taken count is ((0 smax %N) umin_seq (42 /u %N))
+; CHECK-NEXT: exit count for loop.header: (0 smax %N)
+; CHECK-NEXT: exit count for loop.latch: (42 /u %N)
+; CHECK-NEXT: Loop %loop.header: constant max backedge-taken count is i64 42
+; CHECK-NEXT: Loop %loop.header: symbolic max backedge-taken count is ((0 smax %N) umin_seq (42 /u %N))
+; CHECK-NEXT: symbolic max exit count for loop.header: (0 smax %N)
+; CHECK-NEXT: symbolic max exit count for loop.latch: (42 /u %N)
+; CHECK-NEXT: Loop %loop.header: Trip multiple is 1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 1, ptr %gep
+ %c.0 = icmp slt i64 %iv, %N
+ br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %d = udiv i64 42, %N
+ %c.1 = icmp slt i64 %iv, %d
+ br i1 %c.1, label %loop.header, label %exit
+
+exit:
+ %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+ ret i64 %p
+}
+
+; The UDiv in the latch may never be executed. The trip count expressions must
+; account for the fact that evaluating the UDiv unconditionally may trigger UB.
+define i64 @multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds(ptr %dst, i64 %N, i64 %M) {
+; CHECK-LABEL: 'multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds'
+; CHECK-NEXT: Determining loop execution counts for: @multi_exit_exit_count_with_udiv_by_value_in_latch_different_bounds
+; CHECK-NEXT: Loop %loop.header: <multiple exits> backedge-taken count is ((0 smax %N) umin_seq (42 /u %M))
+; CHECK-NEXT: exit count for loop.header: (0 smax %N)
+; CHECK-NEXT: exit count for loop.latch: (42 /u %M)
+; CHECK-NEXT: Loop %loop.header: constant max backedge-taken count is i64 42
+; CHECK-NEXT: Loop %loop.header: symbolic max backedge-taken count is ((0 smax %N) umin_seq (42 /u %M))
+; CHECK-NEXT: symbolic max exit count for loop.header: (0 smax %N)
+; CHECK-NEXT: symbolic max exit count for loop.latch: (42 /u %M)
+; CHECK-NEXT: Loop %loop.header: Trip multiple is 1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 1, ptr %gep
+ %c.0 = icmp slt i64 %iv, %N
+ br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %d = udiv i64 42, %M
+ %c.1 = icmp slt i64 %iv, %d
+ br i1 %c.1, label %loop.header, label %exit
+
+exit:
+ %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+ ret i64 %p
+}
+
+; The UDiv in the latch cannot trigger UB, evaluating it unconditionally in the
+; trip count expression is fine.
+define i64 @multi_exit_exit_count_with_udiv_by_constant_in_latch(ptr %dst, i64 %N) {
+; CHECK-LABEL: 'multi_exit_exit_count_with_udiv_by_constant_in_latch'
+; CHECK-NEXT: Determining loop execution counts for: @multi_exit_exit_count_with_udiv_by_constant_in_latch
+; CHECK-NEXT: Loop %loop.header: <multiple exits> backedge-taken count is ((%N /u 42) umin (0 smax %N))
+; CHECK-NEXT: exit count for loop.header: (0 smax %N)
+; CHECK-NEXT: exit count for loop.latch: (%N /u 42)
+; CHECK-NEXT: Loop %loop.header: constant max backedge-taken count is i64 439208192231179800
+; CHECK-NEXT: Loop %loop.header: symbolic max backedge-taken count is ((%N /u 42) umin (0 smax %N))
+; CHECK-NEXT: symbolic max exit count for loop.header: (0 smax %N)
+; CHECK-NEXT: symbolic max exit count for loop.latch: (%N /u 42)
+; CHECK-NEXT: Loop %loop.header: Trip multiple is 1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 1, ptr %gep
+ %c.0 = icmp slt i64 %iv, %N
+ br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %d = udiv i64 %N, 42
+ %c.1 = icmp slt i64 %iv, %d
+ br i1 %c.1, label %loop.header, label %exit
+
+exit:
+ %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+ ret i64 %p
+}
>From c12b25c2beca5257a01767b3b98a7e71c2f21d3c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 15 Oct 2024 11:51:01 +0100
Subject: [PATCH 7/7] !fixup address latest comments, thanks!
---
.../Utils/ScalarEvolutionExpander.h | 3 -
.../Utils/ScalarEvolutionExpander.cpp | 5 +-
.../trip-count-expansion-may-introduce-ub.ll | 193 ++++++++++++++++--
3 files changed, 173 insertions(+), 28 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 1363084ac4bc32..4591940c404de2 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -424,9 +424,6 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
BasicBlock::iterator findInsertPointAfter(Instruction *I,
Instruction *MustDominate) const;
- static const SCEV *rewriteExpressionToRemoveUB(const SCEV *BTC, Loop *L,
- ScalarEvolution &SE);
-
private:
LLVMContext &getContext() const { return SE.getContext(); }
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 423e1e4b33902b..b392d5a47a3788 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -678,9 +678,8 @@ Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
const SCEV *RHSExpr = S->getRHS();
Value *RHS = expand(RHSExpr);
- if (SafeUDivMode &&
- (!isa<SCEVConstant>(RHSExpr) || SE.isKnownNonZero(RHSExpr))) {
- if (!isa<SCEVConstant>(S->getRHS()))
+ if (SafeUDivMode && !SE.isKnownNonZero(RHSExpr)) {
+ if (!ScalarEvolution::isGuaranteedNotToBePoison(RHSExpr))
RHS = Builder.CreateFreeze(RHS);
RHS = Builder.CreateIntrinsic(RHS->getType(), Intrinsic::umax,
{RHS, ConstantInt::get(RHS->getType(), 1)});
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index 4ac02a1739bc7e..72a52d5b96e077 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -463,8 +463,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch(ptr %dst, i64 %N
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP9:%.*]] = freeze i64 [[N]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP9]], i64 1)
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[TMP10]]
; CHECK-NEXT: [[TMP8:%.*]] = freeze i64 [[TMP0]]
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
@@ -528,6 +527,155 @@ exit:
ret i64 %p
}
+declare void @foo()
+
+define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_call_before_loop(ptr %dst, i64 %N) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_call_before_loop(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call void @foo()
+; CHECK-NEXT: [[TMP0:%.*]] = freeze i64 [[N]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 42, [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze i64 [[TMP2]]
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[SMAX]])
+; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[UMIN]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP4]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP6]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 0
+; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
+; CHECK: loop.header:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT:%.*]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[D:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: exit:
+; CHECK-NEXT: [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
+; CHECK-NEXT: ret i64 [[P]]
+;
+entry:
+ call void @foo()
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 1, ptr %gep
+ %c.0 = icmp slt i64 %iv, %N
+ br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %d = udiv i64 42, %N
+ %c.1 = icmp slt i64 %iv, %d
+ br i1 %c.1, label %loop.header, label %exit
+
+exit:
+ %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch]
+ ret i64 %p
+}
+
+
+define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_loop_may_not_execute(ptr %dst, i64 %N, i1 %c) {
+; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_loop_may_not_execute(
+; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK: loop.header.preheader:
+; CHECK-NEXT: [[TMP0:%.*]] = freeze i64 [[N]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 42, [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze i64 [[TMP2]]
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[SMAX]])
+; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[UMIN]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP4]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 4, i64 [[N_MOD_VF]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP6]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 0
+; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_HEADER_PREHEADER]] ]
+; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
+; CHECK: loop.header:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 1, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C_0:%.*]] = icmp slt i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[C_0]], label [[LOOP_LATCH]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[D:%.*]] = udiv i64 42, [[N]]
+; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: exit.loopexit:
+; CHECK-NEXT: [[P_PH:%.*]] = phi i64 [ 0, [[LOOP_LATCH]] ], [ 1, [[LOOP_HEADER]] ]
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: [[P:%.*]] = phi i64 [ 2, [[ENTRY:%.*]] ], [ [[P_PH]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT: ret i64 [[P]]
+;
+entry:
+ br i1 %c, label %loop.header, label %exit
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 1, ptr %gep
+ %c.0 = icmp slt i64 %iv, %N
+ br i1 %c.0, label %loop.latch, label %exit
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %d = udiv i64 42, %N
+ %c.1 = icmp slt i64 %iv, %d
+ br i1 %c.1, label %loop.header, label %exit
+
+exit:
+ %p = phi i64 [ 1, %loop.header ], [ 0, %loop.latch], [ 2, %entry ]
+ ret i64 %p
+}
+
+
define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_different_bounds(ptr %dst, i64 %N, i64 %M) {
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_different_bounds(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
@@ -555,7 +703,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_different_bounds
; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP9]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -571,7 +719,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch_different_bounds
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[D:%.*]] = udiv i64 42, [[M]]
; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
-; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[P]]
@@ -603,8 +751,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_frozen_value_in_latch(ptr %dst,
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[FR_N:%.*]] = freeze i64 [[N]]
-; CHECK-NEXT: [[TMP0:%.*]] = freeze i64 [[FR_N]]
-; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[FR_N]], i64 1)
; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 42, [[TMP1]]
; CHECK-NEXT: [[TMP10:%.*]] = freeze i64 [[TMP2]]
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
@@ -626,7 +773,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_frozen_value_in_latch(ptr %dst,
; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -642,7 +789,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_frozen_value_in_latch(ptr %dst,
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[D:%.*]] = udiv i64 42, [[FR_N]]
; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
-; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[P]]
@@ -694,7 +841,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_constant_in_latch(ptr %dst, i64
; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -710,7 +857,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_constant_in_latch(ptr %dst, i64
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[D:%.*]] = udiv i64 [[N]], 42
; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV1]], [[D]]
-; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER1]], label [[EXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER1]], label [[EXIT]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER1]] ], [ 0, [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[P]]
@@ -756,7 +903,7 @@ define void @single_exit_tc_with_udiv(ptr %dst, i64 %N) {
; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -770,7 +917,7 @@ define void @single_exit_tc_with_udiv(ptr %dst, i64 %N) {
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[D:%.*]] = udiv i64 42, [[N]]
; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
-; CHECK-NEXT: br i1 [[C_1]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
@@ -794,8 +941,7 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP11:%.*]] = freeze i64 [[N]]
-; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP11]], i64 1)
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[TMP12]]
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[N]], [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 42, [[TMP1]]
@@ -820,7 +966,7 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -836,7 +982,7 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[D:%.*]] = urem i64 42, [[N]]
; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
-; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[P]]
@@ -887,7 +1033,7 @@ define i64 @multi_exit_4_exit_count_with_urem_by_constant_in_latch(ptr %dst, i64
; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -903,7 +1049,7 @@ define i64 @multi_exit_4_exit_count_with_urem_by_constant_in_latch(ptr %dst, i64
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[D:%.*]] = urem i64 [[N]], 42
; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV1]], [[D]]
-; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER1]], label [[EXIT]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER1]], label [[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER1]] ], [ 0, [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[P]]
@@ -1015,8 +1161,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch1(ptr %dst, i64 %
; CHECK-LABEL: define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch1(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = freeze i64 [[N]]
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP9:%.*]] = udiv i64 42, [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = freeze i64 [[TMP9]]
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
@@ -1038,7 +1183,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch1(ptr %dst, i64 %
; CHECK-NEXT: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -1055,7 +1200,7 @@ define i64 @multi_exit_4_exit_count_with_udiv_by_value_in_latch1(ptr %dst, i64 %
; CHECK-NEXT: [[D:%.*]] = udiv i64 42, [[N]]
; CHECK-NEXT: [[X:%.*]] = sub i64 100, [[D]]
; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[IV]], [[D]]
-; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: [[P:%.*]] = phi i64 [ 1, [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[P]]
@@ -1150,4 +1295,8 @@ exit:
; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META2]], [[META1]]}
; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META1]], [[META2]]}
; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META2]], [[META1]]}
+; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META1]], [[META2]]}
+; CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META2]], [[META1]]}
+; CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META1]], [[META2]]}
+; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[META2]], [[META1]]}
;.
More information about the llvm-commits
mailing list