[llvm] [Analysis] Cache generation of SCEV expander overflow checks (PR #213013)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 03:24:25 PDT 2026
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/213013
>From d4a65ce779a8698d3b4b44767ac208356ceb8fdf Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Thu, 30 Jul 2026 12:34:30 +0000
Subject: [PATCH 1/2] [Analysis] Cache generation of SCEV expander overflow
checks
When creating SCEV checks as part of loop vectorisation we
often generate overflow checks, which leads to lots of
duplicated calls to the umul_with_overflow intrinsic. These
calls should be cleaned up during codegen. However, it is
unfortunate that the current LLVM method of calculating the
cost of IR in a block involves looping over each instruction
and adding the costs individually with no thought to the
trivial CSE or DCE optimisations that would take place.
In the absence of a more sophisticated method of cost
analysis, for now I've chosen to explicitly CSE these
overflow checks during SCEV expansion.
---
.../Utils/ScalarEvolutionExpander.h | 6 +
.../Utils/ScalarEvolutionExpander.cpp | 27 +++-
.../AArch64/conditional-branches-cost.ll | 28 +---
.../LoopVectorize/AArch64/induction-costs.ll | 30 ++--
...terleave-group-requires-scalar-epilogue.ll | 7 +-
...-narrow-interleave-to-widen-memory-cost.ll | 43 ++---
...sform-narrow-interleave-to-widen-memory.ll | 18 +--
.../LoopVectorize/X86/cost-model.ll | 3 +-
.../LoopVectorize/X86/induction-step.ll | 13 +-
.../LoopVectorize/X86/interleave-cost.ll | 58 ++-----
.../Transforms/LoopVectorize/induction.ll | 152 ++++++++----------
...-first-order-recurrence-by-versioned-iv.ll | 31 ++--
.../runtime-checks-difference.ll | 3 +-
...ive-path-inner-loop-with-runtime-checks.ll | 7 +-
14 files changed, 179 insertions(+), 247 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 5702002123093..9f8fd66c0c6f4 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -78,6 +78,11 @@ class SCEVExpander : public SCEVUseVisitor<SCEVExpander, Value *> {
DenseMap<std::pair<SCEVUse, Instruction *>, TrackingVH<Value>>
InsertedExpressions;
+ // InsertedOverflowChecks caches Values for reuse, so must track RAUW.
+ DenseMap<std::tuple<Value *, Value *, Instruction *>,
+ std::pair<TrackingVH<Value>, TrackingVH<Value>>>
+ InsertedOverflowChecks;
+
// InsertedValues only flags inserted instructions so needs no RAUW.
DenseSet<AssertingVH<Value>> InsertedValues;
DenseSet<AssertingVH<Value>> InsertedPostIncValues;
@@ -211,6 +216,7 @@ class SCEVExpander : public SCEVUseVisitor<SCEVExpander, Value *> {
/// places within the same BasicBlock can do so.
void clear() {
InsertedExpressions.clear();
+ InsertedOverflowChecks.clear();
InsertedValues.clear();
InsertedPostIncValues.clear();
ReusedValues.clear();
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index aa26b160dded7..db6ca49d74b53 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2321,14 +2321,25 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
// negative. If Step is known to be positive or negative, only create
// either 1. or 2.
auto ComputeEndCheck = [&]() -> Value * {
- // Get the backedge taken count and truncate or extended to the AR type.
- Value *TruncTripCount = Builder.CreateZExtOrTrunc(TripCountVal, Ty);
-
- Value *Mul = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, Ty,
- {AbsStep, TruncTripCount},
- /*FMFSource=*/nullptr, "mul");
- Value *MulV = Builder.CreateExtractValue(Mul, 0, "mul.result");
- Value *OfMul = Builder.CreateExtractValue(Mul, 1, "mul.overflow");
+ // Check to see if we already expanded this here.
+ Value *MulV, *OfMul;
+ auto Key = std::make_tuple(TripCountVal, AbsStep, &*Loc);
+ auto I = InsertedOverflowChecks.find(Key);
+ if (I != InsertedOverflowChecks.end()) {
+ MulV = I->second.first;
+ OfMul = I->second.second;
+ } else {
+ // Get the backedge taken count and truncate or extended to the AR type.
+ Value *TruncTripCount = Builder.CreateZExtOrTrunc(TripCountVal, Ty);
+ Value *Mul = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, Ty,
+ {AbsStep, TruncTripCount},
+ /*FMFSource=*/nullptr, "mul");
+ MulV = Builder.CreateExtractValue(Mul, 0, "mul.result");
+ OfMul = Builder.CreateExtractValue(Mul, 1, "mul.overflow");
+
+ // The type Ty is already encoded in AbsStep.
+ InsertedOverflowChecks[Key] = std::pair<Value *, Value *>(MulV, OfMul);
+ }
Value *Add = nullptr, *Sub = nullptr;
bool NeedPosCheck = !SE.isKnownNegative(Step);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 48c6b259c8b78..a0851d8efd5bb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -673,19 +673,13 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
; DEFAULT-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[DST]]
; DEFAULT-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
; DEFAULT-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
-; DEFAULT-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; DEFAULT-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
-; DEFAULT-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
-; DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT2]]
+; DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; DEFAULT-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; DEFAULT-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW3]]
+; DEFAULT-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; DEFAULT-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; DEFAULT-NEXT: [[MUL5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; DEFAULT-NEXT: [[MUL_RESULT6:%.*]] = extractvalue { i64, i1 } [[MUL5]], 0
-; DEFAULT-NEXT: [[MUL_OVERFLOW7:%.*]] = extractvalue { i64, i1 } [[MUL5]], 1
-; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT6]]
+; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT]]
; DEFAULT-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
-; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW7]]
+; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
; DEFAULT-NEXT: [[TMP13:%.*]] = or i1 [[TMP4]], [[TMP8]]
; DEFAULT-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[TMP12]]
; DEFAULT-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
@@ -831,19 +825,13 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
; PRED-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[DST]]
; PRED-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
; PRED-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
-; PRED-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; PRED-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
-; PRED-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
-; PRED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT2]]
+; PRED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; PRED-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; PRED-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW3]]
+; PRED-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; PRED-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; PRED-NEXT: [[MUL5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; PRED-NEXT: [[MUL_RESULT6:%.*]] = extractvalue { i64, i1 } [[MUL5]], 0
-; PRED-NEXT: [[MUL_OVERFLOW7:%.*]] = extractvalue { i64, i1 } [[MUL5]], 1
-; PRED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT6]]
+; PRED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT]]
; PRED-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
-; PRED-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW7]]
+; PRED-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
; PRED-NEXT: [[TMP13:%.*]] = or i1 [[TMP4]], [[TMP8]]
; PRED-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[TMP12]]
; PRED-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index 29f54d4103db4..7c2f85fd360bc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -400,11 +400,11 @@ define void @zext_iv_increment(ptr %dst, i64 %N) {
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV_WIDE:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV_NEXT_EXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_WIDE:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT_EXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[PATCH_INDEX:%.*]] = getelementptr { i32, i32, i32 }, ptr [[DST]], i64 [[IV_WIDE]], i32 2
; CHECK-NEXT: store i32 0, ptr [[PATCH_INDEX]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
@@ -537,12 +537,9 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[SCEVGEP]]
; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SRC]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW3]]
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP2]] to i32
; CHECK-NEXT: [[MUL4:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 2, i32 [[TMP9]])
; CHECK-NEXT: [[MUL_RESULT5:%.*]] = extractvalue { i32, i1 } [[MUL4]], 0
@@ -551,13 +548,10 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP10]], [[MUL_OVERFLOW6]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp ugt i64 [[TMP2]], 4294967295
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[MUL7:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT8:%.*]] = extractvalue { i64, i1 } [[MUL7]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW9:%.*]] = extractvalue { i64, i1 } [[MUL7]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = sub i64 0, [[MUL_RESULT8]]
+; CHECK-NEXT: [[TMP14:%.*]] = sub i64 0, [[MUL_RESULT]]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP14]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp ugt ptr [[TMP15]], [[BASE]]
-; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[MUL_OVERFLOW9]]
+; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP5]], [[TMP8]]
; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP18]], [[TMP13]]
; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP17]]
@@ -708,14 +702,14 @@ define void at sext_sub_nsw_for_address(ptr %base, i64 %n, ptr %src) #0 {
; CHECK-NEXT: [[CMP_N29:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC23]]
; CHECK-NEXT: br i1 [[CMP_N29]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL30:%.*]] = phi i64 [ [[TMP87]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL31:%.*]] = phi i64 [ [[TMP88]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP27]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL32:%.*]] = phi i32 [ [[TMP90]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL24:%.*]] = phi i64 [ [[TMP87]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP26]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL25:%.*]] = phi i64 [ [[TMP88]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP27]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL26:%.*]] = phi i32 [ [[TMP90]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL30]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[COUNT:%.*]] = phi i64 [ [[BC_RESUME_VAL31]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[COUNT_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV_I32:%.*]] = phi i32 [ [[BC_RESUME_VAL32]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_I32_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL24]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[COUNT:%.*]] = phi i64 [ [[BC_RESUME_VAL25]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[COUNT_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_I32:%.*]] = phi i32 [ [[BC_RESUME_VAL26]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_I32_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr i8, ptr [[GEP_1]], i64 -8
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_2]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll
index 0de23b2e5e483..0c9063628824c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll
@@ -20,12 +20,9 @@ define void @interleave_group_exit_in_header(i64 %n, ptr %dst) {
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[DST]]
; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[TMP4]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW3]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP3]], [[TMP6]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
index e9205e9e38b55..40d0f09431d02 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
@@ -157,12 +157,9 @@ define void @test_interleave_store_one_constant(ptr noalias %src, ptr noalias %d
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[DST]]
; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP4]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
@@ -287,25 +284,16 @@ define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[SCEVGEP]]
; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[B]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW3]]
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[C]], i64 8
-; CHECK-NEXT: [[MUL5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; CHECK-NEXT: [[MUL_RESULT6:%.*]] = extractvalue { i64, i1 } [[MUL5]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW7:%.*]] = extractvalue { i64, i1 } [[MUL5]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT6]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
-; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW7]]
-; CHECK-NEXT: [[MUL8:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; CHECK-NEXT: [[MUL_RESULT9:%.*]] = extractvalue { i64, i1 } [[MUL8]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW10:%.*]] = extractvalue { i64, i1 } [[MUL8]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[C]], i64 [[MUL_RESULT9]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[C]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[C]]
-; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW10]]
+; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP4]], [[TMP8]]
; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP17]], [[TMP12]]
; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP18]], [[TMP16]]
@@ -528,12 +516,9 @@ define void @test_interleave_group_epilogue_with_preheader_phi(ptr %src, ptr %ds
; CHECK-NEXT: [[TMP9:%.*]] = icmp ult ptr [[TMP8]], [[DST]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP7]])
-; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp ult ptr [[TMP11]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP12]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP12]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[IDENT_CHECK]], [[TMP10]]
; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[TMP13]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
@@ -587,12 +572,12 @@ define void @test_interleave_group_epilogue_with_preheader_phi(ptr %src, ptr %ds
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[DST_PRE]], %[[VECTOR_SCEVCHECK]] ], [ [[DST_PRE]], %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL14:%.*]] = phi ptr [ [[TMP27]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP22]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[SRC]], %[[VECTOR_SCEVCHECK]] ], [ [[SRC]], %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL10:%.*]] = phi ptr [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[DST_PRE]], %[[VECTOR_SCEVCHECK]] ], [ [[DST_PRE]], %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL11:%.*]] = phi ptr [ [[TMP27]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP22]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[SRC]], %[[VECTOR_SCEVCHECK]] ], [ [[SRC]], %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[DST_PHI:%.*]] = phi ptr [ [[DST_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL14]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[DST_PHI:%.*]] = phi ptr [ [[DST_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL10]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[SRC_PHI:%.*]] = phi ptr [ [[SRC_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL11]], %[[VEC_EPILOG_SCALAR_PH]] ]
; CHECK-NEXT: store double 1.000000e+00, ptr [[DST_PHI]], align 8
; CHECK-NEXT: [[DST_IM:%.*]] = getelementptr i8, ptr [[DST_PHI]], i64 8
; CHECK-NEXT: store double 1.000000e+00, ptr [[DST_IM]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
index 1f7571ba3d286..f9914ff508423 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
@@ -104,7 +104,7 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 38)
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 26)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
@@ -115,12 +115,9 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[DST]]
; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -164,7 +161,7 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; EPILOGUE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; EPILOGUE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; EPILOGUE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
-; EPILOGUE-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 40)
+; EPILOGUE-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 28)
; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; EPILOGUE: [[VECTOR_SCEVCHECK]]:
@@ -175,12 +172,9 @@ define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) {
; EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[DST]]
; EPILOGUE-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
; EPILOGUE-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
-; EPILOGUE-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
-; EPILOGUE-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
-; EPILOGUE-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
-; EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT3]]
+; EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; EPILOGUE-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; EPILOGUE-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW4]]
+; EPILOGUE-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; EPILOGUE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
; EPILOGUE-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; EPILOGUE: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index b4c861e67dd38..630dcaced1e46 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -466,8 +466,7 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 1
; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMIN]], 4294967295
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = trunc i64 [[UMIN]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = icmp slt i32 [[TMP10]], 0
+; CHECK-NEXT: [[TMP11:%.*]] = icmp slt i32 [[TMP5]], 0
; CHECK-NEXT: [[TMP12:%.*]] = icmp ugt i64 [[UMIN]], 4294967295
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP9]], [[TMP13]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
index 1515cb23c4b0c..c6f5644f8b625 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
@@ -148,7 +148,7 @@ define void @canonical_iv_matches_user_iv(i64 %n, ptr %A, ptr %B) {
; CHECK-LABEL: define void @canonical_iv_matches_user_iv(
; CHECK-SAME: i64 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 6
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
@@ -160,12 +160,9 @@ define void @canonical_iv_matches_user_iv(i64 %n, ptr %A, ptr %B) {
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[SCEVGEP]]
; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B]], i64 24
-; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP0]])
-; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[TMP4]], [[SCEVGEP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP3]], [[TMP6]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
@@ -193,8 +190,8 @@ define void @canonical_iv_matches_user_iv(i64 %n, ptr %A, ptr %B) {
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP15]], 1
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr [16 x i8], ptr [[B]], i64 [[TMP10]]
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [16 x i8], ptr [[B]], i64 [[TMP14]]
-; CHECK-NEXT: store <2 x double> [[BROADCAST_SPLAT]], ptr [[TMP13]], align 8
-; CHECK-NEXT: store <2 x double> [[BROADCAST_SPLAT]], ptr [[TMP16]], align 8
+; CHECK-NEXT: store <2 x double> [[BROADCAST_SPLAT]], ptr [[TMP13]], align 8, !alias.scope [[META9]]
+; CHECK-NEXT: store <2 x double> [[BROADCAST_SPLAT]], ptr [[TMP16]], align 8, !alias.scope [[META9]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
index 8b090ba020fde..51fc28336db7a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
@@ -179,7 +179,7 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[N]], 3
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 56
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP1]], 28
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[N]], 3
@@ -190,54 +190,33 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[A]]
; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 4
-; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW3]]
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 8
-; CHECK-NEXT: [[MUL5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT6:%.*]] = extractvalue { i64, i1 } [[MUL5]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW7:%.*]] = extractvalue { i64, i1 } [[MUL5]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT6]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp ult ptr [[TMP9]], [[SCEVGEP4]]
-; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP10]], [[MUL_OVERFLOW7]]
+; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP10]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[A]], i64 12
-; CHECK-NEXT: [[MUL9:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT10:%.*]] = extractvalue { i64, i1 } [[MUL9]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW11:%.*]] = extractvalue { i64, i1 } [[MUL9]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP8]], i64 [[MUL_RESULT10]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP8]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp ult ptr [[TMP12]], [[SCEVGEP8]]
-; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[MUL_OVERFLOW11]]
+; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP13]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP12:%.*]] = getelementptr i8, ptr [[A]], i64 16
-; CHECK-NEXT: [[MUL13:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT14:%.*]] = extractvalue { i64, i1 } [[MUL13]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW15:%.*]] = extractvalue { i64, i1 } [[MUL13]], 1
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[SCEVGEP12]], i64 [[MUL_RESULT14]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[SCEVGEP12]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp ult ptr [[TMP15]], [[SCEVGEP12]]
-; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[MUL_OVERFLOW15]]
+; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP16:%.*]] = getelementptr i8, ptr [[A]], i64 20
-; CHECK-NEXT: [[MUL17:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT18:%.*]] = extractvalue { i64, i1 } [[MUL17]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW19:%.*]] = extractvalue { i64, i1 } [[MUL17]], 1
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SCEVGEP16]], i64 [[MUL_RESULT18]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SCEVGEP16]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP19:%.*]] = icmp ult ptr [[TMP18]], [[SCEVGEP16]]
-; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[MUL_OVERFLOW19]]
+; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[A]], i64 24
-; CHECK-NEXT: [[MUL21:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT22:%.*]] = extractvalue { i64, i1 } [[MUL21]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW23:%.*]] = extractvalue { i64, i1 } [[MUL21]], 1
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SCEVGEP20]], i64 [[MUL_RESULT22]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SCEVGEP20]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP22:%.*]] = icmp ult ptr [[TMP21]], [[SCEVGEP20]]
-; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP22]], [[MUL_OVERFLOW23]]
+; CHECK-NEXT: [[TMP23:%.*]] = or i1 [[TMP22]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP24:%.*]] = getelementptr i8, ptr [[A]], i64 28
-; CHECK-NEXT: [[MUL25:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT26:%.*]] = extractvalue { i64, i1 } [[MUL25]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW27:%.*]] = extractvalue { i64, i1 } [[MUL25]], 1
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SCEVGEP24]], i64 [[MUL_RESULT26]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SCEVGEP24]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP25:%.*]] = icmp ult ptr [[TMP24]], [[SCEVGEP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = or i1 [[TMP25]], [[MUL_OVERFLOW27]]
+; CHECK-NEXT: [[TMP26:%.*]] = or i1 [[TMP25]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[SCEVGEP28:%.*]] = getelementptr i8, ptr [[B]], i64 4
; CHECK-NEXT: [[MUL29:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]])
; CHECK-NEXT: [[MUL_RESULT30:%.*]] = extractvalue { i64, i1 } [[MUL29]], 0
@@ -245,12 +224,9 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[SCEVGEP28]], i64 [[MUL_RESULT30]]
; CHECK-NEXT: [[TMP28:%.*]] = icmp ult ptr [[TMP27]], [[SCEVGEP28]]
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP28]], [[MUL_OVERFLOW31]]
-; CHECK-NEXT: [[MUL32:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 [[TMP2]])
-; CHECK-NEXT: [[MUL_RESULT33:%.*]] = extractvalue { i64, i1 } [[MUL32]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW34:%.*]] = extractvalue { i64, i1 } [[MUL32]], 1
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT33]]
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp ult ptr [[TMP30]], [[B]]
-; CHECK-NEXT: [[TMP32:%.*]] = or i1 [[TMP31]], [[MUL_OVERFLOW34]]
+; CHECK-NEXT: [[TMP32:%.*]] = or i1 [[TMP31]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP33:%.*]] = or i1 [[TMP5]], [[TMP8]]
; CHECK-NEXT: [[TMP34:%.*]] = or i1 [[TMP33]], [[TMP11]]
; CHECK-NEXT: [[TMP35:%.*]] = or i1 [[TMP34]], [[TMP14]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index a8cf0acaef314..1d1ff470adf5b 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -3331,15 +3331,14 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
-; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[LEN]] to i8
-; CHECK-NEXT: [[TMP7:%.*]] = add i8 [[T]], [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i8 [[TMP7]], [[T]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255
-; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP5]], [[TMP10]]
-; CHECK-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP3]], [[TMP9]]
+; CHECK-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-NEXT: br i1 [[TMP10]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
@@ -3399,15 +3398,14 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; IND-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; IND-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; IND-NEXT: [[TMP10:%.*]] = icmp ugt i32 [[LEN]], 255
-; IND-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP10]]
-; IND-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8
-; IND-NEXT: [[TMP13:%.*]] = add i8 [[T]], [[TMP4]]
-; IND-NEXT: [[TMP6:%.*]] = icmp ult i8 [[TMP13]], [[T]]
-; IND-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255
-; IND-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; IND-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
-; IND-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; IND-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
+; IND-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; IND-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; IND-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; IND-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; IND-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; IND-NEXT: [[TMP10:%.*]] = or i1 [[TMP5]], [[TMP9]]
+; IND-NEXT: br i1 [[TMP10]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; IND: vector.ph:
; IND-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2
; IND-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
@@ -3467,14 +3465,13 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; UNROLL-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; UNROLL-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; UNROLL-NEXT: [[TMP10:%.*]] = icmp ugt i32 [[LEN]], 255
-; UNROLL-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP10]]
-; UNROLL-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8
-; UNROLL-NEXT: [[TMP14:%.*]] = add i8 [[T]], [[TMP4]]
-; UNROLL-NEXT: [[TMP6:%.*]] = icmp ult i8 [[TMP14]], [[T]]
-; UNROLL-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255
-; UNROLL-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; UNROLL-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
+; UNROLL-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
+; UNROLL-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; UNROLL-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; UNROLL-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; UNROLL-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; UNROLL-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; UNROLL-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
; UNROLL-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; UNROLL: vector.ph:
; UNROLL-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4
@@ -3538,15 +3535,14 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
-; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
-; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = trunc i32 [[LEN]] to i8
-; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i8 [[T]], [[TMP6]]
-; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp ult i8 [[TMP7]], [[T]]
; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255
-; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
-; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or i1 [[TMP5]], [[TMP10]]
-; UNROLL-NO-IC-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or i1 [[TMP3]], [[TMP9]]
+; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; UNROLL-NO-IC-NEXT: br i1 [[TMP10]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; UNROLL-NO-IC: vector.ph:
; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4
; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
@@ -3609,14 +3605,13 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; INTERLEAVE-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = icmp ugt i32 [[LEN]], 255
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP10]]
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8
-; INTERLEAVE-NEXT: [[TMP14:%.*]] = add i8 [[T]], [[TMP4]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp ult i8 [[TMP14]], [[T]]
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
; INTERLEAVE-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; INTERLEAVE: vector.ph:
; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 8
@@ -3712,15 +3707,14 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
-; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[LEN]] to i8
-; CHECK-NEXT: [[TMP7:%.*]] = add i8 [[T]], [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i8 [[TMP7]], [[T]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255
-; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP5]], [[TMP10]]
-; CHECK-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP3]], [[TMP9]]
+; CHECK-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP11]], [[TMP13]]
+; CHECK-NEXT: br i1 [[TMP10]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
@@ -3783,14 +3777,13 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; IND-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; IND-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; IND-NEXT: [[TMP10:%.*]] = icmp ugt i32 [[LEN]], 255
-; IND-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP10]]
-; IND-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8
-; IND-NEXT: [[TMP13:%.*]] = add i8 [[T]], [[TMP4]]
-; IND-NEXT: [[TMP6:%.*]] = icmp ult i8 [[TMP13]], [[T]]
-; IND-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255
-; IND-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; IND-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
+; IND-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
+; IND-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; IND-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; IND-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; IND-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; IND-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; IND-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
; IND-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; IND: vector.ph:
; IND-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2
@@ -3854,14 +3847,13 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; UNROLL-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; UNROLL-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; UNROLL-NEXT: [[TMP10:%.*]] = icmp ugt i32 [[LEN]], 255
-; UNROLL-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP10]]
-; UNROLL-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8
-; UNROLL-NEXT: [[TMP15:%.*]] = add i8 [[T]], [[TMP4]]
-; UNROLL-NEXT: [[TMP6:%.*]] = icmp ult i8 [[TMP15]], [[T]]
-; UNROLL-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255
-; UNROLL-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; UNROLL-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
+; UNROLL-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
+; UNROLL-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; UNROLL-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; UNROLL-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; UNROLL-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; UNROLL-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; UNROLL-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
; UNROLL-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; UNROLL: vector.ph:
; UNROLL-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4
@@ -3928,15 +3920,14 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
-; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
-; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = trunc i32 [[LEN]] to i8
-; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i8 [[T]], [[TMP6]]
-; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp ult i8 [[TMP7]], [[T]]
; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255
-; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
-; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or i1 [[TMP5]], [[TMP10]]
-; UNROLL-NO-IC-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or i1 [[TMP3]], [[TMP9]]
+; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP11]], [[TMP13]]
+; UNROLL-NO-IC-NEXT: br i1 [[TMP10]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; UNROLL-NO-IC: vector.ph:
; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4
; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
@@ -4002,14 +3993,13 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; INTERLEAVE-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = icmp ugt i32 [[LEN]], 255
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP10]]
-; INTERLEAVE-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8
-; INTERLEAVE-NEXT: [[TMP15:%.*]] = add i8 [[T]], [[TMP4]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp ult i8 [[TMP15]], [[T]]
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
+; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
; INTERLEAVE-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; INTERLEAVE: vector.ph:
; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll b/llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll
index 52c2c313db130..2b8d936d7e7ce 100644
--- a/llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/replace-first-order-recurrence-by-versioned-iv.ll
@@ -258,15 +258,14 @@ define void @replace_first_order_recurrence_with_versioned_iv_for_uniform_non_po
; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 1, [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
-; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP4]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP1]] to i32
-; CHECK-NEXT: [[TMP12:%.*]] = add i32 1, [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP12]], 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
-; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP16]], [[TMP15]]
-; CHECK-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP4]], [[TMP14]]
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 1, [[TMP2]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp slt i32 [[TMP8]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
+; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP17]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[Y1]], -8
; CHECK-NEXT: [[TMP19:%.*]] = sub i64 [[TMP18]], [[X2]]
@@ -296,12 +295,12 @@ define void @replace_first_order_recurrence_with_versioned_iv_for_uniform_non_po
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[PHI32:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I32NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[PHI64:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[I64NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[PHI32:%.*]] = phi i32 [ [[TMP28]], %[[SCALAR_PH]] ], [ [[I32NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[PHI64:%.*]] = phi i64 [ [[TMP29]], %[[SCALAR_PH]] ], [ [[I64NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[I32NEXT]] = add i32 [[PHI32]], 1
; CHECK-NEXT: [[I64NEXT]] = zext i32 [[I32NEXT]] to i64
; CHECK-NEXT: [[XIP:%.*]] = getelementptr inbounds double, ptr [[X]], i32 [[I32NEXT]]
@@ -457,12 +456,12 @@ define void @for_iv_kept_predicate_needed(ptr %dst, i64 %n) {
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i16 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i16 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[NARROW:%.*]] = phi i16 [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[NARROW:%.*]] = phi i16 [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[PREV:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[EXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[INC]] = add i16 [[NARROW]], 1
@@ -594,11 +593,11 @@ define void @induction_with_predicate_replaceable_by_for(ptr %dst, i32 %len) {
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_PH]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i8 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i8 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[NARROW:%.*]] = phi i8 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[NARROW:%.*]] = phi i8 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[SPHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[EXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
; CHECK-NEXT: store i32 [[SPHI]], ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
index 27bf83f00da66..1c1f483834f50 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll
@@ -465,8 +465,7 @@ define void @remove_diff_checks_via_guards(i32 %x, i32 %y, ptr %A) {
; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[SMAX]], 4294967295
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = trunc i64 [[SMAX]] to i32
-; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[OFFSET]], [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[OFFSET]], [[TMP6]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp slt i32 [[TMP11]], [[OFFSET]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[SMAX]], 4294967295
; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
index 8272e2d9a1542..1b1e65cdb853f 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
@@ -35,12 +35,9 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP3]])
-; CHECK-NEXT: [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[SCEVGEP1]]
-; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW]]
; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP12]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
>From f0547868f318ce4e8daf1a1006790fa2daf81dc6 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Wed, 5 Aug 2026 10:23:40 +0000
Subject: [PATCH 2/2] Address review comment
---
llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index db6ca49d74b53..923ac3ce0bea6 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2323,7 +2323,7 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
auto ComputeEndCheck = [&]() -> Value * {
// Check to see if we already expanded this here.
Value *MulV, *OfMul;
- auto Key = std::make_tuple(TripCountVal, AbsStep, &*Loc);
+ auto Key = std::make_tuple(TripCountVal, AbsStep, Loc);
auto I = InsertedOverflowChecks.find(Key);
if (I != InsertedOverflowChecks.end()) {
MulV = I->second.first;
More information about the llvm-commits
mailing list