[llvm] [LoopVectorize] Enable vectorization of uncountable loops with invariant bound loads via speculative hoisting and runtime versioning. (PR #195027)
Pawan Nirpal via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 01:08:29 PDT 2026
https://github.com/pawan-nirpal-031 updated https://github.com/llvm/llvm-project/pull/195027
>From ce6347d4e088fdc2941e1a2e267cf04b75d6276b Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Tue, 16 Dec 2025 11:07:23 +0530
Subject: [PATCH 01/15] [AArch64] - Allow for aggressive unrolling, with
non-zero LoopMicroOpBufferSize for Oryon
---
llvm/lib/Target/AArch64/AArch64SchedOryon.td | 2 +-
.../aarch64-mcpu-oryon-runtime-unroll.ll | 152 ++++++++++++++++++
2 files changed, 153 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/aarch64-mcpu-oryon-runtime-unroll.ll
diff --git a/llvm/lib/Target/AArch64/AArch64SchedOryon.td b/llvm/lib/Target/AArch64/AArch64SchedOryon.td
index 5b597b91e7459..435eaf99c6175 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedOryon.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedOryon.td
@@ -19,7 +19,7 @@ def OryonModel : SchedMachineModel {
let MicroOpBufferSize = 376;
let LoadLatency = 4;
let MispredictPenalty = 13; // 13 cycles for mispredicted branch.
- let LoopMicroOpBufferSize = 0; // Do not have a LoopMicroOpBuffer
+ let LoopMicroOpBufferSize = 16; // Oryon-1 does not have loop micro op buffer, we enable this pseudo value to allow for aggressive unrolling based on runtime TC.
let PostRAScheduler = 1; // Using PostRA sched.
let CompleteModel = 1;
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mcpu-oryon-runtime-unroll.ll b/llvm/test/CodeGen/AArch64/aarch64-mcpu-oryon-runtime-unroll.ll
new file mode 100644
index 0000000000000..79136cf71c005
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-mcpu-oryon-runtime-unroll.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='loop-unroll' -S %s | FileCheck %s --check-prefix=UNROLLED
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @foo(ptr %mat, ptr %sharr, ptr %barr, i16 %rows, i16 %dimout) #0 {
+; UNROLLED-LABEL: define void @foo(
+; UNROLLED-SAME: ptr [[MAT:%.*]], ptr [[SHARR:%.*]], ptr [[BARR:%.*]], i16 [[ROWS:%.*]], i16 [[DIMOUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; UNROLLED-NEXT: [[ENTRY:.*:]]
+; UNROLLED-NEXT: [[CMP33:%.*]] = icmp sgt i16 [[DIMOUT]], 0
+; UNROLLED-NEXT: br i1 [[CMP33]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END22:.*]]
+; UNROLLED: [[FOR_BODY_LR_PH]]:
+; UNROLLED-NEXT: [[CMP631:%.*]] = icmp sgt i16 [[ROWS]], 0
+; UNROLLED-NEXT: br i1 [[CMP631]], label %[[FOR_BODY_US_PREHEADER:.*]], label %[[FOR_BODY_LR_PH_SPLIT:.*]]
+; UNROLLED: [[FOR_BODY_US_PREHEADER]]:
+; UNROLLED-NEXT: [[WIDE_TRIP_COUNT39:%.*]] = zext nneg i16 [[DIMOUT]] to i64
+; UNROLLED-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i16 [[ROWS]] to i64
+; UNROLLED-NEXT: [[TMP0:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
+; UNROLLED-NEXT: br label %[[FOR_BODY_US:.*]]
+; UNROLLED: [[FOR_BODY_US]]:
+; UNROLLED-NEXT: [[INDVARS_IV36:%.*]] = phi i64 [ 0, %[[FOR_BODY_US_PREHEADER]] ], [ [[INDVARS_IV_NEXT37:%.*]], %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US:.*]] ]
+; UNROLLED-NEXT: store i8 0, ptr [[BARR]], align 1
+; UNROLLED-NEXT: [[INVARIANT_GEP_US:%.*]] = getelementptr i8, ptr [[MAT]], i64 [[INDVARS_IV36]]
+; UNROLLED-NEXT: [[XTRAITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 1
+; UNROLLED-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; UNROLLED-NEXT: br i1 [[TMP1]], label %[[FOR_BODY8_US_EPIL_PREHEADER:.*]], label %[[FOR_BODY_US_NEW:.*]]
+; UNROLLED: [[FOR_BODY_US_NEW]]:
+; UNROLLED-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[XTRAITER]]
+; UNROLLED-NEXT: br label %[[FOR_BODY8_US:.*]]
+; UNROLLED: [[FOR_BODY8_US]]:
+; UNROLLED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_US_NEW]] ], [ [[INDVARS_IV_NEXT_1:%.*]], %[[FOR_INC_US_1:.*]] ]
+; UNROLLED-NEXT: [[TMP2:%.*]] = phi i8 [ 0, %[[FOR_BODY_US_NEW]] ], [ [[TMP8:%.*]], %[[FOR_INC_US_1]] ]
+; UNROLLED-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[FOR_BODY_US_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[FOR_INC_US_1]] ]
+; UNROLLED-NEXT: [[GEP_US:%.*]] = getelementptr [2 x i8], ptr [[INVARIANT_GEP_US]], i64 [[INDVARS_IV]]
+; UNROLLED-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_US]], align 1
+; UNROLLED-NEXT: [[TOBOOL_NOT_US:%.*]] = icmp eq i8 [[TMP3]], 0
+; UNROLLED-NEXT: br i1 [[TOBOOL_NOT_US]], label %[[FOR_INC_US:.*]], label %[[IF_THEN_US:.*]]
+; UNROLLED: [[IF_THEN_US]]:
+; UNROLLED-NEXT: [[ARRAYIDX14_US:%.*]] = getelementptr inbounds nuw i8, ptr [[SHARR]], i64 [[INDVARS_IV]]
+; UNROLLED-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX14_US]], align 1
+; UNROLLED-NEXT: [[XOR30_US:%.*]] = xor i8 [[TMP2]], [[TMP4]]
+; UNROLLED-NEXT: store i8 [[XOR30_US]], ptr [[BARR]], align 1
+; UNROLLED-NEXT: br label %[[FOR_INC_US]]
+; UNROLLED: [[FOR_INC_US]]:
+; UNROLLED-NEXT: [[TMP5:%.*]] = phi i8 [ [[TMP2]], %[[FOR_BODY8_US]] ], [ [[XOR30_US]], %[[IF_THEN_US]] ]
+; UNROLLED-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; UNROLLED-NEXT: [[GEP_US_1:%.*]] = getelementptr [2 x i8], ptr [[INVARIANT_GEP_US]], i64 [[INDVARS_IV_NEXT]]
+; UNROLLED-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_US_1]], align 1
+; UNROLLED-NEXT: [[TOBOOL_NOT_US_1:%.*]] = icmp eq i8 [[TMP6]], 0
+; UNROLLED-NEXT: br i1 [[TOBOOL_NOT_US_1]], label %[[FOR_INC_US_1]], label %[[IF_THEN_US_1:.*]]
+; UNROLLED: [[IF_THEN_US_1]]:
+; UNROLLED-NEXT: [[ARRAYIDX14_US_1:%.*]] = getelementptr inbounds nuw i8, ptr [[SHARR]], i64 [[INDVARS_IV_NEXT]]
+; UNROLLED-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX14_US_1]], align 1
+; UNROLLED-NEXT: [[XOR30_US_1:%.*]] = xor i8 [[TMP5]], [[TMP7]]
+; UNROLLED-NEXT: store i8 [[XOR30_US_1]], ptr [[BARR]], align 1
+; UNROLLED-NEXT: br label %[[FOR_INC_US_1]]
+; UNROLLED: [[FOR_INC_US_1]]:
+; UNROLLED-NEXT: [[TMP8]] = phi i8 [ [[TMP5]], %[[FOR_INC_US]] ], [ [[XOR30_US_1]], %[[IF_THEN_US_1]] ]
+; UNROLLED-NEXT: [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; UNROLLED-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; UNROLLED-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; UNROLLED-NEXT: br i1 [[NITER_NCMP_1]], label %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US_UNR_LCSSA:.*]], label %[[FOR_BODY8_US]]
+; UNROLLED: [[FOR_COND3_FOR_INC20_CRIT_EDGE_US_UNR_LCSSA]]:
+; UNROLLED-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1]], %[[FOR_INC_US_1]] ]
+; UNROLLED-NEXT: [[DOTUNR:%.*]] = phi i8 [ [[TMP8]], %[[FOR_INC_US_1]] ]
+; UNROLLED-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; UNROLLED-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_BODY8_US_EPIL_PREHEADER]], label %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US]]
+; UNROLLED: [[FOR_BODY8_US_EPIL_PREHEADER]]:
+; UNROLLED-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[FOR_BODY_US]] ], [ [[INDVARS_IV_UNR]], %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US_UNR_LCSSA]] ]
+; UNROLLED-NEXT: [[DOTEPIL_INIT:%.*]] = phi i8 [ 0, %[[FOR_BODY_US]] ], [ [[DOTUNR]], %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US_UNR_LCSSA]] ]
+; UNROLLED-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; UNROLLED-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]])
+; UNROLLED-NEXT: br label %[[FOR_BODY8_US_EPIL:.*]]
+; UNROLLED: [[FOR_BODY8_US_EPIL]]:
+; UNROLLED-NEXT: [[GEP_US_EPIL:%.*]] = getelementptr [2 x i8], ptr [[INVARIANT_GEP_US]], i64 [[INDVARS_IV_EPIL_INIT]]
+; UNROLLED-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_US_EPIL]], align 1
+; UNROLLED-NEXT: [[TOBOOL_NOT_US_EPIL:%.*]] = icmp eq i8 [[TMP9]], 0
+; UNROLLED-NEXT: br i1 [[TOBOOL_NOT_US_EPIL]], label %[[FOR_INC_US_EPIL:.*]], label %[[IF_THEN_US_EPIL:.*]]
+; UNROLLED: [[IF_THEN_US_EPIL]]:
+; UNROLLED-NEXT: [[ARRAYIDX14_US_EPIL:%.*]] = getelementptr inbounds nuw i8, ptr [[SHARR]], i64 [[INDVARS_IV_EPIL_INIT]]
+; UNROLLED-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX14_US_EPIL]], align 1
+; UNROLLED-NEXT: [[XOR30_US_EPIL:%.*]] = xor i8 [[DOTEPIL_INIT]], [[TMP10]]
+; UNROLLED-NEXT: store i8 [[XOR30_US_EPIL]], ptr [[BARR]], align 1
+; UNROLLED-NEXT: br label %[[FOR_INC_US_EPIL]]
+; UNROLLED: [[FOR_INC_US_EPIL]]:
+; UNROLLED-NEXT: br label %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US]]
+; UNROLLED: [[FOR_COND3_FOR_INC20_CRIT_EDGE_US]]:
+; UNROLLED-NEXT: [[INDVARS_IV_NEXT37]] = add nuw nsw i64 [[INDVARS_IV36]], 1
+; UNROLLED-NEXT: [[EXITCOND40_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT37]], [[WIDE_TRIP_COUNT39]]
+; UNROLLED-NEXT: br i1 [[EXITCOND40_NOT]], label %[[FOR_END22_LOOPEXIT:.*]], label %[[FOR_BODY_US]]
+; UNROLLED: [[FOR_BODY_LR_PH_SPLIT]]:
+; UNROLLED-NEXT: store i8 0, ptr [[BARR]], align 1
+; UNROLLED-NEXT: br label %[[FOR_END22]]
+; UNROLLED: [[FOR_END22_LOOPEXIT]]:
+; UNROLLED-NEXT: br label %[[FOR_END22]]
+; UNROLLED: [[FOR_END22]]:
+; UNROLLED-NEXT: ret void
+;
+entry:
+ %cmp33 = icmp sgt i16 %dimout, 0
+ br i1 %cmp33, label %for.body.lr.ph, label %for.end22
+
+for.body.lr.ph: ; preds = %entry
+ %cmp631 = icmp sgt i16 %rows, 0
+ br i1 %cmp631, label %for.body.us.preheader, label %for.body.lr.ph.split
+
+for.body.us.preheader: ; preds = %for.body.lr.ph
+ %wide.trip.count39 = zext nneg i16 %dimout to i64
+ %wide.trip.count = zext nneg i16 %rows to i64
+ br label %for.body.us
+
+for.body.us: ; preds = %for.body.us.preheader, %for.cond3.for.inc20_crit_edge.us
+ %indvars.iv36 = phi i64 [ 0, %for.body.us.preheader ], [ %indvars.iv.next37, %for.cond3.for.inc20_crit_edge.us ]
+ store i8 0, ptr %barr, align 1
+ %invariant.gep.us = getelementptr i8, ptr %mat, i64 %indvars.iv36
+ br label %for.body8.us
+
+for.body8.us: ; preds = %for.body.us, %for.inc.us
+ %indvars.iv = phi i64 [ 0, %for.body.us ], [ %indvars.iv.next, %for.inc.us ]
+ %0 = phi i8 [ 0, %for.body.us ], [ %3, %for.inc.us ]
+ %gep.us = getelementptr [2 x i8], ptr %invariant.gep.us, i64 %indvars.iv
+ %1 = load i8, ptr %gep.us, align 1
+ %tobool.not.us = icmp eq i8 %1, 0
+ br i1 %tobool.not.us, label %for.inc.us, label %if.then.us
+
+if.then.us: ; preds = %for.body8.us
+ %arrayidx14.us = getelementptr inbounds nuw i8, ptr %sharr, i64 %indvars.iv
+ %2 = load i8, ptr %arrayidx14.us, align 1
+ %xor30.us = xor i8 %0, %2
+ store i8 %xor30.us, ptr %barr, align 1
+ br label %for.inc.us
+
+for.inc.us: ; preds = %if.then.us, %for.body8.us
+ %3 = phi i8 [ %0, %for.body8.us ], [ %xor30.us, %if.then.us ]
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond3.for.inc20_crit_edge.us, label %for.body8.us
+
+for.cond3.for.inc20_crit_edge.us: ; preds = %for.inc.us
+ %indvars.iv.next37 = add nuw nsw i64 %indvars.iv36, 1
+ %exitcond40.not = icmp eq i64 %indvars.iv.next37, %wide.trip.count39
+ br i1 %exitcond40.not, label %for.end22, label %for.body.us
+
+for.body.lr.ph.split: ; preds = %for.body.lr.ph
+ store i8 0, ptr %barr, align 1
+ br label %for.end22
+
+for.end22: ; preds = %for.cond3.for.inc20_crit_edge.us, %for.body.lr.ph.split, %entry
+ ret void
+}
+
+attributes #0 = { "target-cpu"="oryon-1" "target-features"="+neon,+sve" }
>From 86f0e12d4478bb09207f0fe749358e1e6a491d3a Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Thu, 30 Apr 2026 00:31:17 -0700
Subject: [PATCH 02/15] [LoopVectorize] Enable vectorization of uncountable
loops with invariant bound loads via speculative hoisting and runtime
versioning
---
.../Vectorize/LoopVectorizationLegality.h | 2 +
.../Vectorize/LoopVectorizationLegality.cpp | 117 +++++++++
.../Transforms/Vectorize/LoopVectorize.cpp | 194 +++++++++++++++
.../dynamic-bound-array-element.ll | 177 ++++++++++++++
.../dynamic-bound-chained-loads.ll | 99 ++++++++
.../LoopVectorize/dynamic-bound-fcmp-exit.ll | 56 +++++
.../dynamic-bound-loop-varying.ll | 79 ++++++
.../LoopVectorize/dynamic-bound-n-stores.ll | 226 ++++++++++++++++++
.../LoopVectorize/dynamic-bound-simple.ll | 160 +++++++++++++
.../dynamic-bound-unsafe-calls.ll | 150 ++++++++++++
10 files changed, 1260 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 49e8bb1e85526..7cc5f037ff90f 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -350,6 +350,8 @@ class LoopVectorizationLegality {
/// reductions found in the loop.
bool isInvariantStoreOfReduction(StoreInst *SI);
+ LoadInst* tryToFindDyanmicBoundLoadCandidate(Loop *L, AAResults &AA);
+
/// Returns True if given address is invariant and is used to store recurrent
/// expression
bool isInvariantAddressOfReduction(Value *V);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index bee08eeba9927..4c141782007e8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -579,6 +579,123 @@ class SCEVAddRecForUniformityRewriter
} // namespace
+static Value *stripSimpleCasts(Value *V) {
+ while (isa<SExtInst>(V) || isa<ZExtInst>(V) || isa<TruncInst>(V))
+ V = cast<Instruction>(V)->getOperand(0);
+ return V;
+}
+
+static LoadInst *getLoopBoundLoadCandidate(Loop *L) {
+ Instruction *Icmp = L->getLatchCmpInst();
+ if (!Icmp)
+ return nullptr;
+
+ Value *Op0 = stripSimpleCasts(Icmp->getOperand(0));
+ Value *Op1 = stripSimpleCasts(Icmp->getOperand(1));
+
+ if (!isa<LoadInst>(Op0) && !isa<LoadInst>(Op1))
+ return nullptr;
+
+ LoadInst *BoundLoad =
+ isa<LoadInst>(Op0) ? dyn_cast<LoadInst>(Op0) : dyn_cast<LoadInst>(Op1);
+
+ // Reject non int load types.
+ if (!BoundLoad->getType()->isIntegerTy())
+ return nullptr;
+
+ return BoundLoad;
+}
+
+static bool isLoadSpeculativelyHoistable(Loop *L, LoadInst *LoadCandidate,
+ PredicatedScalarEvolution &PSE,
+ AAResults &AA) {
+
+ if (!L || !LoadCandidate)
+ return false;
+
+ if (!L->contains(LoadCandidate))
+ return false;
+
+ Value *Ptr = LoadCandidate->getPointerOperand();
+ if (!L->isLoopInvariant(Ptr))
+ return false;
+
+ // We don't want to deal with global pointers for now.
+ if (isa<GlobalValue>(Ptr))
+ return false;
+
+ if (!LoadCandidate->isSimple() || LoadCandidate->isVolatile())
+ return false;
+
+ const SCEV *PtrScev = PSE.getSCEV(Ptr);
+ if (!PtrScev || !PSE.getSE()->isLoopInvariant(PtrScev, L))
+ return false;
+
+ for (auto *User : Ptr->users()) {
+ Instruction *UserI = dyn_cast<Instruction>(User);
+ if (!UserI || UserI == dyn_cast<Instruction>(LoadCandidate))
+ continue;
+
+ if (!L->contains(UserI))
+ continue;
+
+ if (UserI->mayWriteToMemory())
+ return false;
+ }
+
+ MemoryLocation LoadLoc = MemoryLocation::get(LoadCandidate);
+ // No call in loop may write to Ptr via aliasing.
+ for (BasicBlock *BB : L->blocks()) {
+ for (Instruction &I : *BB) {
+ auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB)
+ continue;
+
+ MemoryEffects ME = AA.getMemoryEffects(CB);
+ // If call doesn't write anything it's safe.
+ if (!isModSet(ME.getModRef()))
+ continue;
+
+ // If call writes to escaped or global memory reject.
+ if (isModSet(ME.getModRef(IRMemLocation::Other)))
+ return false;
+
+ // Check if any args alias with loadloc.
+ for (Value *Arg : CB->args()) {
+ if (!Arg->getType()->isPointerTy())
+ continue;
+ if (AA.alias(MemoryLocation::getBeforeOrAfter(Arg), LoadLoc) !=
+ AliasResult::NoAlias)
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+LoadInst *
+LoopVectorizationLegality::tryToFindDyanmicBoundLoadCandidate(Loop *L,
+ AAResults &AA) {
+ if (!L->isInnermost())
+ return nullptr;
+
+ LoadInst *LoadCandidate = getLoopBoundLoadCandidate(L);
+ if (!LoadCandidate)
+ return nullptr;
+
+ // Reject if loop body may have some non-vectorizable calls, like pow/ instrs
+ // with side-effects, here even if the load were hoisted it would not allow
+ // the loop to vectorize anyway.
+ if (!canVectorizeInstrs())
+ return nullptr;
+
+ if (!isLoadSpeculativelyHoistable(L, LoadCandidate, PSE, AA))
+ return nullptr;
+
+ return LoadCandidate;
+}
+
bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const {
if (isInvariant(V))
return true;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6c9298a2cf98d..0614b13066f7f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -144,6 +144,7 @@
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
+#include "llvm/Transforms/Utils/Cloning.h"
#include <algorithm>
#include <cassert>
#include <cmath>
@@ -378,6 +379,8 @@ static cl::opt<bool> EnableEarlyExitVectorization(
// after prolog. See `emitIterationCountCheck`.
static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
+static bool AllowInvariantBoundRetry = true;
+
/// A helper function that returns true if the given type is irregular. The
/// type is irregular if its allocated size doesn't equal the store size of an
/// element of the corresponding vector type.
@@ -8163,6 +8166,179 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi.eraseFromParent();
}
+static bool isSpeculativeBoundVersioningProfitable() { return true; }
+
+
+/*
+
+ preheader
+ |
+ rtcheck-block
+ / \
+ / \
+ for.preheader ver.preheader
+ | |
+ for.body ver.for.body
+
+*/
+static Loop *tryToVersionLoopForInvariantBoundLoad(
+ Loop *L, LoadInst *BoundLoad, const DataLayout &DL, DominatorTree &DT,
+ LoopInfo &LI, AAResults &AA, PredicatedScalarEvolution &PSE,
+ AssumptionCache *AC) {
+ LoadInst *InnerLoad = BoundLoad;
+ if (!InnerLoad)
+ return nullptr;
+ auto *PreHeader = L->getLoopPreheader();
+ if (!PreHeader)
+ return nullptr;
+
+ Value *BoundPtrOl = InnerLoad->getPointerOperand();
+ if (auto *BoundPtrI = dyn_cast<Instruction>(BoundPtrOl))
+ if (!DT.properlyDominates(BoundPtrI->getParent(), PreHeader))
+ return nullptr;
+
+ // All the relevent base pointers of participating stores should dominate both
+ // loops.
+ LoopAccessInfoManager PreFlightLAIs(*PSE.getSE(), AA, DT, LI, nullptr,
+ nullptr, AC);
+ const LoopAccessInfo &PreLAI = PreFlightLAIs.getInfo(*L);
+ const auto &RtPtrChecking = *PreLAI.getRuntimePointerChecking();
+
+ for (const auto &Check : RtPtrChecking.getChecks()) {
+ for (const RuntimeCheckingPtrGroup *Group : {Check.first, Check.second}) {
+ for (unsigned Idx : Group->Members) {
+ Value *Ptr = RtPtrChecking.Pointers[Idx].PointerValue;
+ Value *Base = getUnderlyingObject(Ptr);
+ if (auto *BaseI = dyn_cast<Instruction>(Base))
+ if (!DT.properlyDominates(BaseI->getParent(), PreHeader)) {
+ return nullptr;
+ }
+ }
+ }
+ }
+
+ Function *ParentF = PreHeader->getParent();
+ LLVMContext &Ctx = ParentF->getContext();
+
+ BasicBlock *RtCheckBB =
+ BasicBlock::Create(Ctx, "ivbound.rtcheck", ParentF, PreHeader);
+
+ // Temporary terminator for SCEVExapnder to have valid insert point.
+ IRBuilder<> TmpBuilder(RtCheckBB);
+ Instruction *TmpTerm = TmpBuilder.CreateUnreachable();
+
+ // Re-wire the CFG.
+ // Redirect the predecessors of preheader the new rtcheck block.
+ SmallVector<BasicBlock *, 4> PreHeaderPreds(predecessors(PreHeader));
+ for (auto *Pred : PreHeaderPreds)
+ Pred->getTerminator()->replaceUsesOfWith(PreHeader, RtCheckBB);
+
+ BasicBlock *OldPreheadIdom = DT.getNode(PreHeader)->getIDom()->getBlock();
+ DT.addNewBlock(RtCheckBB, OldPreheadIdom);
+ DT.changeImmediateDominator(PreHeader, RtCheckBB);
+
+ // Fix the preheader phis via rtcblock phis.
+ for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
+ // Create the rtcblock phi.
+ PHINode *RtcPN =
+ PHINode::Create(PN.getType(), PreHeaderPreds.size(),
+ PN.getName() + ".rtcphi", &RtCheckBB->front());
+ for (unsigned i = 0; i < PN.getNumIncomingValues(); i++) {
+ BasicBlock *IBB = PN.getIncomingBlock(i);
+ assert(is_contained(PreHeaderPreds, IBB) &&
+ "Incoming Basic Block not part of predecessors list!");
+ RtcPN->addIncoming(PN.getIncomingValue(i), IBB);
+ }
+ PN.replaceAllUsesWith(RtcPN);
+ PN.eraseFromParent();
+ }
+
+ // Clone the loop including preheader.
+ ValueToValueMapTy VMap;
+ SmallVector<BasicBlock *, 4> NewBlocks;
+ Loop *VerLoop = cloneLoopWithPreheader(PreHeader, RtCheckBB, L, VMap, ".lver",
+ &LI, &DT, NewBlocks);
+
+ remapInstructionsInBlocks(NewBlocks, VMap);
+
+ BasicBlock *VerPreHeader = cast<BasicBlock>(VMap[PreHeader]);
+ BasicBlock *VerLatch = cast<BasicBlock>(VMap[L->getLoopLatch()]);
+
+ if (Loop *ParLoop = L->getParentLoop())
+ ParLoop->addBasicBlockToLoop(RtCheckBB, LI);
+
+ VerLatch->getTerminator()->setMetadata(LLVMContext::MD_loop, nullptr);
+
+ LoadInst *HoistLoad = cast<LoadInst>(InnerLoad->clone());
+ HoistLoad->setName(InnerLoad->getName() + ".speculatively.hoisted");
+ // Put this hoisted load in RtcheckBB, since SCEVExapnder needs it.
+ HoistLoad->insertBefore(TmpTerm);
+ Instruction *ClonedInnerLoad = cast<Instruction>(VMap[InnerLoad]);
+ ClonedInnerLoad->replaceAllUsesWith(HoistLoad);
+ ClonedInnerLoad->eraseFromParent();
+
+ // fix lcssa phis.
+ SmallVector<BasicBlock *, 4> ExitBlocks;
+ L->getExitBlocks(ExitBlocks);
+ for (BasicBlock *ExBB : ExitBlocks) {
+ DT.changeImmediateDominator(ExBB, RtCheckBB);
+ for (PHINode &Phi : ExBB->phis()) {
+ int idx = Phi.getBasicBlockIndex(L->getLoopLatch());
+ if (idx < 0)
+ continue;
+ Value *OrigVal = Phi.getIncomingValue(idx);
+ Value *NewVal =
+ VMap.count(OrigVal) ? cast<Value>(VMap[OrigVal]) : OrigVal;
+ Phi.addIncoming(NewVal, VerLatch);
+ }
+ }
+
+ formDedicatedExitBlocks(L, &DT, &LI, nullptr, true);
+ formDedicatedExitBlocks(VerLoop, &DT, &LI, nullptr, true);
+
+ // RTC condition generation using SCEV.
+ auto &SE = *PSE.getSE();
+ LoopAccessInfoManager NewLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
+ const LoopAccessInfo &VerLAI = NewLAIs.getInfo(*VerLoop);
+
+ // Generate LAA based ptr runtime checks.
+ SCEVExpander MemExp(*VerLAI.getRuntimePointerChecking()->getSE(),
+ "ivbound.mem.check");
+ Value *MemRTC = nullptr;
+ if (VerLAI.getRuntimePointerChecking()->Need)
+ MemRTC = addRuntimeChecks(TmpTerm, VerLoop,
+ VerLAI.getRuntimePointerChecking()->getChecks(),
+ MemExp);
+
+ // Generate SCEV predicate runtime checks.
+ SCEVExpander SCEVEx(SE, "ivbound.scev.check");
+ const SCEVPredicate &Preds = VerLAI.getPSE().getPredicate();
+ Value *SCEVrtc = SCEVEx.expandCodeForPredicate(&Preds, TmpTerm);
+
+ Value *RtCheckCond = nullptr;
+ if (MemRTC && SCEVrtc) {
+ IRBuilder<> Builder(TmpTerm);
+ RtCheckCond = Builder.CreateOr(MemRTC, SCEVrtc, "ivbound.safe");
+ } else {
+ RtCheckCond = MemRTC ? MemRTC : SCEVrtc;
+ }
+
+ TmpTerm->eraseFromParent();
+ IRBuilder<> Builder(RtCheckBB);
+
+ // If No runtime checks needed then ver-loop is unconditionally safe.
+ // Use constant false to disregard orignal loop path.
+ if (!RtCheckCond)
+ RtCheckCond = ConstantInt::getFalse(Ctx);
+
+ Builder.CreateCondBr(RtCheckCond, PreHeader, VerPreHeader);
+ // RTCheck now idom's both preheaders.
+ DT.insertEdge(RtCheckBB, PreHeader);
+ DT.insertEdge(RtCheckBB, VerPreHeader);
+
+ return VerLoop;
+}
+
bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&
"VPlan-native path is not enabled. Only process inner loops.");
@@ -8215,6 +8391,24 @@ bool LoopVectorizePass::processLoop(Loop *L) {
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
+ if (AllowInvariantBoundRetry) {
+ // There has to be a better way to do this? Trying to avoid changing processLoop Signature.
+ AllowInvariantBoundRetry = false;
+ const DataLayout &DL = F->getDataLayout();
+ // 1) Check if the loop qualifies for loop invariant bound load.
+ // 2) Generate the versioned loop with runtime checks using the dynamic loop bound.
+ // load. Return this versioned loop to attempt vectorization
+ // again. 3) If the versioned loop is generated successfully, re-try
+ // vectorization on the versioned loop.
+ if (LoadInst *BoundLoad = LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
+ if (isSpeculativeBoundVersioningProfitable())
+ if (Loop *Cand = tryToVersionLoopForInvariantBoundLoad(
+ L, BoundLoad, DL, *DT, *LI, *AA, PSE, AC)) {
+ SE->forgetLoop(L);
+ return processLoop(Cand);
+ }
+ }
+ }
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
Hints.emitRemarkWithHints();
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
new file mode 100644
index 0000000000000..78d4431437138
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; TEST10: Loop bound loaded from an array element A[2], where A is also
+; written inside the loop. The load and store use different indices, so
+; pointer identity does not flag a conflict, but runtime checks will
+; guard correctness.
+; Verify the load is hoisted and runtime checks are generated.
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < A[2]; i++)
+; A[i] = B[i] + C[i];
+; }
+;
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) #0 {
+; CHECK-LABEL: define dso_local void @foo(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C9:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: [[B8:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[A7:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[IVBOUND_RTCHECK]]:
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], false
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[A7]], [[B8]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 32
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[A7]], [[C9]]
+; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = icmp ult i64 [[TMP5]], 32
+; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK10]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD14]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD15]], [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP12]], i64 4
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX1_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1_LVER]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX3_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX3_LVER]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NEXT: [[ARRAYIDX5_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX5_LVER]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP19]], [[TMP18]]
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+;
+entry:
+ %arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
+ %0 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %cmp11 = icmp sgt i32 %0, 0
+ br i1 %cmp11, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx1 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx1, align 4, !tbaa !5
+ %arrayidx3 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx3, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx5, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
+; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]], [[META10:![0-9]+]]}
+; CHECK: [[META9]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META10]] = !{!"llvm.loop.unroll.disable"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
new file mode 100644
index 0000000000000..ffe3d2511b71b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; Negative test: the upper bound is **PtrLen (a chained / indirect load).
+; The worklist correctly traces both loads, but LoopAccessAnalysis cannot
+; compute SCEV bounds for the indirect load (load i32, ptr %ptr) because
+; %ptr itself is loaded inside the loop. The pass must reject the loop.
+;
+; C source:
+; void foo(int *A, int *B, int *C, int **PtrLen) {
+; for (int i = 0; i < **PtrLen; i++)
+; A[i] = B[i] + C[i];
+; }
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %PtrLen) #0 {
+; CHECK-LABEL: define dso_local void @foo(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readonly captures(none) [[PTRLEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PTR0:%.*]] = load ptr, ptr [[PTRLEN]], align 8, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[VAL0:%.*]] = load i32, ptr [[PTR0]], align 4, !tbaa [[TBAA5:![0-9]+]]
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[VAL0]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[PTR:%.*]] = load ptr, ptr [[PTRLEN]], align 8, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR]], align 4, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[VAL]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP2]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+;
+entry:
+ %ptr0 = load ptr, ptr %PtrLen, align 8, !tbaa !5
+ %val0 = load i32, ptr %ptr0, align 4, !tbaa !10
+ %cmp9 = icmp sgt i32 %val0, 0
+ br i1 %cmp9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !10
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !10
+ %add = add nsw i32 %2, %1
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx4, align 4, !tbaa !10
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %ptr = load ptr, ptr %PtrLen, align 8, !tbaa !5
+ %val = load i32, ptr %ptr, align 4, !tbaa !10
+ %3 = sext i32 %val to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %3
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !12
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 8}
+!6 = !{!7, i64 8, !"p1 int"}
+!7 = !{!8, i64 8, !"any pointer"}
+!8 = !{!9, i64 1, !"omnipotent char"}
+!9 = !{!"Simple C++ TBAA"}
+!10 = !{!11, !11, i64 0, i64 4}
+!11 = !{!8, i64 4, !"int"}
+!12 = distinct !{!12, !13, !14}
+!13 = !{!"llvm.loop.mustprogress"}
+!14 = !{!"llvm.loop.unroll.disable"}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 8}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 8, !"p1 int"}
+; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 8, !"any pointer"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]], i64 1, !"omnipotent char"}
+; CHECK: [[META4]] = !{!"Simple C++ TBAA"}
+; CHECK: [[TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0, i64 4}
+; CHECK: [[META6]] = !{[[META3]], i64 4, !"int"}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
+; CHECK: [[META8]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META9]] = !{!"llvm.loop.unroll.disable"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
new file mode 100644
index 0000000000000..915d728090a8e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; LoopVectorize speculative load hoist shouldn't kick in as SCEV cannot compute the trip count of a float-bounded loop, so the
+; pass enters the dynamic-bound path. It then rejects because the
+; branch condition is an FCmpInst, not an ICmpInst.
+;
+; void foo(float *A, float *B, float *Limit) {
+; for (float x = 0.0f; x < *Limit; x += 1.0f)
+; A[(int)x] = B[(int)x];
+; }
+
+define dso_local void @foo(ptr noundef %A, ptr noundef readonly %B, ptr noundef readonly %Limit) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[FIV:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[FIV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load i32, ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[B]], ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[FIV_NEXT]] = fadd float [[FIV]], 1.000000e+00
+; CHECK-NEXT: [[LIM:%.*]] = load float, ptr [[LIMIT:%.*]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[FIV_NEXT]], [[LIM]]
+; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_EXIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %fiv = phi float [ 0.0, %entry ], [ %fiv.next, %for.body ]
+ %gep.b = getelementptr inbounds i32, ptr %B, i64 %iv
+ %b = load i32, ptr %gep.b, align 4
+ %gep.a = getelementptr inbounds i32, ptr %A, i64 %iv
+ store i32 %b, ptr %gep.a, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %fiv.next = fadd float %fiv, 1.0
+ %lim = load float, ptr %Limit, align 4
+ %cmp = fcmp olt float %fiv.next, %lim
+ br i1 %cmp, label %for.body, label %for.exit, !llvm.loop !0
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
new file mode 100644
index 0000000000000..bdb4ae8b96b5f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; LoopVectorize speculative load hoist shouldn't kick in at all if the bound is Len[i], which depends on the
+; induction variable hence not invariant. The dependency chain traces back to a loop header
+; PHI, so the pass will reject this loop instead of versioning it.
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < Len[i]; i++)
+; A[i] = B[i] + C[i];
+; }
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %Len) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[CMP12:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP12]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.cond.cleanup.loopexit:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: ret void
+; CHECK: for.body:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i32, ptr [[C:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX6]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[LEN]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP4]]
+; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
+;
+entry:
+ %0 = load i32, ptr %Len, align 4, !tbaa !5
+ %cmp12 = icmp sgt i32 %0, 0
+ br i1 %cmp12, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx2, align 4, !tbaa !5
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx4, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx6 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx6, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %arrayidx = getelementptr inbounds nuw i32, ptr %Len, i64 %indvars.iv.next
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
new file mode 100644
index 0000000000000..c313f217d4393
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
@@ -0,0 +1,226 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64"
+
+define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %LoopBound) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @_Z3fooPiS_S_(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[LOOPBOUND:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4:![0-9]+]]
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[IVBOUND_RTCHECK]]:
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[FOUND_CONFLICT]], false
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX9:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX9]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[SMAX3]], 2
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
+; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[A]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[B]], [[SCEVGEP4]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX9]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX9]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD11]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD12]], [[WIDE_LOAD10]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP9]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP10]], splat (i32 1)
+; CHECK-NEXT: [[TMP13:%.*]] = add nsw <4 x i32> [[TMP11]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
+; CHECK-NEXT: store <4 x i32> [[TMP13]], ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX9]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2_LVER:%.*]] = add nsw i32 [[ADD_LVER]], 1
+; CHECK-NEXT: store i32 [[ADD2_LVER]], ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP19]], [[TMP18]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD]], 1
+; CHECK-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+;
+entry:
+ %0 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %cmp6 = icmp sgt i32 %0, 0
+ br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !4
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !4
+ %add = add nsw i32 %2, %1
+ store i32 %add, ptr %arrayidx2, align 4, !tbaa !4
+ %add2 = add nsw i32 %add, 1
+ store i32 %add2, ptr %arrayidx, align 4, !tbaa !4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !8
+}
+
+define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %LoopBound) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @bar(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[LOOPBOUND:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD]], 1
+; CHECK-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[TMP2]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP3]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP17]]
+;
+entry:
+ br label %for.body.preheader
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.cond.cleanup, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !4
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !4
+ %add = add nsw i32 %2, %1
+ store i32 %add, ptr %arrayidx2, align 4, !tbaa !4
+ %add2 = add nsw i32 %add, 1
+ store i32 %add2, ptr %arrayidx, align 4, !tbaa !4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !8
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+aes,+crc,+fp-armv8,+neon,+perfmon,+sha2,+v8a,-fmv" }
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!3}
+!llvm.errno.tbaa = !{!4}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"uwtable", i32 2}
+!2 = !{i32 7, !"frame-pointer", i32 4}
+!3 = !{!"clang version 23.0.0git (https://github.com/pawan-nirpal-031/llvm-project.git 93914ac0a0b80824d385c817cdc2d74ec6436b43)"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C++ TBAA"}
+!8 = distinct !{!8, !9}
+!9 = !{!"llvm.loop.mustprogress"}
+;.
+; CHECK: [[INT_TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
+; CHECK: [[META5]] = !{!"int", [[META6:![0-9]+]], i64 0}
+; CHECK: [[META6]] = !{!"omnipotent char", [[META7:![0-9]+]], i64 0}
+; CHECK: [[META7]] = !{!"Simple C++ TBAA"}
+; CHECK: [[META8]] = !{[[META9:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]], [[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], !"LVerDomain"}
+; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META10]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]]}
+; CHECK: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META15]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META14]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META18:![0-9]+]]}
+; CHECK: [[META18]] = !{!"llvm.loop.mustprogress"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
new file mode 100644
index 0000000000000..1ab4ef27079ca
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -0,0 +1,160 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+
+define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
+; CHECK-LABEL: define void @_Z3fooPiS_S_(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[LOOPBOUND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3:![0-9]+]]
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[IVBOUND_RTCHECK]]:
+; CHECK-NEXT: [[DOTRTCBLOCK_LOAD:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
+; CHECK-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX1]], 2
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND3:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
+; CHECK-NEXT: [[FOUND_CONFLICT1:%.*]] = and i1 [[BOUND2]], [[BOUND3]]
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[FOUND_CONFLICT1]], false
+; CHECK-NEXT: br i1 [[TMP9]], label %[[FOR_BODY_PREHEADER_LVER:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP10]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX3]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP11]], i64 1)
+; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX3]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX3]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP13]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP14]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META7]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP15]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP15]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10:![0-9]+]], !noalias [[META7]]
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
+; CHECK-NEXT: [[TMP17:%.*]] = add nsw <4 x i32> [[WIDE_LOAD5]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
+; CHECK-NEXT: store <4 x i32> [[TMP17]], ptr [[TMP15]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
+; CHECK-NEXT: store <4 x i32> [[TMP18]], ptr [[TMP16]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX3]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP21]], [[TMP20]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP22:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP22]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP24]], [[TMP23]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP26]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+;
+entry:
+ %0 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %cmp6 = icmp sgt i32 %0, 0
+ br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !4
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !4
+ %add = add nsw i32 %2, %1
+ store i32 %add, ptr %arrayidx2, align 4, !tbaa !4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !8
+}
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.errno.tbaa = !{!4}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"uwtable", i32 2}
+!2 = !{i32 7, !"frame-pointer", i32 4}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C++ TBAA"}
+!8 = distinct !{!8, !9}
+!9 = !{!"llvm.loop.mustprogress"}
+;.
+; CHECK: [[INT_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
+; CHECK: [[META4]] = !{!"int", [[META5:![0-9]+]], i64 0}
+; CHECK: [[META5]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
+; CHECK: [[META6]] = !{!"Simple C++ TBAA"}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"}
+; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
+; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
+; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META13]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]]}
+; CHECK: [[META17]] = !{!"llvm.loop.mustprogress"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
new file mode 100644
index 0000000000000..72468a2fe233e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+$warypen = comdat any
+
+; Function Attrs: uwtable
+define weak_odr dso_local noundef double @baz(ptr noundef nonnull align 8 dereferenceable(88) %arg, double noundef %arg1) local_unnamed_addr #0 comdat($warypen) {
+; CHECK-LABEL: define weak_odr dso_local noundef double @baz(
+; CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(88) [[ARG:%.*]], double noundef [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] comdat($warypen) {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 72
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GETELEMENTPTR]], align 8
+; CHECK-NEXT: [[ICMP:%.*]] = icmp eq i32 [[LOAD]], 0
+; CHECK-NEXT: br i1 [[ICMP]], label %[[BB6:.*]], label %[[BB2:.*]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: [[GETELEMENTPTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 80
+; CHECK-NEXT: [[LOAD4:%.*]] = load ptr, ptr [[GETELEMENTPTR3]], align 8, !tbaa [[DOUBLEPTR_TBAA14:![0-9]+]]
+; CHECK-NEXT: br label %[[BB8:.*]]
+; CHECK: [[BB5:.*]]:
+; CHECK-NEXT: [[PHI:%.*]] = phi double [ [[FADD:%.*]], %[[BB8]] ]
+; CHECK-NEXT: br label %[[BB6]]
+; CHECK: [[BB6]]:
+; CHECK-NEXT: [[PHI7:%.*]] = phi double [ 0.000000e+00, %[[BB]] ], [ [[PHI]], %[[BB5]] ]
+; CHECK-NEXT: [[FDIV:%.*]] = fdiv double 1.000000e+00, [[ARG1]]
+; CHECK-NEXT: [[CALL:%.*]] = tail call double @pow(double noundef [[PHI7]], double noundef [[FDIV]]) #[[ATTR3:[0-9]+]], !tbaa [[INT_TBAA10:![0-9]+]]
+; CHECK-NEXT: ret double [[CALL]]
+; CHECK: [[BB8]]:
+; CHECK-NEXT: [[PHI9:%.*]] = phi i64 [ 0, %[[BB2]] ], [ [[ADD:%.*]], %[[BB8]] ]
+; CHECK-NEXT: [[PHI10:%.*]] = phi double [ 0.000000e+00, %[[BB2]] ], [ [[FADD]], %[[BB8]] ]
+; CHECK-NEXT: [[GETELEMENTPTR11:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[LOAD4]], i64 [[PHI9]]
+; CHECK-NEXT: [[LOAD12:%.*]] = load double, ptr [[GETELEMENTPTR11]], align 8
+; CHECK-NEXT: [[CALL13:%.*]] = tail call noundef double @llvm.fabs.f64(double [[LOAD12]])
+; CHECK-NEXT: [[CALL14:%.*]] = tail call double @pow(double noundef [[CALL13]], double noundef [[ARG1]]) #[[ATTR3]], !tbaa [[INT_TBAA10]]
+; CHECK-NEXT: [[FADD]] = fadd double [[PHI10]], [[CALL14]]
+; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[PHI9]], 1
+; CHECK-NEXT: [[LOAD15:%.*]] = load i32, ptr [[GETELEMENTPTR]], align 8
+; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[LOAD15]] to i64
+; CHECK-NEXT: [[ICMP16:%.*]] = icmp samesign ult i64 [[ADD]], [[ZEXT]]
+; CHECK-NEXT: br i1 [[ICMP16]], label %[[BB8]], label %[[BB5]], !llvm.loop [[LOOP30:![0-9]+]]
+;
+bb:
+ %getelementptr = getelementptr inbounds nuw i8, ptr %arg, i64 72
+ %load = load i32, ptr %getelementptr, align 8
+ %icmp = icmp eq i32 %load, 0
+ br i1 %icmp, label %bb6, label %bb2
+
+bb2: ; preds = %bb
+ %getelementptr3 = getelementptr inbounds nuw i8, ptr %arg, i64 80
+ %load4 = load ptr, ptr %getelementptr3, align 8, !tbaa !14
+ br label %bb8
+
+bb5: ; preds = %bb8
+ %phi = phi double [ %fadd, %bb8 ]
+ br label %bb6
+
+bb6: ; preds = %bb5, %bb
+ %phi7 = phi double [ 0.000000e+00, %bb ], [ %phi, %bb5 ]
+ %fdiv = fdiv double 1.000000e+00, %arg1
+ %call = tail call double @pow(double noundef %phi7, double noundef %fdiv) #3, !tbaa !10
+ ret double %call
+
+bb8: ; preds = %bb8, %bb2
+ %phi9 = phi i64 [ 0, %bb2 ], [ %add, %bb8 ]
+ %phi10 = phi double [ 0.000000e+00, %bb2 ], [ %fadd, %bb8 ]
+ %getelementptr11 = getelementptr inbounds nuw [8 x i8], ptr %load4, i64 %phi9
+ %load12 = load double, ptr %getelementptr11, align 8
+ %call13 = tail call noundef double @llvm.fabs.f64(double %load12)
+ %call14 = tail call double @pow(double noundef %call13, double noundef %arg1) #3, !tbaa !10
+ %fadd = fadd double %phi10, %call14
+ %add = add nuw nsw i64 %phi9, 1
+ %load15 = load i32, ptr %getelementptr, align 8
+ %zext = zext i32 %load15 to i64
+ %icmp16 = icmp samesign ult i64 %add, %zext
+ br i1 %icmp16, label %bb8, label %bb5, !llvm.loop !30
+}
+
+; Function Attrs: mustprogress nocallback nofree nounwind willreturn memory(errnomem: write)
+declare double @pow(double noundef, double noundef) local_unnamed_addr #1
+
+; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare double @llvm.fabs.f64(double) #2
+
+attributes #0 = { uwtable "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+aes,+crc,+fp-armv8,+neon,+outline-atomics,+perfmon,+sha2,+v8a,-fmv" }
+attributes #1 = { mustprogress nocallback nofree nounwind willreturn memory(errnomem: write) "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+aes,+crc,+fp-armv8,+neon,+outline-atomics,+perfmon,+sha2,+v8a,-fmv" }
+attributes #2 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8}
+!llvm.ident = !{!9}
+!llvm.errno.tbaa = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 23.0.0git (ssh://git-hexagon-lv.qualcomm.com:29418/llvm/llvm-project fa6bb24a9808ef39e1ecebfc7d42c885fca2261f)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cc", directory: "well")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 8, !"PIC Level", i32 2}
+!4 = !{i32 7, !"PIE Level", i32 2}
+!5 = !{i32 7, !"uwtable", i32 2}
+!6 = !{i32 7, !"frame-pointer", i32 4}
+!7 = !{i32 1, !"ThinLTO", i32 0}
+!8 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!9 = !{!"clang version 23.0.0git (ssh://git-hexagon-lv.qualcomm.com:29418/llvm/llvm-project fa6bb24a9808ef39e1ecebfc7d42c885fca2261f)"}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"Simple C++ TBAA"}
+!14 = !{!15, !29, i64 80}
+!15 = !{!"sym1", !16, i64 0, !11, i64 72, !11, i64 76, !29, i64 80}
+!16 = !{!"sym2", !11, i64 8, !17, i64 16, !28, i64 64}
+!17 = !{!"sym3", !18, i64 0}
+!18 = !{!"sym4", !19, i64 0}
+!19 = !{!"sym5", !20, i64 0, !22, i64 8}
+!20 = !{!"sym6", !21, i64 0}
+!21 = !{!"sym7"}
+!22 = !{!"sym8", !23, i64 0, !27, i64 32}
+!23 = !{!"sym9", !24, i64 0, !25, i64 8, !25, i64 16, !25, i64 24}
+!24 = !{!"sym10", !12, i64 0}
+!25 = !{!"p1 sym9", !26, i64 0}
+!26 = !{!"any pointer", !12, i64 0}
+!27 = !{!"long", !12, i64 0}
+!28 = !{!"p1 sym11", !26, i64 0}
+!29 = !{!"p1 double", !26, i64 0}
+!30 = distinct !{!30}
+;.
+; CHECK: [[INT_TBAA10]] = !{[[META11:![0-9]+]], [[META11]], i64 0}
+; CHECK: [[META11]] = !{!"int", [[META12:![0-9]+]], i64 0}
+; CHECK: [[META12]] = !{!"omnipotent char", [[META13:![0-9]+]], i64 0}
+; CHECK: [[META13]] = !{!"Simple C++ TBAA"}
+; CHECK: [[DOUBLEPTR_TBAA14]] = !{[[META15:![0-9]+]], [[META29:![0-9]+]], i64 80}
+; CHECK: [[META15]] = !{!"sym1", [[META16:![0-9]+]], i64 0, [[META11]], i64 72, [[META11]], i64 76, [[META29]], i64 80}
+; CHECK: [[META16]] = !{!"sym2", [[META11]], i64 8, [[META17:![0-9]+]], i64 16, [[META28:![0-9]+]], i64 64}
+; CHECK: [[META17]] = !{!"sym3", [[META18:![0-9]+]], i64 0}
+; CHECK: [[META18]] = !{!"sym4", [[META19:![0-9]+]], i64 0}
+; CHECK: [[META19]] = !{!"sym5", [[META20:![0-9]+]], i64 0, [[META22:![0-9]+]], i64 8}
+; CHECK: [[META20]] = !{!"sym6", [[META21:![0-9]+]], i64 0}
+; CHECK: [[META21]] = !{!"sym7"}
+; CHECK: [[META22]] = !{!"sym8", [[META23:![0-9]+]], i64 0, [[META27:![0-9]+]], i64 32}
+; CHECK: [[META23]] = !{!"sym9", [[META24:![0-9]+]], i64 0, [[META25:![0-9]+]], i64 8, [[META25]], i64 16, [[META25]], i64 24}
+; CHECK: [[META24]] = !{!"sym10", [[META12]], i64 0}
+; CHECK: [[META25]] = !{!"p1 sym9", [[META26:![0-9]+]], i64 0}
+; CHECK: [[META26]] = !{!"any pointer", [[META12]], i64 0}
+; CHECK: [[META27]] = !{!"long", [[META12]], i64 0}
+; CHECK: [[META28]] = !{!"p1 sym11", [[META26]], i64 0}
+; CHECK: [[META29]] = !{!"p1 double", [[META26]], i64 0}
+; CHECK: [[LOOP30]] = distinct !{[[LOOP30]]}
+;.
>From ec754166819516a776e92cbbe307523aa1796949 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Thu, 30 Apr 2026 01:03:18 -0700
Subject: [PATCH 03/15] clang-format and comments
---
.../Vectorize/LoopVectorizationLegality.h | 2 +-
.../Vectorize/LoopVectorizationLegality.cpp | 2 +-
.../lib/Transforms/Vectorize/LoopVectorize.cpp | 18 ++++++++++--------
3 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 7cc5f037ff90f..21a3b11cad04d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -350,7 +350,7 @@ class LoopVectorizationLegality {
/// reductions found in the loop.
bool isInvariantStoreOfReduction(StoreInst *SI);
- LoadInst* tryToFindDyanmicBoundLoadCandidate(Loop *L, AAResults &AA);
+ LoadInst *tryToFindDyanmicBoundLoadCandidate(Loop *L, AAResults &AA);
/// Returns True if given address is invariant and is used to store recurrent
/// expression
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 4c141782007e8..894279c08cc0c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -676,7 +676,7 @@ static bool isLoadSpeculativelyHoistable(Loop *L, LoadInst *LoadCandidate,
LoadInst *
LoopVectorizationLegality::tryToFindDyanmicBoundLoadCandidate(Loop *L,
- AAResults &AA) {
+ AAResults &AA) {
if (!L->isInnermost())
return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0614b13066f7f..bb8b23b8947ec 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -136,6 +136,7 @@
#include "llvm/Support/NativeFormatting.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
@@ -144,7 +145,6 @@
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
-#include "llvm/Transforms/Utils/Cloning.h"
#include <algorithm>
#include <cassert>
#include <cmath>
@@ -8166,9 +8166,10 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi.eraseFromParent();
}
+// Is profitable to generate runtime check dominated version of the loop? which
+// has speculatively hoisted the loop bound load.
static bool isSpeculativeBoundVersioningProfitable() { return true; }
-
/*
preheader
@@ -8239,7 +8240,6 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
// Fix the preheader phis via rtcblock phis.
for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
- // Create the rtcblock phi.
PHINode *RtcPN =
PHINode::Create(PN.getType(), PreHeaderPreds.size(),
PN.getName() + ".rtcphi", &RtCheckBB->front());
@@ -8391,16 +8391,18 @@ bool LoopVectorizePass::processLoop(Loop *L) {
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
- if (AllowInvariantBoundRetry) {
- // There has to be a better way to do this? Trying to avoid changing processLoop Signature.
+ if (AllowInvariantBoundRetry) {
+ // There has to be a better way to do this? Trying to avoid changing
+ // processLoop Signature.
AllowInvariantBoundRetry = false;
const DataLayout &DL = F->getDataLayout();
// 1) Check if the loop qualifies for loop invariant bound load.
- // 2) Generate the versioned loop with runtime checks using the dynamic loop bound.
- // load. Return this versioned loop to attempt vectorization
+ // 2) Generate the versioned loop with runtime checks using the dynamic
+ // loop bound. load. Return this versioned loop to attempt vectorization
// again. 3) If the versioned loop is generated successfully, re-try
// vectorization on the versioned loop.
- if (LoadInst *BoundLoad = LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
+ if (LoadInst *BoundLoad =
+ LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
if (isSpeculativeBoundVersioningProfitable())
if (Loop *Cand = tryToVersionLoopForInvariantBoundLoad(
L, BoundLoad, DL, *DT, *LI, *AA, PSE, AC)) {
>From 74191cee9dc10c34aed2a975a688618f6dfb6e1c Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Thu, 30 Apr 2026 01:33:35 -0700
Subject: [PATCH 04/15] use iteartors instead of raw pointers for creating
instrs
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb8b23b8947ec..41117a13aef58 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8242,7 +8242,7 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
PHINode *RtcPN =
PHINode::Create(PN.getType(), PreHeaderPreds.size(),
- PN.getName() + ".rtcphi", &RtCheckBB->front());
+ PN.getName() + ".rtcphi", RtCheckBB->begin());
for (unsigned i = 0; i < PN.getNumIncomingValues(); i++) {
BasicBlock *IBB = PN.getIncomingBlock(i);
assert(is_contained(PreHeaderPreds, IBB) &&
@@ -8272,7 +8272,7 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
LoadInst *HoistLoad = cast<LoadInst>(InnerLoad->clone());
HoistLoad->setName(InnerLoad->getName() + ".speculatively.hoisted");
// Put this hoisted load in RtcheckBB, since SCEVExapnder needs it.
- HoistLoad->insertBefore(TmpTerm);
+ HoistLoad->insertBefore(TmpTerm->getIterator());
Instruction *ClonedInnerLoad = cast<Instruction>(VMap[InnerLoad]);
ClonedInnerLoad->replaceAllUsesWith(HoistLoad);
ClonedInnerLoad->eraseFromParent();
>From 4137209acc2f0afccaf3fa1a0f1446c13c622dfe Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Fri, 1 May 2026 01:37:44 -0700
Subject: [PATCH 05/15] use metadata to re-attempt vectoriztion
---
.../Transforms/Vectorize/LoopVectorize.cpp | 44 ++++++++++++++++---
1 file changed, 38 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 41117a13aef58..03b6b0bc2decf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -379,8 +379,6 @@ static cl::opt<bool> EnableEarlyExitVectorization(
// after prolog. See `emitIterationCountCheck`.
static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
-static bool AllowInvariantBoundRetry = true;
-
/// A helper function that returns true if the given type is irregular. The
/// type is irregular if its allocated size doesn't equal the store size of an
/// element of the corresponding vector type.
@@ -8336,6 +8334,28 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
DT.insertEdge(RtCheckBB, PreHeader);
DT.insertEdge(RtCheckBB, VerPreHeader);
+ assert(VerLatch &&
+ "Versioned loop must have a single latch, loop not in canonical form");
+
+ // Emit metadata llvm.loop.speculative.bound.hoist.versioned to prevent
+ // repeted recursive calls with same loop to processLoop.
+ Instruction *VerLatchTerm = VerLatch->getTerminator();
+ LLVMContext &ctx = VerLatchTerm->getContext();
+ MDNode *Marker = MDNode::get(
+ ctx, {(Metadata *)MDString::get(
+ Ctx, "llvm.loop.speculative.bound.hoist.versioned")});
+ MDNode *ExistingMD = VerLatchTerm->getMetadata(LLVMContext::MD_loop);
+ SmallVector<Metadata *, 4> Mds;
+ if (ExistingMD)
+ Mds.append(ExistingMD->op_begin(), ExistingMD->op_end());
+ else
+ Mds.push_back(nullptr);
+ Mds.push_back(Marker);
+ MDNode *NewLoopMD = MDNode::getDistinct(ctx, Mds);
+ if (!ExistingMD)
+ NewLoopMD->replaceOperandWith(0, NewLoopMD);
+ VerLatchTerm->setMetadata(LLVMContext::MD_loop, NewLoopMD);
+
return VerLoop;
}
@@ -8391,10 +8411,22 @@ bool LoopVectorizePass::processLoop(Loop *L) {
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
- if (AllowInvariantBoundRetry) {
- // There has to be a better way to do this? Trying to avoid changing
- // processLoop Signature.
- AllowInvariantBoundRetry = false;
+ bool VisitedSpeculativeHoist = false;
+ if (BasicBlock *Latch = L->getLoopLatch()) {
+ if (MDNode *LoopMD =
+ Latch->getTerminator()->getMetadata(LLVMContext::MD_loop)) {
+ for (unsigned i = 1, e = LoopMD->getNumOperands(); i < e; ++i) {
+ if (auto *Op = dyn_cast<MDNode>(LoopMD->getOperand(i)))
+ if (Op->getNumOperands() > 0)
+ if (auto *S = dyn_cast<MDString>(Op->getOperand(0)))
+ if (S->getString() ==
+ "llvm.loop.speculative.bound.hoist.versioned")
+ VisitedSpeculativeHoist = true;
+ }
+ }
+ }
+
+ if (!VisitedSpeculativeHoist) {
const DataLayout &DL = F->getDataLayout();
// 1) Check if the loop qualifies for loop invariant bound load.
// 2) Generate the versioned loop with runtime checks using the dynamic
>From d8e04099ea52be95b9a7be549bd651aa7feb0ee2 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Mon, 4 May 2026 13:42:18 +0530
Subject: [PATCH 06/15] test fixes
---
.../dynamic-bound-array-element.ll | 19 +--
.../LoopVectorize/dynamic-bound-n-stores.ll | 108 ++++++++++++++++--
.../LoopVectorize/dynamic-bound-simple.ll | 17 +--
3 files changed, 115 insertions(+), 29 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
index 78d4431437138..2729e5d8c5cde 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -97,7 +97,7 @@ define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
@@ -121,7 +121,7 @@ define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]]
;
entry:
%arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
@@ -167,11 +167,12 @@ attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: r
; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
-; CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]], [[META10:![0-9]+]]}
-; CHECK: [[META9]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META10]] = !{!"llvm.loop.unroll.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; CHECK: [[META5]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]], [[META6]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]]}
+; CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META11]] = !{!"llvm.loop.unroll.disable"}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
index c313f217d4393..0146089151411 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
@@ -82,7 +82,7 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
@@ -107,7 +107,7 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]]
;
entry:
%0 = load i32, ptr %LoopBound, align 4, !tbaa !4
@@ -146,11 +146,87 @@ define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY_PREHEADER:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP17]], i64 1)
+; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP18]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP18]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[FOUND_CONFLICT]], false
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER1:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX9:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP19]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX9]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP20]], i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[SMAX3]], 2
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[A]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[B]], [[SCEVGEP4]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX9]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX9]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP:.*]]:
-; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23:![0-9]+]], !noalias [[META20]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
+; CHECK-NEXT: [[TMP9:%.*]] = add nsw <4 x i32> [[WIDE_LOAD11]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD12]], [[WIDE_LOAD10]]
+; CHECK-NEXT: store <4 x i32> [[TMP9]], ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[TMP9]], splat (i32 1)
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP10]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP5]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
+; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX9]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[FOR_COND_CLEANUP]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP15]], [[TMP14]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2_LVER:%.*]] = add nsw i32 [[ADD_LVER]], 1
+; CHECK-NEXT: store i32 [[ADD2_LVER]], ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP16]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT1]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER1]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP1:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP1]]
+; CHECK: [[FOR_COND_CLEANUP1]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY1]] ], [ 0, %[[FOR_BODY_PREHEADER1]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
@@ -163,7 +239,7 @@ define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[TMP2]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP3]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP17]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY1]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP18]]
;
entry:
br label %for.body.preheader
@@ -217,10 +293,18 @@ attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: r
; CHECK: [[META10]] = distinct !{[[META10]], !"LVerDomain"}
; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
; CHECK: [[META12]] = distinct !{[[META12]], [[META10]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]]}
-; CHECK: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META15]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META14]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META18:![0-9]+]]}
-; CHECK: [[META18]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]], [[META16:![0-9]+]]}
+; CHECK: [[META14]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META15]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META16]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META14]], [[META15]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META19:![0-9]+]]}
+; CHECK: [[META19]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META20]] = !{[[META21:![0-9]+]]}
+; CHECK: [[META21]] = distinct !{[[META21]], [[META22:![0-9]+]]}
+; CHECK: [[META22]] = distinct !{[[META22]], !"LVerDomain"}
+; CHECK: [[META23]] = !{[[META24:![0-9]+]]}
+; CHECK: [[META24]] = distinct !{[[META24]], [[META22]]}
+; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META14]], [[META15]], [[META16]]}
+; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META14]], [[META15]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
index 1ab4ef27079ca..537dd105f5d67 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -75,7 +75,7 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
; CHECK-NEXT: [[TMP22:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP22]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
@@ -98,7 +98,7 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
; CHECK-NEXT: [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP26]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP17:![0-9]+]]
;
entry:
%0 = load i32, ptr %LoopBound, align 4, !tbaa !4
@@ -151,10 +151,11 @@ for.body: ; preds = %for.body.preheader,
; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"}
; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
-; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META13]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]]}
-; CHECK: [[META17]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META14:![0-9]+]], [[META15:![0-9]+]]}
+; CHECK: [[META13]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META15]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META13]], [[META14]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META18:![0-9]+]]}
+; CHECK: [[META18]] = !{!"llvm.loop.mustprogress"}
;.
>From 66f8c718114e590f0319e97b023330e5883e4d0c Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Fri, 8 May 2026 07:03:29 -0700
Subject: [PATCH 07/15] fixing some correctness issues
---
.../Vectorize/LoopVectorizationLegality.cpp | 39 ++++
.../Transforms/Vectorize/LoopVectorize.cpp | 59 ++++-
.../dynamic-bound-array-element.ll | 208 ++++++++++++------
3 files changed, 232 insertions(+), 74 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 894279c08cc0c..9bb357695066f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -590,6 +590,10 @@ static LoadInst *getLoopBoundLoadCandidate(Loop *L) {
if (!Icmp)
return nullptr;
+ // Only handle loops with a single exiting block (the latch).
+ if (!L->getExitingBlock())
+ return nullptr;
+
Value *Op0 = stripSimpleCasts(Icmp->getOperand(0));
Value *Op1 = stripSimpleCasts(Icmp->getOperand(1));
@@ -599,6 +603,11 @@ static LoadInst *getLoopBoundLoadCandidate(Loop *L) {
LoadInst *BoundLoad =
isa<LoadInst>(Op0) ? dyn_cast<LoadInst>(Op0) : dyn_cast<LoadInst>(Op1);
+ // The other operand must be loop variant.
+ Value *OtherOp = isa<LoadInst>(Op0) ? Op1 : Op0;
+ if (L->isLoopInvariant(OtherOp))
+ return nullptr;
+
// Reject non int load types.
if (!BoundLoad->getType()->isIntegerTy())
return nullptr;
@@ -643,6 +652,25 @@ static bool isLoadSpeculativelyHoistable(Loop *L, LoadInst *LoadCandidate,
return false;
}
+ // Reject cases where the BaseObjects are the same for any store with the
+ // loadCandiate.
+ SmallVector<const Value *, 4> BoundBases;
+ getUnderlyingObjects(Ptr, BoundBases);
+
+ for (BasicBlock *BB : L->getBlocks()) {
+ for (Instruction &I : *BB) {
+ auto *SI = dyn_cast<StoreInst>(&I);
+ if (!SI)
+ continue;
+ SmallVector<const Value *, 4> StrBases;
+ getUnderlyingObjects(SI->getPointerOperand(), StrBases);
+
+ for (const Value *SB : StrBases)
+ if ((is_contained(BoundBases, SB)))
+ return false;
+ }
+ }
+
MemoryLocation LoadLoc = MemoryLocation::get(LoadCandidate);
// No call in loop may write to Ptr via aliasing.
for (BasicBlock *BB : L->blocks()) {
@@ -680,6 +708,17 @@ LoopVectorizationLegality::tryToFindDyanmicBoundLoadCandidate(Loop *L,
if (!L->isInnermost())
return nullptr;
+ // Reject if the preheader is a loop header, this means the inner loop
+ // exits back into an outer loop body, which breaks SCEV's loop hierarchy
+ // after cloning.
+ BasicBlock *Preheader = L->getLoopPreheader();
+ if (!Preheader || LI->isLoopHeader(Preheader))
+ return nullptr;
+
+ // Reject loop with multi-exit blocks.
+ if (!L->getUniqueExitBlock())
+ return nullptr;
+
LoadInst *LoadCandidate = getLoopBoundLoadCandidate(L);
if (!LoadCandidate)
return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 03b6b0bc2decf..a3ef4365d1ae7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8184,6 +8184,11 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
Loop *L, LoadInst *BoundLoad, const DataLayout &DL, DominatorTree &DT,
LoopInfo &LI, AAResults &AA, PredicatedScalarEvolution &PSE,
AssumptionCache *AC) {
+
+ assert(
+ L->getUniqueExitBlock() &&
+ "Only loop with single exit block valid for bound ptr specultive hoist!");
+
LoadInst *InnerLoad = BoundLoad;
if (!InnerLoad)
return nullptr;
@@ -8202,6 +8207,7 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
nullptr, AC);
const LoopAccessInfo &PreLAI = PreFlightLAIs.getInfo(*L);
const auto &RtPtrChecking = *PreLAI.getRuntimePointerChecking();
+ auto &SE = *PSE.getSE();
for (const auto &Check : RtPtrChecking.getChecks()) {
for (const RuntimeCheckingPtrGroup *Group : {Check.first, Check.second}) {
@@ -8209,15 +8215,30 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
Value *Ptr = RtPtrChecking.Pointers[Idx].PointerValue;
Value *Base = getUnderlyingObject(Ptr);
if (auto *BaseI = dyn_cast<Instruction>(Base))
- if (!DT.properlyDominates(BaseI->getParent(), PreHeader)) {
+ if (!DT.properlyDominates(BaseI->getParent(), PreHeader))
return nullptr;
- }
+ const SCEV *PtrSCEV = SE.getSCEV(Ptr);
+ if (isa<SCEVCouldNotCompute>(PtrSCEV))
+ return nullptr;
+ if (SCEVExprContains(PtrSCEV, [&](const SCEV *S) {
+ const auto *SU = dyn_cast<SCEVUnknown>(S);
+ if (!SU)
+ return false;
+ auto *I = dyn_cast<Instruction>(SU->getValue());
+ if (!I)
+ return false;
+ if (!SE.isLoopInvariant(S, L))
+ return false;
+ return !DT.properlyDominates(I->getParent(), PreHeader);
+ }))
+ return nullptr;
}
}
}
Function *ParentF = PreHeader->getParent();
LLVMContext &Ctx = ParentF->getContext();
+ BasicBlock *OrigExitBlock = L->getUniqueExitBlock();
BasicBlock *RtCheckBB =
BasicBlock::Create(Ctx, "ivbound.rtcheck", ParentF, PreHeader);
@@ -8294,8 +8315,39 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
formDedicatedExitBlocks(L, &DT, &LI, nullptr, true);
formDedicatedExitBlocks(VerLoop, &DT, &LI, nullptr, true);
+ // Fix any preheader defns that are used in exit block. We don't work with
+ // multi-exit loops for now.
+ BasicBlock *NewExitBlock = L->getUniqueExitBlock(),
+ *VerNewExitBlock = VerLoop->getUniqueExitBlock();
+ if (NewExitBlock && VerNewExitBlock) {
+ for (Instruction &I : *PreHeader) {
+ if (I.isTerminator())
+ continue;
+ Value *CloneI = VMap.count(&I) ? VMap[&I] : nullptr;
+ if (!CloneI)
+ continue;
+ SmallVector<Use *, 4> ExitUses;
+ for (Use &U : I.uses()) {
+ auto *UI = dyn_cast<Instruction>(U.getUser());
+ BasicBlock *UserBlock = UI->getParent();
+ // This user is outside both loops and it's phi node should be inserted
+ // in original exit block, which will dominate all it's downstream uses.
+ if (UI && UserBlock != PreHeader && !L->contains(UserBlock) &&
+ !VerLoop->contains(UserBlock))
+ ExitUses.push_back(&U);
+ }
+ if (ExitUses.empty())
+ continue;
+ PHINode *MergePhi = PHINode::Create(
+ I.getType(), 2, I.getName() + ".merge", OrigExitBlock->begin());
+ MergePhi->addIncoming(&I, NewExitBlock);
+ MergePhi->addIncoming(CloneI, VerNewExitBlock);
+ for (Use *U : ExitUses)
+ U->set(MergePhi);
+ }
+ }
+
// RTC condition generation using SCEV.
- auto &SE = *PSE.getSE();
LoopAccessInfoManager NewLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
const LoopAccessInfo &VerLAI = NewLAIs.getInfo(*VerLoop);
@@ -8439,6 +8491,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (Loop *Cand = tryToVersionLoopForInvariantBoundLoad(
L, BoundLoad, DL, *DT, *LI, *AA, PSE, AC)) {
SE->forgetLoop(L);
+ SE->forgetLoop(Cand);
return processLoop(Cand);
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
index 2729e5d8c5cde..1419b049a976f 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -1,34 +1,96 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
-;
-; TEST10: Loop bound loaded from an array element A[2], where A is also
-; written inside the loop. The load and store use different indices, so
-; pointer identity does not flag a conflict, but runtime checks will
-; guard correctness.
-; Verify the load is hoisted and runtime checks are generated.
-;
-; void foo(int *A, int *B, int *C, int *Len) {
+
+; Not safe to hoist Speculatively.
+; void foo(int *A, int *B, int *C, int *Len) {
; for (int i = 0; i < A[2]; i++)
-; A[i] = B[i] + C[i];
-; }
+; A[i] = B[i] + C[i];
+; }
;
+; Can Speculatively hoist
+; void foo2(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < B[2]; i++)
+; A[i] = B[i] + C[i];
+; }
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
-target triple = "aarch64-unknown-linux-gnu"
+target triple = "aarch64-unknown-linux-gnueabi"
-define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) #0 {
-; CHECK-LABEL: define dso_local void @foo(
-; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @_Z3fooPiS_S_S_(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @_Z3fooPiS_S_S_(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5:![0-9]+]]
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP4]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+;
+entry:
+ %arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
+ %0 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %cmp11 = icmp sgt i32 %0, 0
+ br i1 %cmp11, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx1 = getelementptr inbounds nuw [4 x i8], ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx1, align 4, !tbaa !5
+ %arrayidx3 = getelementptr inbounds nuw [4 x i8], ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx3, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx5 = getelementptr inbounds nuw [4 x i8], ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx5, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !9
+}
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @_Z4foo2PiS_S_S_(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) local_unnamed_addr #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[C9:%.*]] = ptrtoaddr ptr [[C]] to i64
; CHECK-NEXT: [[B8:%.*]] = ptrtoaddr ptr [[B]] to i64
; CHECK-NEXT: [[A7:%.*]] = ptrtoaddr ptr [[A]] to i64
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP11]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[IVBOUND_RTCHECK]]:
-; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
@@ -62,23 +124,23 @@ define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD14]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD15]], [[WIDE_LOAD13]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP12]], i64 4
-; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
@@ -87,17 +149,17 @@ define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
; CHECK: [[FOR_BODY_LVER]]:
; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX1_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1_LVER]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[ARRAYIDX3_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX3_LVER]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX1_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1_LVER]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX3_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX3_LVER]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
-; CHECK-NEXT: [[ARRAYIDX5_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX5_LVER]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX5_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX5_LVER]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
@@ -110,69 +172,73 @@ define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP19]], [[TMP18]]
-; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
;
entry:
- %arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
+ %arrayidx = getelementptr inbounds nuw i8, ptr %B, i64 8
%0 = load i32, ptr %arrayidx, align 4, !tbaa !5
%cmp11 = icmp sgt i32 %0, 0
br i1 %cmp11, label %for.body, label %for.cond.cleanup
-for.cond.cleanup:
+for.cond.cleanup: ; preds = %for.body, %entry
ret void
-for.body:
+for.body: ; preds = %entry, %for.body
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
- %arrayidx1 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %arrayidx1 = getelementptr inbounds nuw [4 x i8], ptr %B, i64 %indvars.iv
%1 = load i32, ptr %arrayidx1, align 4, !tbaa !5
- %arrayidx3 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %arrayidx3 = getelementptr inbounds nuw [4 x i8], ptr %C, i64 %indvars.iv
%2 = load i32, ptr %arrayidx3, align 4, !tbaa !5
%add = add nsw i32 %2, %1
- %arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %arrayidx5 = getelementptr inbounds nuw [4 x i8], ptr %A, i64 %indvars.iv
store i32 %add, ptr %arrayidx5, align 4, !tbaa !5
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
%3 = load i32, ptr %arrayidx, align 4, !tbaa !5
%4 = sext i32 %3 to i64
%cmp = icmp slt i64 %indvars.iv.next, %4
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !11
}
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+!llvm.ident = !{!4}
+!llvm.errno.tbaa = !{!5}
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 8, !"PIC Level", i32 2}
-!2 = !{i32 7, !"PIE Level", i32 2}
-!3 = !{i32 7, !"uwtable", i32 2}
-!4 = !{!"clang version 21.1.8"}
-!5 = !{!6, !6, i64 0, i64 4}
-!6 = !{!7, i64 4, !"int"}
-!7 = !{!8, i64 1, !"omnipotent char"}
+!0 = !{i32 8, !"PIC Level", i32 2}
+!1 = !{i32 7, !"PIE Level", i32 2}
+!2 = !{i32 7, !"uwtable", i32 2}
+!3 = !{i32 7, !"frame-pointer", i32 4}
+!4 = !{!"clang version 23.0.0git (ssh://git-hexagon-lv.qualcomm.com:29418/llvm/llvm-project 3273b055baf0dcd8351e5e5ae744a6d45028fd9d)"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
!8 = !{!"Simple C++ TBAA"}
-!9 = distinct !{!9, !10, !11}
+!9 = distinct !{!9, !10}
!10 = !{!"llvm.loop.mustprogress"}
-!11 = !{!"llvm.loop.unroll.disable"}
+!11 = distinct !{!11, !10}
;.
-; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
-; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
-; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
-; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
-; CHECK: [[META5]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
-; CHECK: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]], [[META6]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]]}
+; CHECK: [[INT_TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
+; CHECK: [[META6]] = !{!"int", [[META7:![0-9]+]], i64 0}
+; CHECK: [[META7]] = !{!"omnipotent char", [[META8:![0-9]+]], i64 0}
+; CHECK: [[META8]] = !{!"Simple C++ TBAA"}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]]}
; CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META11]] = !{!"llvm.loop.unroll.disable"}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META12:![0-9]+]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
+; CHECK: [[META12]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META12]], [[META13]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META10]]}
;.
>From b915839e0d1036430ed882b8880db2ac0cdcc5d5 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Mon, 11 May 2026 06:40:27 -0700
Subject: [PATCH 08/15] adding store range check rtc and bail out on indirect
indexing case
---
.../Transforms/Vectorize/LoopVectorize.cpp | 53 +++-
.../dynamic-bound-array-element.ll | 6 +-
.../dynamic-bound-array-range-ld.ll | 248 ++++++++++++++++++
.../LoopVectorize/dynamic-bound-n-stores.ll | 20 +-
.../LoopVectorize/dynamic-bound-simple.ll | 6 +-
5 files changed, 324 insertions(+), 9 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a3ef4365d1ae7..2395937f0f854 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8209,6 +8209,13 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
const auto &RtPtrChecking = *PreLAI.getRuntimePointerChecking();
auto &SE = *PSE.getSE();
+ // If LAA could not compute complete pointer bounds (e.g. a gather/scatter
+ // access with an unknown indirect index), we cannot reliably determine the
+ // write ranges needed for the ivbound safety check. Bail out rather than
+ // proceeding with an incomplete or missing write-range check.
+ if (!PreLAI.canVectorizeMemory())
+ return nullptr;
+
for (const auto &Check : RtPtrChecking.getChecks()) {
for (const RuntimeCheckingPtrGroup *Group : {Check.first, Check.second}) {
for (unsigned Idx : Group->Members) {
@@ -8365,12 +8372,48 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
const SCEVPredicate &Preds = VerLAI.getPSE().getPredicate();
Value *SCEVrtc = SCEVEx.expandCodeForPredicate(&Preds, TmpTerm);
- Value *RtCheckCond = nullptr;
- if (MemRTC && SCEVrtc) {
+ // Check whether BoundPtrOl falls within any write group's [Low, High) byte
+ // range at runtime. If it does, the speculative hoist of the bound load is
+ // unsafe (a store in the loop will overwrite *BoundPtr), so we must fall
+ // back to the scalar loop.
+ SCEVExpander PreExp(SE, "ivbound.wg.check");
+ Value *BoundInWriteRange = nullptr;
+ for (const RuntimeCheckingPtrGroup &Group : RtPtrChecking.CheckingGroups) {
+ bool HasWrite = llvm::any_of(Group.Members, [&](unsigned Idx) {
+ return RtPtrChecking.Pointers[Idx].IsWritePtr;
+ });
+ if (!HasWrite)
+ continue;
+ Type *PtrArithTy = PointerType::get(Ctx, Group.AddressSpace);
+ Value *GroupLow = PreExp.expandCodeFor(Group.Low, PtrArithTy, TmpTerm);
+ Value *GroupHigh = PreExp.expandCodeFor(Group.High, PtrArithTy, TmpTerm);
+ if (Group.NeedsFreeze) {
+ IRBuilder<> FreezeB(TmpTerm);
+ GroupLow = FreezeB.CreateFreeze(GroupLow, "ivbound.wg.low.fr");
+ GroupHigh = FreezeB.CreateFreeze(GroupHigh, "ivbound.wg.high.fr");
+ }
IRBuilder<> Builder(TmpTerm);
- RtCheckCond = Builder.CreateOr(MemRTC, SCEVrtc, "ivbound.safe");
- } else {
- RtCheckCond = MemRTC ? MemRTC : SCEVrtc;
+ Value *Cmp0 =
+ Builder.CreateICmpULE(GroupLow, BoundPtrOl, "ivbound.wg.low.chk");
+ Value *Cmp1 =
+ Builder.CreateICmpULT(BoundPtrOl, GroupHigh, "ivbound.wg.high.chk");
+ Value *InRange = Builder.CreateAnd(Cmp0, Cmp1, "ivbound.in.write.range");
+ BoundInWriteRange =
+ BoundInWriteRange
+ ? Builder.CreateOr(BoundInWriteRange, InRange, "ivbound.wg.rdx")
+ : InRange;
+ }
+
+ Value *RtCheckCond = nullptr;
+ for (Value *Cond : {MemRTC, SCEVrtc, BoundInWriteRange}) {
+ if (!Cond)
+ continue;
+ if (!RtCheckCond) {
+ RtCheckCond = Cond;
+ } else {
+ IRBuilder<> Builder(TmpTerm);
+ RtCheckCond = Builder.CreateOr(RtCheckCond, Cond, "ivbound.safe");
+ }
}
TmpTerm->eraseFromParent();
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
index 1419b049a976f..0c36ec330a212 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -104,8 +104,12 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[ARRAYIDX]]
+; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[ARRAYIDX]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], false
-; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK-NEXT: [[IVBOUND_SAFE7:%.*]] = or i1 [[IVBOUND_SAFE]], [[IVBOUND_IN_WRITE_RANGE]]
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
new file mode 100644
index 0000000000000..0c36ec330a212
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
@@ -0,0 +1,248 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+
+; Not safe to hoist Speculatively.
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < A[2]; i++)
+; A[i] = B[i] + C[i];
+; }
+;
+; Can Speculatively hoist
+; void foo2(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < B[2]; i++)
+; A[i] = B[i] + C[i];
+; }
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnueabi"
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @_Z3fooPiS_S_S_(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @_Z3fooPiS_S_S_(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5:![0-9]+]]
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP4]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+;
+entry:
+ %arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
+ %0 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %cmp11 = icmp sgt i32 %0, 0
+ br i1 %cmp11, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx1 = getelementptr inbounds nuw [4 x i8], ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx1, align 4, !tbaa !5
+ %arrayidx3 = getelementptr inbounds nuw [4 x i8], ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx3, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx5 = getelementptr inbounds nuw [4 x i8], ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx5, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !9
+}
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @_Z4foo2PiS_S_S_(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C9:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: [[B8:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[A7:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[IVBOUND_RTCHECK]]:
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[ARRAYIDX]]
+; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[ARRAYIDX]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], false
+; CHECK-NEXT: [[IVBOUND_SAFE7:%.*]] = or i1 [[IVBOUND_SAFE]], [[IVBOUND_IN_WRITE_RANGE]]
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[A7]], [[B8]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 32
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[A7]], [[C9]]
+; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = icmp ult i64 [[TMP5]], 32
+; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK10]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD14]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD15]], [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP12]], i64 4
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX1_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1_LVER]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX3_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX3_LVER]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NEXT: [[ARRAYIDX5_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX5_LVER]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP19]], [[TMP18]]
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+;
+entry:
+ %arrayidx = getelementptr inbounds nuw i8, ptr %B, i64 8
+ %0 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %cmp11 = icmp sgt i32 %0, 0
+ br i1 %cmp11, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.body, %entry
+ ret void
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx1 = getelementptr inbounds nuw [4 x i8], ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx1, align 4, !tbaa !5
+ %arrayidx3 = getelementptr inbounds nuw [4 x i8], ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx3, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx5 = getelementptr inbounds nuw [4 x i8], ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx5, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !11
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+!llvm.ident = !{!4}
+!llvm.errno.tbaa = !{!5}
+
+!0 = !{i32 8, !"PIC Level", i32 2}
+!1 = !{i32 7, !"PIE Level", i32 2}
+!2 = !{i32 7, !"uwtable", i32 2}
+!3 = !{i32 7, !"frame-pointer", i32 4}
+!4 = !{!"clang version 23.0.0git (ssh://git-hexagon-lv.qualcomm.com:29418/llvm/llvm-project 3273b055baf0dcd8351e5e5ae744a6d45028fd9d)"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = distinct !{!11, !10}
+;.
+; CHECK: [[INT_TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
+; CHECK: [[META6]] = !{!"int", [[META7:![0-9]+]], i64 0}
+; CHECK: [[META7]] = !{!"omnipotent char", [[META8:![0-9]+]], i64 0}
+; CHECK: [[META8]] = !{!"Simple C++ TBAA"}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]]}
+; CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META12:![0-9]+]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
+; CHECK: [[META12]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META12]], [[META13]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META10]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
index 0146089151411..f3cb82b7497bd 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
@@ -21,8 +21,16 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[LOOPBOUND]]
+; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
+; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK3:%.*]] = icmp ule ptr [[B]], [[LOOPBOUND]]
+; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK4:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE5:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK3]], [[IVBOUND_WG_HIGH_CHK4]]
+; CHECK-NEXT: [[IVBOUND_WG_RDX:%.*]] = or i1 [[IVBOUND_IN_WRITE_RANGE]], [[IVBOUND_IN_WRITE_RANGE5]]
; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[FOUND_CONFLICT]], false
-; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK-NEXT: [[IVBOUND_SAFE6:%.*]] = or i1 [[IVBOUND_SAFE]], [[IVBOUND_WG_RDX]]
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE6]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[SMAX9:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
@@ -155,8 +163,16 @@ define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[LOOPBOUND]]
+; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
+; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK3:%.*]] = icmp ule ptr [[B]], [[LOOPBOUND]]
+; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK4:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE5:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK3]], [[IVBOUND_WG_HIGH_CHK4]]
+; CHECK-NEXT: [[IVBOUND_WG_RDX:%.*]] = or i1 [[IVBOUND_IN_WRITE_RANGE]], [[IVBOUND_IN_WRITE_RANGE5]]
; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[FOUND_CONFLICT]], false
-; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER1:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK-NEXT: [[IVBOUND_SAFE6:%.*]] = or i1 [[IVBOUND_SAFE]], [[IVBOUND_WG_RDX]]
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE6]], label %[[FOR_BODY_PREHEADER1:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[SMAX9:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP19]], i64 1)
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
index 537dd105f5d67..e5127696ed112 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -20,8 +20,12 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK-NEXT: [[BOUND2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
; CHECK-NEXT: [[BOUND3:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
; CHECK-NEXT: [[FOUND_CONFLICT1:%.*]] = and i1 [[BOUND2]], [[BOUND3]]
+; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[LOOPBOUND]]
+; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[FOUND_CONFLICT1]], false
-; CHECK-NEXT: br i1 [[TMP9]], label %[[FOR_BODY_PREHEADER_LVER:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK-NEXT: [[IVBOUND_SAFE3:%.*]] = or i1 [[TMP9]], [[IVBOUND_IN_WRITE_RANGE]]
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE3]], label %[[FOR_BODY_PREHEADER_LVER:.*]], label %[[FOR_BODY_PREHEADER:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP10]], i64 1)
>From dbf677e93a50050e3c0693b78c1e8f8996a25be5 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Wed, 20 May 2026 07:06:42 -0700
Subject: [PATCH 09/15] move versioning util in Loop versioning
---
.../llvm/Transforms/Utils/LoopVersioning.h | 35 +++
llvm/lib/Transforms/Utils/LoopVersioning.cpp | 277 +++++++++++++++++
.../Transforms/Vectorize/LoopVectorize.cpp | 294 +-----------------
3 files changed, 315 insertions(+), 291 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/LoopVersioning.h b/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
index ea4fe27c90f5c..16838a9fcface 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
@@ -156,6 +156,41 @@ class LoopVersioningPass : public PassInfoMixin<LoopVersioningPass> {
public:
PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
};
+
+class AAResults;
+class AssumptionCache;
+class LoadInst;
+class PredicatedScalarEvolution;
+
+/// Version \p L for speculative hoisting of an invariant bound load.
+///
+/// When a loop's trip count depends on a value loaded through a pointer that
+/// may alias stores inside the loop, SCEV cannot prove the bound is invariant
+/// and the loop is treated as uncountable. This function handles that pattern
+/// by emitting a versioned CFG:
+///
+/// preheader
+/// |
+/// ivbound.rtcheck <- new block: hoisted load + RTCs
+/// / \
+/// for.preheader ver.preheader <- original (fallback) / clone (fast path)
+/// | |
+/// for.body ver.for.body
+///
+/// The bound load is speculatively hoisted into the rtcheck block. Runtime
+/// alias checks (LAA mem-checks + SCEV predicates + write-range checks) guard
+/// whether the hoist is safe. If safe, control flows to the versioned clone
+/// (ver.for.body) which now has a countable trip count and can be vectorized.
+/// If unsafe, control falls back to the original loop which re-reads the bound
+/// on every iteration.
+///
+/// Returns the versioned (fast-path) loop clone on success, or nullptr if the
+/// loop does not qualify or the transform cannot be applied safely.
+Loop *versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
+ DominatorTree &DT, LoopInfo &LI,
+ AAResults &AA,
+ PredicatedScalarEvolution &PSE,
+ AssumptionCache *AC);
}
#endif
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index b90466a8c49cf..a0fdd1110b2bc 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -15,12 +15,18 @@
#include "llvm/Transforms/Utils/LoopVersioning.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfDataUtils.h"
@@ -268,6 +274,277 @@ void LoopVersioning::annotateInstWithNoAlias(Instruction *VersionedInst,
VersionedInst->setMetadata(LLVMContext::MD_noalias, NoAliasMD);
}
+Loop *llvm::versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
+ DominatorTree &DT, LoopInfo &LI,
+ AAResults &AA,
+ PredicatedScalarEvolution &PSE,
+ AssumptionCache *AC) {
+ assert(L->getUniqueExitBlock() &&
+ "Only loops with a single exit block are valid for bound ptr "
+ "speculative hoist!");
+
+ if (!BoundLoad)
+ return nullptr;
+ BasicBlock *PreHeader = L->getLoopPreheader();
+ if (!PreHeader)
+ return nullptr;
+
+ Value *BoundPtrOl = BoundLoad->getPointerOperand();
+ if (auto *BoundPtrI = dyn_cast<Instruction>(BoundPtrOl))
+ if (!DT.properlyDominates(BoundPtrI->getParent(), PreHeader))
+ return nullptr;
+
+ // All the relevant base pointers of participating stores should dominate both
+ // loops.
+ ScalarEvolution &SE = *PSE.getSE();
+ LoopAccessInfoManager PreFlightLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
+ const LoopAccessInfo &PreLAI = PreFlightLAIs.getInfo(*L);
+ const RuntimePointerChecking &RtPtrChecking =
+ *PreLAI.getRuntimePointerChecking();
+
+ // If LAA could not compute complete pointer bounds (e.g. a gather/scatter
+ // access with an unknown indirect index), we cannot reliably determine the
+ // write ranges needed for the ivbound safety check. Bail out rather than
+ // proceeding with an incomplete or missing write-range check.
+ if (!PreLAI.canVectorizeMemory())
+ return nullptr;
+
+ for (const auto &Check : RtPtrChecking.getChecks()) {
+ for (const RuntimeCheckingPtrGroup *Group : {Check.first, Check.second}) {
+ for (unsigned Idx : Group->Members) {
+ Value *Ptr = RtPtrChecking.Pointers[Idx].PointerValue;
+ Value *Base = getUnderlyingObject(Ptr);
+ if (auto *BaseI = dyn_cast<Instruction>(Base))
+ if (!DT.properlyDominates(BaseI->getParent(), PreHeader))
+ return nullptr;
+ const SCEV *PtrSCEV = SE.getSCEV(Ptr);
+ if (isa<SCEVCouldNotCompute>(PtrSCEV))
+ return nullptr;
+ if (SCEVExprContains(PtrSCEV, [&](const SCEV *S) {
+ const auto *SU = dyn_cast<SCEVUnknown>(S);
+ if (!SU)
+ return false;
+ auto *I = dyn_cast<Instruction>(SU->getValue());
+ if (!I)
+ return false;
+ if (!SE.isLoopInvariant(S, L))
+ return false;
+ return !DT.properlyDominates(I->getParent(), PreHeader);
+ }))
+ return nullptr;
+ }
+ }
+ }
+
+ Function *ParentF = PreHeader->getParent();
+ LLVMContext &Ctx = ParentF->getContext();
+ BasicBlock *OrigExitBlock = L->getUniqueExitBlock();
+
+ BasicBlock *RtCheckBB =
+ BasicBlock::Create(Ctx, "ivbound.rtcheck", ParentF, PreHeader);
+
+ // Temporary terminator so SCEVExpander has a valid insertion point.
+ IRBuilder<> TmpBuilder(RtCheckBB);
+ Instruction *TmpTerm = TmpBuilder.CreateUnreachable();
+
+ // Re-wire the CFG: redirect predecessors of the preheader to RtCheckBB.
+ SmallVector<BasicBlock *, 4> PreHeaderPreds(predecessors(PreHeader));
+ for (auto *Pred : PreHeaderPreds)
+ Pred->getTerminator()->replaceUsesOfWith(PreHeader, RtCheckBB);
+
+ BasicBlock *OldPreheadIdom = DT.getNode(PreHeader)->getIDom()->getBlock();
+ DT.addNewBlock(RtCheckBB, OldPreheadIdom);
+ DT.changeImmediateDominator(PreHeader, RtCheckBB);
+
+ // Fix preheader PHIs: move their incoming values into new PHIs in RtCheckBB.
+ for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
+ PHINode *RtcPN =
+ PHINode::Create(PN.getType(), PreHeaderPreds.size(),
+ PN.getName() + ".rtcphi", RtCheckBB->begin());
+ for (unsigned i = 0; i < PN.getNumIncomingValues(); i++) {
+ BasicBlock *IBB = PN.getIncomingBlock(i);
+ assert(is_contained(PreHeaderPreds, IBB) &&
+ "Incoming basic block not part of predecessors list!");
+ RtcPN->addIncoming(PN.getIncomingValue(i), IBB);
+ }
+ PN.replaceAllUsesWith(RtcPN);
+ PN.eraseFromParent();
+ }
+
+ // Clone the loop including its preheader.
+ ValueToValueMapTy VMap;
+ SmallVector<BasicBlock *, 4> NewBlocks;
+ Loop *VerLoop = cloneLoopWithPreheader(PreHeader, RtCheckBB, L, VMap,
+ ".lver", &LI, &DT, NewBlocks);
+ remapInstructionsInBlocks(NewBlocks, VMap);
+
+ BasicBlock *VerPreHeader = cast<BasicBlock>(VMap[PreHeader]);
+ BasicBlock *VerLatch = cast<BasicBlock>(VMap[L->getLoopLatch()]);
+
+ if (Loop *ParLoop = L->getParentLoop())
+ ParLoop->addBasicBlockToLoop(RtCheckBB, LI);
+
+ // Clear any existing loop metadata on the versioned latch; it will be
+ // replaced below with the speculative-hoist marker.
+ VerLatch->getTerminator()->setMetadata(LLVMContext::MD_loop, nullptr);
+
+ // Speculatively hoist the bound load into RtCheckBB so the versioned loop
+ // sees a loop-invariant trip count.
+ LoadInst *HoistLoad = cast<LoadInst>(BoundLoad->clone());
+ HoistLoad->setName(BoundLoad->getName() + ".speculatively.hoisted");
+ HoistLoad->insertBefore(TmpTerm->getIterator());
+ Instruction *ClonedBoundLoad = cast<Instruction>(VMap[BoundLoad]);
+ ClonedBoundLoad->replaceAllUsesWith(HoistLoad);
+ ClonedBoundLoad->eraseFromParent();
+
+ // Fix LCSSA PHIs: add incoming edges from the versioned latch.
+ SmallVector<BasicBlock *, 4> ExitBlocks;
+ L->getExitBlocks(ExitBlocks);
+ for (BasicBlock *ExBB : ExitBlocks) {
+ DT.changeImmediateDominator(ExBB, RtCheckBB);
+ for (PHINode &Phi : ExBB->phis()) {
+ int Idx = Phi.getBasicBlockIndex(L->getLoopLatch());
+ if (Idx < 0)
+ continue;
+ Value *OrigVal = Phi.getIncomingValue(Idx);
+ Value *NewVal =
+ VMap.count(OrigVal) ? cast<Value>(VMap[OrigVal]) : OrigVal;
+ Phi.addIncoming(NewVal, VerLatch);
+ }
+ }
+
+ formDedicatedExitBlocks(L, &DT, &LI, nullptr, true);
+ formDedicatedExitBlocks(VerLoop, &DT, &LI, nullptr, true);
+
+ // Fix any preheader definitions used outside both loops by inserting a merge
+ // PHI in the original exit block.
+ BasicBlock *NewExitBlock = L->getUniqueExitBlock();
+ BasicBlock *VerNewExitBlock = VerLoop->getUniqueExitBlock();
+ if (NewExitBlock && VerNewExitBlock) {
+ for (Instruction &I : *PreHeader) {
+ if (I.isTerminator())
+ continue;
+ Value *CloneI = VMap.count(&I) ? VMap[&I] : nullptr;
+ if (!CloneI)
+ continue;
+ SmallVector<Use *, 4> ExitUses;
+ for (Use &U : I.uses()) {
+ auto *UI = dyn_cast<Instruction>(U.getUser());
+ BasicBlock *UserBlock = UI->getParent();
+ if (UI && UserBlock != PreHeader && !L->contains(UserBlock) &&
+ !VerLoop->contains(UserBlock))
+ ExitUses.push_back(&U);
+ }
+ if (ExitUses.empty())
+ continue;
+ PHINode *MergePhi = PHINode::Create(I.getType(), 2,
+ I.getName() + ".merge",
+ OrigExitBlock->begin());
+ MergePhi->addIncoming(&I, NewExitBlock);
+ MergePhi->addIncoming(CloneI, VerNewExitBlock);
+ for (Use *U : ExitUses)
+ U->set(MergePhi);
+ }
+ }
+
+ // Generate runtime checks on the versioned loop using LAA.
+ LoopAccessInfoManager NewLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
+ const LoopAccessInfo &VerLAI = NewLAIs.getInfo(*VerLoop);
+
+ SCEVExpander MemExp(*VerLAI.getRuntimePointerChecking()->getSE(),
+ "ivbound.mem.check");
+ Value *MemRTC = nullptr;
+ if (VerLAI.getRuntimePointerChecking()->Need)
+ MemRTC = addRuntimeChecks(TmpTerm, VerLoop,
+ VerLAI.getRuntimePointerChecking()->getChecks(),
+ MemExp);
+
+ SCEVExpander SCEVEx(SE, "ivbound.scev.check");
+ const SCEVPredicate &Preds = VerLAI.getPSE().getPredicate();
+ Value *SCEVrtc = SCEVEx.expandCodeForPredicate(&Preds, TmpTerm);
+
+ // Check whether BoundPtrOl falls within any write group's [Low, High) byte
+ // range at runtime. If it does, the speculative hoist is unsafe (a store
+ // in the loop would overwrite *BoundPtr), so fall back to the scalar loop.
+ SCEVExpander PreExp(SE, "ivbound.wg.check");
+ Value *BoundInWriteRange = nullptr;
+ for (const RuntimeCheckingPtrGroup &Group : RtPtrChecking.CheckingGroups) {
+ bool HasWrite = llvm::any_of(Group.Members, [&](unsigned Idx) {
+ return RtPtrChecking.Pointers[Idx].IsWritePtr;
+ });
+ if (!HasWrite)
+ continue;
+ Type *PtrArithTy = PointerType::get(Ctx, Group.AddressSpace);
+ Value *GroupLow = PreExp.expandCodeFor(Group.Low, PtrArithTy, TmpTerm);
+ Value *GroupHigh = PreExp.expandCodeFor(Group.High, PtrArithTy, TmpTerm);
+ if (Group.NeedsFreeze) {
+ IRBuilder<> FreezeB(TmpTerm);
+ GroupLow = FreezeB.CreateFreeze(GroupLow, "ivbound.wg.low.fr");
+ GroupHigh = FreezeB.CreateFreeze(GroupHigh, "ivbound.wg.high.fr");
+ }
+ IRBuilder<> Builder(TmpTerm);
+ Value *Cmp0 =
+ Builder.CreateICmpULE(GroupLow, BoundPtrOl, "ivbound.wg.low.chk");
+ Value *Cmp1 =
+ Builder.CreateICmpULT(BoundPtrOl, GroupHigh, "ivbound.wg.high.chk");
+ Value *InRange = Builder.CreateAnd(Cmp0, Cmp1, "ivbound.in.write.range");
+ BoundInWriteRange =
+ BoundInWriteRange
+ ? Builder.CreateOr(BoundInWriteRange, InRange, "ivbound.wg.rdx")
+ : InRange;
+ }
+
+ // Combine all runtime check conditions: alias checks, SCEV predicates, and
+ // the bound-pointer write-range check.
+ Value *RtCheckCond = nullptr;
+ for (Value *Cond : {MemRTC, SCEVrtc, BoundInWriteRange}) {
+ if (!Cond)
+ continue;
+ if (!RtCheckCond) {
+ RtCheckCond = Cond;
+ } else {
+ IRBuilder<> Builder(TmpTerm);
+ RtCheckCond = Builder.CreateOr(RtCheckCond, Cond, "ivbound.safe");
+ }
+ }
+
+ TmpTerm->eraseFromParent();
+ IRBuilder<> Builder(RtCheckBB);
+
+ // If no runtime checks are needed the versioned loop is unconditionally safe;
+ // use constant false so the original (fallback) path is never taken.
+ if (!RtCheckCond)
+ RtCheckCond = ConstantInt::getFalse(Ctx);
+
+ // Branch: If conflict is detected then take the original loop path, else take the versioned loop path.
+ Builder.CreateCondBr(RtCheckCond, PreHeader, VerPreHeader);
+ DT.insertEdge(RtCheckBB, PreHeader);
+ DT.insertEdge(RtCheckBB, VerPreHeader);
+
+ assert(VerLatch &&
+ "Versioned loop must have a single latch; loop not in canonical form");
+
+ // Emit metadata on the versioned latch to prevent repeated re-versioning of
+ // the same loop if processLoop is called again on the versioned clone.
+ Instruction *VerLatchTerm = VerLatch->getTerminator();
+ MDNode *Marker = MDNode::get(
+ Ctx, {(Metadata *)MDString::get(
+ Ctx, "llvm.loop.speculative.bound.hoist.versioned")});
+ MDNode *ExistingMD = VerLatchTerm->getMetadata(LLVMContext::MD_loop);
+ SmallVector<Metadata *, 4> Mds;
+ if (ExistingMD)
+ Mds.append(ExistingMD->op_begin(), ExistingMD->op_end());
+ else
+ Mds.push_back(nullptr);
+ Mds.push_back(Marker);
+ MDNode *NewLoopMD = MDNode::getDistinct(Ctx, Mds);
+ if (!ExistingMD)
+ NewLoopMD->replaceOperandWith(0, NewLoopMD);
+ VerLatchTerm->setMetadata(LLVMContext::MD_loop, NewLoopMD);
+
+ return VerLoop;
+}
+
namespace {
bool runImpl(LoopInfo *LI, LoopAccessInfoManager &LAIs, DominatorTree *DT,
ScalarEvolution *SE) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2395937f0f854..9072fcbc6b816 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -136,7 +136,6 @@
#include "llvm/Support/NativeFormatting.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
@@ -8168,292 +8167,6 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
// has speculatively hoisted the loop bound load.
static bool isSpeculativeBoundVersioningProfitable() { return true; }
-/*
-
- preheader
- |
- rtcheck-block
- / \
- / \
- for.preheader ver.preheader
- | |
- for.body ver.for.body
-
-*/
-static Loop *tryToVersionLoopForInvariantBoundLoad(
- Loop *L, LoadInst *BoundLoad, const DataLayout &DL, DominatorTree &DT,
- LoopInfo &LI, AAResults &AA, PredicatedScalarEvolution &PSE,
- AssumptionCache *AC) {
-
- assert(
- L->getUniqueExitBlock() &&
- "Only loop with single exit block valid for bound ptr specultive hoist!");
-
- LoadInst *InnerLoad = BoundLoad;
- if (!InnerLoad)
- return nullptr;
- auto *PreHeader = L->getLoopPreheader();
- if (!PreHeader)
- return nullptr;
-
- Value *BoundPtrOl = InnerLoad->getPointerOperand();
- if (auto *BoundPtrI = dyn_cast<Instruction>(BoundPtrOl))
- if (!DT.properlyDominates(BoundPtrI->getParent(), PreHeader))
- return nullptr;
-
- // All the relevent base pointers of participating stores should dominate both
- // loops.
- LoopAccessInfoManager PreFlightLAIs(*PSE.getSE(), AA, DT, LI, nullptr,
- nullptr, AC);
- const LoopAccessInfo &PreLAI = PreFlightLAIs.getInfo(*L);
- const auto &RtPtrChecking = *PreLAI.getRuntimePointerChecking();
- auto &SE = *PSE.getSE();
-
- // If LAA could not compute complete pointer bounds (e.g. a gather/scatter
- // access with an unknown indirect index), we cannot reliably determine the
- // write ranges needed for the ivbound safety check. Bail out rather than
- // proceeding with an incomplete or missing write-range check.
- if (!PreLAI.canVectorizeMemory())
- return nullptr;
-
- for (const auto &Check : RtPtrChecking.getChecks()) {
- for (const RuntimeCheckingPtrGroup *Group : {Check.first, Check.second}) {
- for (unsigned Idx : Group->Members) {
- Value *Ptr = RtPtrChecking.Pointers[Idx].PointerValue;
- Value *Base = getUnderlyingObject(Ptr);
- if (auto *BaseI = dyn_cast<Instruction>(Base))
- if (!DT.properlyDominates(BaseI->getParent(), PreHeader))
- return nullptr;
- const SCEV *PtrSCEV = SE.getSCEV(Ptr);
- if (isa<SCEVCouldNotCompute>(PtrSCEV))
- return nullptr;
- if (SCEVExprContains(PtrSCEV, [&](const SCEV *S) {
- const auto *SU = dyn_cast<SCEVUnknown>(S);
- if (!SU)
- return false;
- auto *I = dyn_cast<Instruction>(SU->getValue());
- if (!I)
- return false;
- if (!SE.isLoopInvariant(S, L))
- return false;
- return !DT.properlyDominates(I->getParent(), PreHeader);
- }))
- return nullptr;
- }
- }
- }
-
- Function *ParentF = PreHeader->getParent();
- LLVMContext &Ctx = ParentF->getContext();
- BasicBlock *OrigExitBlock = L->getUniqueExitBlock();
-
- BasicBlock *RtCheckBB =
- BasicBlock::Create(Ctx, "ivbound.rtcheck", ParentF, PreHeader);
-
- // Temporary terminator for SCEVExapnder to have valid insert point.
- IRBuilder<> TmpBuilder(RtCheckBB);
- Instruction *TmpTerm = TmpBuilder.CreateUnreachable();
-
- // Re-wire the CFG.
- // Redirect the predecessors of preheader the new rtcheck block.
- SmallVector<BasicBlock *, 4> PreHeaderPreds(predecessors(PreHeader));
- for (auto *Pred : PreHeaderPreds)
- Pred->getTerminator()->replaceUsesOfWith(PreHeader, RtCheckBB);
-
- BasicBlock *OldPreheadIdom = DT.getNode(PreHeader)->getIDom()->getBlock();
- DT.addNewBlock(RtCheckBB, OldPreheadIdom);
- DT.changeImmediateDominator(PreHeader, RtCheckBB);
-
- // Fix the preheader phis via rtcblock phis.
- for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
- PHINode *RtcPN =
- PHINode::Create(PN.getType(), PreHeaderPreds.size(),
- PN.getName() + ".rtcphi", RtCheckBB->begin());
- for (unsigned i = 0; i < PN.getNumIncomingValues(); i++) {
- BasicBlock *IBB = PN.getIncomingBlock(i);
- assert(is_contained(PreHeaderPreds, IBB) &&
- "Incoming Basic Block not part of predecessors list!");
- RtcPN->addIncoming(PN.getIncomingValue(i), IBB);
- }
- PN.replaceAllUsesWith(RtcPN);
- PN.eraseFromParent();
- }
-
- // Clone the loop including preheader.
- ValueToValueMapTy VMap;
- SmallVector<BasicBlock *, 4> NewBlocks;
- Loop *VerLoop = cloneLoopWithPreheader(PreHeader, RtCheckBB, L, VMap, ".lver",
- &LI, &DT, NewBlocks);
-
- remapInstructionsInBlocks(NewBlocks, VMap);
-
- BasicBlock *VerPreHeader = cast<BasicBlock>(VMap[PreHeader]);
- BasicBlock *VerLatch = cast<BasicBlock>(VMap[L->getLoopLatch()]);
-
- if (Loop *ParLoop = L->getParentLoop())
- ParLoop->addBasicBlockToLoop(RtCheckBB, LI);
-
- VerLatch->getTerminator()->setMetadata(LLVMContext::MD_loop, nullptr);
-
- LoadInst *HoistLoad = cast<LoadInst>(InnerLoad->clone());
- HoistLoad->setName(InnerLoad->getName() + ".speculatively.hoisted");
- // Put this hoisted load in RtcheckBB, since SCEVExapnder needs it.
- HoistLoad->insertBefore(TmpTerm->getIterator());
- Instruction *ClonedInnerLoad = cast<Instruction>(VMap[InnerLoad]);
- ClonedInnerLoad->replaceAllUsesWith(HoistLoad);
- ClonedInnerLoad->eraseFromParent();
-
- // fix lcssa phis.
- SmallVector<BasicBlock *, 4> ExitBlocks;
- L->getExitBlocks(ExitBlocks);
- for (BasicBlock *ExBB : ExitBlocks) {
- DT.changeImmediateDominator(ExBB, RtCheckBB);
- for (PHINode &Phi : ExBB->phis()) {
- int idx = Phi.getBasicBlockIndex(L->getLoopLatch());
- if (idx < 0)
- continue;
- Value *OrigVal = Phi.getIncomingValue(idx);
- Value *NewVal =
- VMap.count(OrigVal) ? cast<Value>(VMap[OrigVal]) : OrigVal;
- Phi.addIncoming(NewVal, VerLatch);
- }
- }
-
- formDedicatedExitBlocks(L, &DT, &LI, nullptr, true);
- formDedicatedExitBlocks(VerLoop, &DT, &LI, nullptr, true);
-
- // Fix any preheader defns that are used in exit block. We don't work with
- // multi-exit loops for now.
- BasicBlock *NewExitBlock = L->getUniqueExitBlock(),
- *VerNewExitBlock = VerLoop->getUniqueExitBlock();
- if (NewExitBlock && VerNewExitBlock) {
- for (Instruction &I : *PreHeader) {
- if (I.isTerminator())
- continue;
- Value *CloneI = VMap.count(&I) ? VMap[&I] : nullptr;
- if (!CloneI)
- continue;
- SmallVector<Use *, 4> ExitUses;
- for (Use &U : I.uses()) {
- auto *UI = dyn_cast<Instruction>(U.getUser());
- BasicBlock *UserBlock = UI->getParent();
- // This user is outside both loops and it's phi node should be inserted
- // in original exit block, which will dominate all it's downstream uses.
- if (UI && UserBlock != PreHeader && !L->contains(UserBlock) &&
- !VerLoop->contains(UserBlock))
- ExitUses.push_back(&U);
- }
- if (ExitUses.empty())
- continue;
- PHINode *MergePhi = PHINode::Create(
- I.getType(), 2, I.getName() + ".merge", OrigExitBlock->begin());
- MergePhi->addIncoming(&I, NewExitBlock);
- MergePhi->addIncoming(CloneI, VerNewExitBlock);
- for (Use *U : ExitUses)
- U->set(MergePhi);
- }
- }
-
- // RTC condition generation using SCEV.
- LoopAccessInfoManager NewLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
- const LoopAccessInfo &VerLAI = NewLAIs.getInfo(*VerLoop);
-
- // Generate LAA based ptr runtime checks.
- SCEVExpander MemExp(*VerLAI.getRuntimePointerChecking()->getSE(),
- "ivbound.mem.check");
- Value *MemRTC = nullptr;
- if (VerLAI.getRuntimePointerChecking()->Need)
- MemRTC = addRuntimeChecks(TmpTerm, VerLoop,
- VerLAI.getRuntimePointerChecking()->getChecks(),
- MemExp);
-
- // Generate SCEV predicate runtime checks.
- SCEVExpander SCEVEx(SE, "ivbound.scev.check");
- const SCEVPredicate &Preds = VerLAI.getPSE().getPredicate();
- Value *SCEVrtc = SCEVEx.expandCodeForPredicate(&Preds, TmpTerm);
-
- // Check whether BoundPtrOl falls within any write group's [Low, High) byte
- // range at runtime. If it does, the speculative hoist of the bound load is
- // unsafe (a store in the loop will overwrite *BoundPtr), so we must fall
- // back to the scalar loop.
- SCEVExpander PreExp(SE, "ivbound.wg.check");
- Value *BoundInWriteRange = nullptr;
- for (const RuntimeCheckingPtrGroup &Group : RtPtrChecking.CheckingGroups) {
- bool HasWrite = llvm::any_of(Group.Members, [&](unsigned Idx) {
- return RtPtrChecking.Pointers[Idx].IsWritePtr;
- });
- if (!HasWrite)
- continue;
- Type *PtrArithTy = PointerType::get(Ctx, Group.AddressSpace);
- Value *GroupLow = PreExp.expandCodeFor(Group.Low, PtrArithTy, TmpTerm);
- Value *GroupHigh = PreExp.expandCodeFor(Group.High, PtrArithTy, TmpTerm);
- if (Group.NeedsFreeze) {
- IRBuilder<> FreezeB(TmpTerm);
- GroupLow = FreezeB.CreateFreeze(GroupLow, "ivbound.wg.low.fr");
- GroupHigh = FreezeB.CreateFreeze(GroupHigh, "ivbound.wg.high.fr");
- }
- IRBuilder<> Builder(TmpTerm);
- Value *Cmp0 =
- Builder.CreateICmpULE(GroupLow, BoundPtrOl, "ivbound.wg.low.chk");
- Value *Cmp1 =
- Builder.CreateICmpULT(BoundPtrOl, GroupHigh, "ivbound.wg.high.chk");
- Value *InRange = Builder.CreateAnd(Cmp0, Cmp1, "ivbound.in.write.range");
- BoundInWriteRange =
- BoundInWriteRange
- ? Builder.CreateOr(BoundInWriteRange, InRange, "ivbound.wg.rdx")
- : InRange;
- }
-
- Value *RtCheckCond = nullptr;
- for (Value *Cond : {MemRTC, SCEVrtc, BoundInWriteRange}) {
- if (!Cond)
- continue;
- if (!RtCheckCond) {
- RtCheckCond = Cond;
- } else {
- IRBuilder<> Builder(TmpTerm);
- RtCheckCond = Builder.CreateOr(RtCheckCond, Cond, "ivbound.safe");
- }
- }
-
- TmpTerm->eraseFromParent();
- IRBuilder<> Builder(RtCheckBB);
-
- // If No runtime checks needed then ver-loop is unconditionally safe.
- // Use constant false to disregard orignal loop path.
- if (!RtCheckCond)
- RtCheckCond = ConstantInt::getFalse(Ctx);
-
- Builder.CreateCondBr(RtCheckCond, PreHeader, VerPreHeader);
- // RTCheck now idom's both preheaders.
- DT.insertEdge(RtCheckBB, PreHeader);
- DT.insertEdge(RtCheckBB, VerPreHeader);
-
- assert(VerLatch &&
- "Versioned loop must have a single latch, loop not in canonical form");
-
- // Emit metadata llvm.loop.speculative.bound.hoist.versioned to prevent
- // repeted recursive calls with same loop to processLoop.
- Instruction *VerLatchTerm = VerLatch->getTerminator();
- LLVMContext &ctx = VerLatchTerm->getContext();
- MDNode *Marker = MDNode::get(
- ctx, {(Metadata *)MDString::get(
- Ctx, "llvm.loop.speculative.bound.hoist.versioned")});
- MDNode *ExistingMD = VerLatchTerm->getMetadata(LLVMContext::MD_loop);
- SmallVector<Metadata *, 4> Mds;
- if (ExistingMD)
- Mds.append(ExistingMD->op_begin(), ExistingMD->op_end());
- else
- Mds.push_back(nullptr);
- Mds.push_back(Marker);
- MDNode *NewLoopMD = MDNode::getDistinct(ctx, Mds);
- if (!ExistingMD)
- NewLoopMD->replaceOperandWith(0, NewLoopMD);
- VerLatchTerm->setMetadata(LLVMContext::MD_loop, NewLoopMD);
-
- return VerLoop;
-}
-
bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&
"VPlan-native path is not enabled. Only process inner loops.");
@@ -8522,17 +8235,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
}
if (!VisitedSpeculativeHoist) {
- const DataLayout &DL = F->getDataLayout();
// 1) Check if the loop qualifies for loop invariant bound load.
// 2) Generate the versioned loop with runtime checks using the dynamic
- // loop bound. load. Return this versioned loop to attempt vectorization
+ // loop bound load. Return this versioned loop to attempt vectorization
// again. 3) If the versioned loop is generated successfully, re-try
// vectorization on the versioned loop.
if (LoadInst *BoundLoad =
LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
if (isSpeculativeBoundVersioningProfitable())
- if (Loop *Cand = tryToVersionLoopForInvariantBoundLoad(
- L, BoundLoad, DL, *DT, *LI, *AA, PSE, AC)) {
+ if (Loop *Cand = versionLoopForInvariantBoundLoad(L, BoundLoad, *DT,
+ *LI, *AA, PSE, AC)) {
SE->forgetLoop(L);
SE->forgetLoop(Cand);
return processLoop(Cand);
>From d95a5433447556745177ee2adb2f969a1d3f6f1e Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Wed, 20 May 2026 07:59:23 -0700
Subject: [PATCH 10/15] fix typo and add comment for stub
---
.../llvm/Transforms/Vectorize/LoopVectorizationLegality.h | 2 +-
.../Transforms/Vectorize/LoopVectorizationLegality.cpp | 2 +-
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 +++++---
3 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 21a3b11cad04d..48609fa66862c 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -350,7 +350,7 @@ class LoopVectorizationLegality {
/// reductions found in the loop.
bool isInvariantStoreOfReduction(StoreInst *SI);
- LoadInst *tryToFindDyanmicBoundLoadCandidate(Loop *L, AAResults &AA);
+ LoadInst *tryToFindDynamicBoundLoadCandidate(Loop *L, AAResults &AA);
/// Returns True if given address is invariant and is used to store recurrent
/// expression
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 9bb357695066f..4637dc0075af7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -703,7 +703,7 @@ static bool isLoadSpeculativelyHoistable(Loop *L, LoadInst *LoadCandidate,
}
LoadInst *
-LoopVectorizationLegality::tryToFindDyanmicBoundLoadCandidate(Loop *L,
+LoopVectorizationLegality::tryToFindDynamicBoundLoadCandidate(Loop *L,
AAResults &AA) {
if (!L->isInnermost())
return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9072fcbc6b816..da7dbe352a1df 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8163,8 +8163,10 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi.eraseFromParent();
}
-// Is profitable to generate runtime check dominated version of the loop? which
-// has speculatively hoisted the loop bound load.
+// Profitability stub for speculative bound-load versioning. Currently always
+// returns true. Intended to be progressively refined with cost-model
+// heuristics (e.g. minimum trip count, vectorization width threshold) to rule
+// out obvious performance regressions before committing to the IR transform.
static bool isSpeculativeBoundVersioningProfitable() { return true; }
bool LoopVectorizePass::processLoop(Loop *L) {
@@ -8241,7 +8243,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// again. 3) If the versioned loop is generated successfully, re-try
// vectorization on the versioned loop.
if (LoadInst *BoundLoad =
- LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
+ LVL.tryToFindDynamicBoundLoadCandidate(L, *AA)) {
if (isSpeculativeBoundVersioningProfitable())
if (Loop *Cand = versionLoopForInvariantBoundLoad(L, BoundLoad, *DT,
*LI, *AA, PSE, AC)) {
>From 676cf129167defd5524cbe0037150ae52b0036b1 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Wed, 20 May 2026 08:03:48 -0700
Subject: [PATCH 11/15] clang-format
---
llvm/lib/Transforms/Utils/LoopVersioning.cpp | 12 ++++++------
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index a0fdd1110b2bc..d2aeb81198c02 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -374,8 +374,8 @@ Loop *llvm::versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
// Clone the loop including its preheader.
ValueToValueMapTy VMap;
SmallVector<BasicBlock *, 4> NewBlocks;
- Loop *VerLoop = cloneLoopWithPreheader(PreHeader, RtCheckBB, L, VMap,
- ".lver", &LI, &DT, NewBlocks);
+ Loop *VerLoop = cloneLoopWithPreheader(PreHeader, RtCheckBB, L, VMap, ".lver",
+ &LI, &DT, NewBlocks);
remapInstructionsInBlocks(NewBlocks, VMap);
BasicBlock *VerPreHeader = cast<BasicBlock>(VMap[PreHeader]);
@@ -437,9 +437,8 @@ Loop *llvm::versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
}
if (ExitUses.empty())
continue;
- PHINode *MergePhi = PHINode::Create(I.getType(), 2,
- I.getName() + ".merge",
- OrigExitBlock->begin());
+ PHINode *MergePhi = PHINode::Create(
+ I.getType(), 2, I.getName() + ".merge", OrigExitBlock->begin());
MergePhi->addIncoming(&I, NewExitBlock);
MergePhi->addIncoming(CloneI, VerNewExitBlock);
for (Use *U : ExitUses)
@@ -516,7 +515,8 @@ Loop *llvm::versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
if (!RtCheckCond)
RtCheckCond = ConstantInt::getFalse(Ctx);
- // Branch: If conflict is detected then take the original loop path, else take the versioned loop path.
+ // Branch: If conflict is detected then take the original loop path, else take
+ // the versioned loop path.
Builder.CreateCondBr(RtCheckCond, PreHeader, VerPreHeader);
DT.insertEdge(RtCheckBB, PreHeader);
DT.insertEdge(RtCheckBB, VerPreHeader);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index da7dbe352a1df..bb7f90084d154 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8245,8 +8245,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (LoadInst *BoundLoad =
LVL.tryToFindDynamicBoundLoadCandidate(L, *AA)) {
if (isSpeculativeBoundVersioningProfitable())
- if (Loop *Cand = versionLoopForInvariantBoundLoad(L, BoundLoad, *DT,
- *LI, *AA, PSE, AC)) {
+ if (Loop *Cand = versionLoopForInvariantBoundLoad(
+ L, BoundLoad, *DT, *LI, *AA, PSE, AC)) {
SE->forgetLoop(L);
SE->forgetLoop(Cand);
return processLoop(Cand);
>From 36c3e4aba2311797f2d56ccf01be2af0e89c8c86 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Mon, 25 May 2026 03:59:51 -0700
Subject: [PATCH 12/15] useful profitablity stub and disable by default
---
.../Transforms/Vectorize/LoopVectorize.cpp | 87 ++++++++++++-------
.../dynamic-bound-array-element.ll | 2 +-
.../dynamic-bound-array-range-ld.ll | 2 +-
.../dynamic-bound-chained-loads.ll | 2 +-
.../LoopVectorize/dynamic-bound-fcmp-exit.ll | 2 +-
.../dynamic-bound-loop-varying.ll | 2 +-
.../LoopVectorize/dynamic-bound-n-stores.ll | 2 +-
.../LoopVectorize/dynamic-bound-simple.ll | 2 +-
.../dynamic-bound-unsafe-calls.ll | 2 +-
9 files changed, 63 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb7f90084d154..0fe083dc49004 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -374,6 +374,12 @@ static cl::opt<bool> EnableEarlyExitVectorization(
cl::desc(
"Enable vectorization of early exit loops with uncountable exits."));
+static cl::opt<bool> EnableInvariantBoundVersioning(
+ "enable-invariant-bound-versioning", cl::init(false), cl::Hidden,
+ cl::desc("Enable speculative hoisting of loop bound loads via runtime "
+ "versioning to allow vectorization of otherwise uncountable "
+ "loops."));
+
// Likelyhood of bypassing the vectorized loop because there are zero trips left
// after prolog. See `emitIterationCountCheck`.
static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
@@ -8163,11 +8169,25 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi.eraseFromParent();
}
-// Profitability stub for speculative bound-load versioning. Currently always
-// returns true. Intended to be progressively refined with cost-model
-// heuristics (e.g. minimum trip count, vectorization width threshold) to rule
-// out obvious performance regressions before committing to the IR transform.
-static bool isSpeculativeBoundVersioningProfitable() { return true; }
+// Returns true if it is profitable to emit a runtime-check-dominated versioned
+// loop with the bound load speculatively hoisted.
+//
+// The check uses a conservative heuristic that can be progressively refined:
+//
+// RTC count threshold: if LAA already requires more runtime pointer checks
+// than VectorizeMemoryCheckThreshold the additional ivbound checks would
+// push compile-time and runtime overhead too high.
+// Profile-guided or block-frequency-based heuristics can be added here later.
+static bool isSpeculativeBoundVersioningProfitable(const LoopAccessInfo &LAI) {
+ if (LAI.getNumRuntimePointerChecks() > VectorizeMemoryCheckThreshold) {
+ LLVM_DEBUG(dbgs() << "LV: Skipping speculative bound versioning: too many "
+ "existing runtime checks ("
+ << LAI.getNumRuntimePointerChecks() << ")\n");
+ return false;
+ }
+
+ return true;
+}
bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&
@@ -8221,36 +8241,39 @@ bool LoopVectorizePass::processLoop(Loop *L) {
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
- bool VisitedSpeculativeHoist = false;
- if (BasicBlock *Latch = L->getLoopLatch()) {
- if (MDNode *LoopMD =
- Latch->getTerminator()->getMetadata(LLVMContext::MD_loop)) {
- for (unsigned i = 1, e = LoopMD->getNumOperands(); i < e; ++i) {
- if (auto *Op = dyn_cast<MDNode>(LoopMD->getOperand(i)))
- if (Op->getNumOperands() > 0)
- if (auto *S = dyn_cast<MDString>(Op->getOperand(0)))
- if (S->getString() ==
- "llvm.loop.speculative.bound.hoist.versioned")
- VisitedSpeculativeHoist = true;
+ if (EnableInvariantBoundVersioning) {
+ bool VisitedSpeculativeHoist = false;
+ if (BasicBlock *Latch = L->getLoopLatch()) {
+ if (MDNode *LoopMD =
+ Latch->getTerminator()->getMetadata(LLVMContext::MD_loop)) {
+ for (unsigned i = 1, e = LoopMD->getNumOperands(); i < e; ++i) {
+ if (auto *Op = dyn_cast<MDNode>(LoopMD->getOperand(i)))
+ if (Op->getNumOperands() > 0)
+ if (auto *S = dyn_cast<MDString>(Op->getOperand(0)))
+ if (S->getString() ==
+ "llvm.loop.speculative.bound.hoist.versioned")
+ VisitedSpeculativeHoist = true;
+ }
}
}
- }
- if (!VisitedSpeculativeHoist) {
- // 1) Check if the loop qualifies for loop invariant bound load.
- // 2) Generate the versioned loop with runtime checks using the dynamic
- // loop bound load. Return this versioned loop to attempt vectorization
- // again. 3) If the versioned loop is generated successfully, re-try
- // vectorization on the versioned loop.
- if (LoadInst *BoundLoad =
- LVL.tryToFindDynamicBoundLoadCandidate(L, *AA)) {
- if (isSpeculativeBoundVersioningProfitable())
- if (Loop *Cand = versionLoopForInvariantBoundLoad(
- L, BoundLoad, *DT, *LI, *AA, PSE, AC)) {
- SE->forgetLoop(L);
- SE->forgetLoop(Cand);
- return processLoop(Cand);
- }
+ if (!VisitedSpeculativeHoist) {
+ // 1) Check if the loop qualifies for loop invariant bound load.
+ // 2) Generate the versioned loop with runtime checks using the dynamic
+ // loop bound load. Return this versioned loop to attempt vectorization
+ // again. 3) If the versioned loop is generated successfully, re-try
+ // vectorization on the versioned loop.
+ if (LoadInst *BoundLoad =
+ LVL.tryToFindDynamicBoundLoadCandidate(L, *AA)) {
+ const LoopAccessInfo &PreLAI = LAIs->getInfo(*L);
+ if (isSpeculativeBoundVersioningProfitable(PreLAI))
+ if (Loop *Cand = versionLoopForInvariantBoundLoad(
+ L, BoundLoad, *DT, *LI, *AA, PSE, AC)) {
+ SE->forgetLoop(L);
+ SE->forgetLoop(Cand);
+ return processLoop(Cand);
+ }
+ }
}
}
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
index 0c36ec330a212..95b87c0b17c5c 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+; RUN: opt < %s -S -passes='loop-vectorize' -enable-invariant-bound-versioning=true | FileCheck %s
; Not safe to hoist Speculatively.
; void foo(int *A, int *B, int *C, int *Len) {
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
index 0c36ec330a212..95b87c0b17c5c 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+; RUN: opt < %s -S -passes='loop-vectorize' -enable-invariant-bound-versioning=true | FileCheck %s
; Not safe to hoist Speculatively.
; void foo(int *A, int *B, int *C, int *Len) {
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
index ffe3d2511b71b..b56f7347f63b0 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+; RUN: opt < %s -S -passes='loop-vectorize' -enable-invariant-bound-versioning=true | FileCheck %s
;
; Negative test: the upper bound is **PtrLen (a chained / indirect load).
; The worklist correctly traces both loads, but LoopAccessAnalysis cannot
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
index 915d728090a8e..cb142b212a07a 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+; RUN: opt < %s -S -passes='loop-vectorize' -enable-invariant-bound-versioning=true | FileCheck %s
;
; LoopVectorize speculative load hoist shouldn't kick in as SCEV cannot compute the trip count of a float-bounded loop, so the
; pass enters the dynamic-bound path. It then rejects because the
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
index bdb4ae8b96b5f..c0a0c620948ca 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+; RUN: opt < %s -S -passes='loop-vectorize' -enable-invariant-bound-versioning=true | FileCheck %s
;
; LoopVectorize speculative load hoist shouldn't kick in at all if the bound is Len[i], which depends on the
; induction variable hence not invariant. The dependency chain traces back to a loop header
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
index f3cb82b7497bd..63a554c91d11f 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+; RUN: opt < %s -S -passes='loop-vectorize' -enable-invariant-bound-versioning=true | FileCheck %s
;
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
target triple = "aarch64"
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
index e5127696ed112..79a4ac0c6de6a 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -enable-invariant-bound-versioning -mtriple=aarch64-unknown-linux-gnu -S %s | FileCheck %s
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
index 72468a2fe233e..53221238858d9 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+; RUN: opt < %s -S -passes='loop-vectorize' -enable-invariant-bound-versioning=true | FileCheck %s
;
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
>From e7b65f2bbc6d8396bf85ca422c757ad15e79b015 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Wed, 27 May 2026 08:07:32 -0700
Subject: [PATCH 13/15] early cost model changes
---
.../Transforms/Vectorize/LoopVectorize.cpp | 52 ++++++++++++++++---
1 file changed, 44 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0fe083dc49004..20743f3262a7b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8172,13 +8172,15 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
// Returns true if it is profitable to emit a runtime-check-dominated versioned
// loop with the bound load speculatively hoisted.
//
-// The check uses a conservative heuristic that can be progressively refined:
-//
-// RTC count threshold: if LAA already requires more runtime pointer checks
-// than VectorizeMemoryCheckThreshold the additional ivbound checks would
-// push compile-time and runtime overhead too high.
-// Profile-guided or block-frequency-based heuristics can be added here later.
-static bool isSpeculativeBoundVersioningProfitable(const LoopAccessInfo &LAI) {
+// Uses the LV cost model to estimate the cost of the runtime checks that
+// versioning would add, and applies the MinTC2 heuristic from
+// isOutsideLoopWorkProfitable to reject cases where the expected trip count
+// is too small to amortize the check overhead.
+static bool isSpeculativeBoundVersioningProfitable(
+ const LoopAccessInfo &LAI, Loop *L, PredicatedScalarEvolution &PSE,
+ DominatorTree *DT, LoopInfo *LI, TargetTransformInfo *TTI,
+ OptimizationRemarkEmitter *ORE) {
+ // Hard cap: too many existing checks means too much overhead.
if (LAI.getNumRuntimePointerChecks() > VectorizeMemoryCheckThreshold) {
LLVM_DEBUG(dbgs() << "LV: Skipping speculative bound versioning: too many "
"existing runtime checks ("
@@ -8186,6 +8188,39 @@ static bool isSpeculativeBoundVersioningProfitable(const LoopAccessInfo &LAI) {
return false;
}
+ // Estimate the actual cost of the runtime checks using GeneratedRTChecks,
+ // mirroring what processLoop does on the versioned clone but evaluated on
+ // the original loop before any IR changes. Use scalar VF=1 / IC=1 since
+ // we are only interested in the check overhead, not the vector body cost.
+ GeneratedRTChecks Checks(PSE, DT, LI, TTI, TTI::TCK_RecipThroughput);
+ Checks.create(L, LAI, LAI.getPSE().getPredicate(), ElementCount::getFixed(1),
+ /*IC=*/1, *ORE);
+ InstructionCost RtC = Checks.getCost();
+ if (!RtC.isValid()) {
+ LLVM_DEBUG(dbgs() << "LV: Skipping speculative bound versioning: "
+ "runtime check cost is invalid/too high\n");
+ return false;
+ }
+
+ // MinTC2 heuristic (mirrors isOutsideLoopWorkProfitable):
+ // require RtC * 10 / ScalarCostProxy < TC
+ // We use VectorizeMemoryCheckThreshold as a conservative scalar-cost proxy
+ // since the full LoopVectorizationCostModel has not been built yet.
+ // For the target loops (uncountable bound load), getSmallBestKnownTC
+ // typically returns nothing, making this a no-op in the common case.
+ // It correctly rejects loops whose trip count is statically known to be
+ // too small to amortize the check overhead.
+ if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
+ uint64_t MinTC2 =
+ divideCeil(RtC.getValue() * 10, VectorizeMemoryCheckThreshold);
+ if (ExpectedTC->isFixed() && ExpectedTC->getFixedValue() < MinTC2) {
+ LLVM_DEBUG(dbgs() << "LV: Skipping speculative bound versioning: "
+ "expected TC ("
+ << *ExpectedTC << ") < MinTC (" << MinTC2 << ")\n");
+ return false;
+ }
+ }
+
return true;
}
@@ -8266,7 +8301,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (LoadInst *BoundLoad =
LVL.tryToFindDynamicBoundLoadCandidate(L, *AA)) {
const LoopAccessInfo &PreLAI = LAIs->getInfo(*L);
- if (isSpeculativeBoundVersioningProfitable(PreLAI))
+ if (isSpeculativeBoundVersioningProfitable(PreLAI, L, PSE, DT, LI,
+ TTI, ORE))
if (Loop *Cand = versionLoopForInvariantBoundLoad(
L, BoundLoad, *DT, *LI, *AA, PSE, AC)) {
SE->forgetLoop(L);
>From db360a520bef48213cde4662a65954aa2cc6d4a9 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Wed, 27 May 2026 08:51:04 -0700
Subject: [PATCH 14/15] finer cost changes and load elim beneift cost
---
.../Transforms/Vectorize/LoopVectorize.cpp | 63 +++++++++++++++----
1 file changed, 52 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 20743f3262a7b..0978ee9d41cf9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8173,13 +8173,31 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
// loop with the bound load speculatively hoisted.
//
// Uses the LV cost model to estimate the cost of the runtime checks that
-// versioning would add, and applies the MinTC2 heuristic from
-// isOutsideLoopWorkProfitable to reject cases where the expected trip count
-// is too small to amortize the check overhead.
+// versioning would add, and applies two minimum-trip-count thresholds:
+//
+// MinTC2 (mirrors isOutsideLoopWorkProfitable):
+// RtC * 10 / VectorizeMemoryCheckThreshold < TC
+// Guards against loops where the RTC overhead is a large fraction of the
+// total scalar cost.
+//
+// MinTC_load (load-elimination benefit):
+// ceil(RtC / LoadSaving) < TC
+// The per-iteration saving from eliminating the bound load in the versioned
+// clone. This directly models the benefit of the transform: a cheap load
+// requires more iterations to amortize the RTC cost; an expensive load
+// pays back quickly.
+//
+// Versioning is rejected when the known trip count is below
+// max(MinTC2, MinTC_load).
static bool isSpeculativeBoundVersioningProfitable(
- const LoopAccessInfo &LAI, Loop *L, PredicatedScalarEvolution &PSE,
- DominatorTree *DT, LoopInfo *LI, TargetTransformInfo *TTI,
- OptimizationRemarkEmitter *ORE) {
+ const LoopAccessInfo &LAI, Loop *L, LoadInst *BoundLoad,
+ PredicatedScalarEvolution &PSE, DominatorTree *DT, LoopInfo *LI,
+ TargetTransformInfo *TTI, OptimizationRemarkEmitter *ORE) {
+
+ // We expect this function to only be called when we have identified a candidate load for speculative bound versioning, Should this be an assertion instead?
+ if (!BoundLoad)
+ return false;
+
// Hard cap: too many existing checks means too much overhead.
if (LAI.getNumRuntimePointerChecks() > VectorizeMemoryCheckThreshold) {
LLVM_DEBUG(dbgs() << "LV: Skipping speculative bound versioning: too many "
@@ -8202,6 +8220,27 @@ static bool isSpeculativeBoundVersioningProfitable(
return false;
}
+ // Compute the per-iteration saving from eliminating the bound load in the
+ // versioned clone. The load executes once per scalar iteration; hoisting it
+ // out of the loop saves this cost on every iteration of the fast path.
+ Type *LoadTy = BoundLoad->getType();
+ Align LoadAlign = BoundLoad->getAlign();
+ unsigned AS = BoundLoad->getPointerAddressSpace();
+ TTI::OperandValueInfo OpInfo =
+ TTI::getOperandInfo(BoundLoad->getPointerOperand());
+ InstructionCost LoadSaving =
+ TTI->getMemoryOpCost(Instruction::Load, LoadTy, LoadAlign, AS,
+ TTI::TCK_RecipThroughput, OpInfo, BoundLoad);
+ if (!LoadSaving.isValid())
+ return false;
+
+ // MinTC_load: minimum trip count for the load saving alone to amortize RtC.
+ // LoadSaving * TC >= RtC => TC >= ceil(RtC / LoadSaving)
+ // When LoadSaving is zero this threshold is left at 0 (no-op).
+ uint64_t MinTC_load = 0;
+ if (LoadSaving.getValue() > 0)
+ MinTC_load = divideCeil(RtC.getValue(), LoadSaving.getValue());
+
// MinTC2 heuristic (mirrors isOutsideLoopWorkProfitable):
// require RtC * 10 / ScalarCostProxy < TC
// We use VectorizeMemoryCheckThreshold as a conservative scalar-cost proxy
@@ -8213,10 +8252,12 @@ static bool isSpeculativeBoundVersioningProfitable(
if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
uint64_t MinTC2 =
divideCeil(RtC.getValue() * 10, VectorizeMemoryCheckThreshold);
- if (ExpectedTC->isFixed() && ExpectedTC->getFixedValue() < MinTC2) {
+ uint64_t MinTC = std::max(MinTC2, MinTC_load);
+ if (ExpectedTC->isFixed() && ExpectedTC->getFixedValue() < MinTC) {
LLVM_DEBUG(dbgs() << "LV: Skipping speculative bound versioning: "
- "expected TC ("
- << *ExpectedTC << ") < MinTC (" << MinTC2 << ")\n");
+ "expected TC (" << *ExpectedTC << ") < MinTC ("
+ << MinTC << ") [RtC=" << RtC
+ << ", LoadSaving=" << LoadSaving << "]\n");
return false;
}
}
@@ -8301,8 +8342,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (LoadInst *BoundLoad =
LVL.tryToFindDynamicBoundLoadCandidate(L, *AA)) {
const LoopAccessInfo &PreLAI = LAIs->getInfo(*L);
- if (isSpeculativeBoundVersioningProfitable(PreLAI, L, PSE, DT, LI,
- TTI, ORE))
+ if (isSpeculativeBoundVersioningProfitable(PreLAI, L, BoundLoad, PSE,
+ DT, LI, TTI, ORE))
if (Loop *Cand = versionLoopForInvariantBoundLoad(
L, BoundLoad, *DT, *LI, *AA, PSE, AC)) {
SE->forgetLoop(L);
>From 8be59053745bb72752501827011093ee306cfdd5 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Tue, 9 Jun 2026 01:07:14 -0700
Subject: [PATCH 15/15] using LoopVersioning for transform instead of manual
versioning
---
llvm/lib/Transforms/Utils/LoopVersioning.cpp | 244 +++++------------
.../Transforms/Vectorize/LoopVectorize.cpp | 17 +-
.../dynamic-bound-array-element.ll | 120 ++++----
.../dynamic-bound-array-range-ld.ll | 120 ++++----
.../LoopVectorize/dynamic-bound-n-stores.ll | 258 ++++++++++--------
.../LoopVectorize/dynamic-bound-simple.ll | 110 ++++----
6 files changed, 403 insertions(+), 466 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index d2aeb81198c02..1715f958b9fdd 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -294,21 +294,19 @@ Loop *llvm::versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
if (!DT.properlyDominates(BoundPtrI->getParent(), PreHeader))
return nullptr;
- // All the relevant base pointers of participating stores should dominate both
- // loops.
+ // Run LAA on L to obtain the pointer checks and SCEV predicates we need.
ScalarEvolution &SE = *PSE.getSE();
LoopAccessInfoManager PreFlightLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
const LoopAccessInfo &PreLAI = PreFlightLAIs.getInfo(*L);
const RuntimePointerChecking &RtPtrChecking =
*PreLAI.getRuntimePointerChecking();
- // If LAA could not compute complete pointer bounds (e.g. a gather/scatter
- // access with an unknown indirect index), we cannot reliably determine the
- // write ranges needed for the ivbound safety check. Bail out rather than
- // proceeding with an incomplete or missing write-range check.
+ // If LAA could not compute complete pointer bounds bail out.
if (!PreLAI.canVectorizeMemory())
return nullptr;
+ // All loop-invariant base pointers referenced by the checks must dominate
+ // the preheader so they are available in the check block.
for (const auto &Check : RtPtrChecking.getChecks()) {
for (const RuntimeCheckingPtrGroup *Group : {Check.first, Check.second}) {
for (unsigned Idx : Group->Members) {
@@ -336,135 +334,35 @@ Loop *llvm::versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
}
}
- Function *ParentF = PreHeader->getParent();
- LLVMContext &Ctx = ParentF->getContext();
- BasicBlock *OrigExitBlock = L->getUniqueExitBlock();
-
- BasicBlock *RtCheckBB =
- BasicBlock::Create(Ctx, "ivbound.rtcheck", ParentF, PreHeader);
-
- // Temporary terminator so SCEVExpander has a valid insertion point.
- IRBuilder<> TmpBuilder(RtCheckBB);
- Instruction *TmpTerm = TmpBuilder.CreateUnreachable();
-
- // Re-wire the CFG: redirect predecessors of the preheader to RtCheckBB.
- SmallVector<BasicBlock *, 4> PreHeaderPreds(predecessors(PreHeader));
- for (auto *Pred : PreHeaderPreds)
- Pred->getTerminator()->replaceUsesOfWith(PreHeader, RtCheckBB);
-
- BasicBlock *OldPreheadIdom = DT.getNode(PreHeader)->getIDom()->getBlock();
- DT.addNewBlock(RtCheckBB, OldPreheadIdom);
- DT.changeImmediateDominator(PreHeader, RtCheckBB);
-
- // Fix preheader PHIs: move their incoming values into new PHIs in RtCheckBB.
- for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
- PHINode *RtcPN =
- PHINode::Create(PN.getType(), PreHeaderPreds.size(),
- PN.getName() + ".rtcphi", RtCheckBB->begin());
- for (unsigned i = 0; i < PN.getNumIncomingValues(); i++) {
- BasicBlock *IBB = PN.getIncomingBlock(i);
- assert(is_contained(PreHeaderPreds, IBB) &&
- "Incoming basic block not part of predecessors list!");
- RtcPN->addIncoming(PN.getIncomingValue(i), IBB);
- }
- PN.replaceAllUsesWith(RtcPN);
- PN.eraseFromParent();
- }
-
- // Clone the loop including its preheader.
- ValueToValueMapTy VMap;
- SmallVector<BasicBlock *, 4> NewBlocks;
- Loop *VerLoop = cloneLoopWithPreheader(PreHeader, RtCheckBB, L, VMap, ".lver",
- &LI, &DT, NewBlocks);
- remapInstructionsInBlocks(NewBlocks, VMap);
-
- BasicBlock *VerPreHeader = cast<BasicBlock>(VMap[PreHeader]);
- BasicBlock *VerLatch = cast<BasicBlock>(VMap[L->getLoopLatch()]);
-
- if (Loop *ParLoop = L->getParentLoop())
- ParLoop->addBasicBlockToLoop(RtCheckBB, LI);
-
- // Clear any existing loop metadata on the versioned latch; it will be
- // replaced below with the speculative-hoist marker.
- VerLatch->getTerminator()->setMetadata(LLVMContext::MD_loop, nullptr);
-
- // Speculatively hoist the bound load into RtCheckBB so the versioned loop
- // sees a loop-invariant trip count.
+ // Use LoopVersioning to perform the CFG split, clone, check generation,
+ // PHI fixup, and formDedicatedExitBlocks. After versionLoop():
+ // - L is the fast path (versioned, no-alias assumed)
+ // - NonVersionedLoop is the fallback clone (original semantics)
+ // The check block is L->getLoopPreheader() before the split, which becomes
+ // the lver.check block after versionLoop() renames it.
+ LoopVersioning LVer(PreLAI, RtPtrChecking.getChecks(), L, &LI, &DT, &SE);
+ LVer.versionLoop();
+
+ // The check block is the block that now precedes both loop preheaders.
+ // After versionLoop() it is the block that was originally L's preheader
+ // (renamed to lver.check), whose terminator is the conditional branch.
+ BasicBlock *CheckBB = L->getLoopPreheader()->getSinglePredecessor();
+ assert(CheckBB && "Expected a single predecessor check block after versioning");
+ Instruction *CheckTerm = CheckBB->getTerminator();
+
+ // Speculatively hoist BoundLoad into the check block so the fast-path loop
+ // (L) sees a loop-invariant trip count. The fallback clone already has the
+ // original bound load in place — no changes needed there.
LoadInst *HoistLoad = cast<LoadInst>(BoundLoad->clone());
HoistLoad->setName(BoundLoad->getName() + ".speculatively.hoisted");
- HoistLoad->insertBefore(TmpTerm->getIterator());
- Instruction *ClonedBoundLoad = cast<Instruction>(VMap[BoundLoad]);
- ClonedBoundLoad->replaceAllUsesWith(HoistLoad);
- ClonedBoundLoad->eraseFromParent();
-
- // Fix LCSSA PHIs: add incoming edges from the versioned latch.
- SmallVector<BasicBlock *, 4> ExitBlocks;
- L->getExitBlocks(ExitBlocks);
- for (BasicBlock *ExBB : ExitBlocks) {
- DT.changeImmediateDominator(ExBB, RtCheckBB);
- for (PHINode &Phi : ExBB->phis()) {
- int Idx = Phi.getBasicBlockIndex(L->getLoopLatch());
- if (Idx < 0)
- continue;
- Value *OrigVal = Phi.getIncomingValue(Idx);
- Value *NewVal =
- VMap.count(OrigVal) ? cast<Value>(VMap[OrigVal]) : OrigVal;
- Phi.addIncoming(NewVal, VerLatch);
- }
- }
-
- formDedicatedExitBlocks(L, &DT, &LI, nullptr, true);
- formDedicatedExitBlocks(VerLoop, &DT, &LI, nullptr, true);
-
- // Fix any preheader definitions used outside both loops by inserting a merge
- // PHI in the original exit block.
- BasicBlock *NewExitBlock = L->getUniqueExitBlock();
- BasicBlock *VerNewExitBlock = VerLoop->getUniqueExitBlock();
- if (NewExitBlock && VerNewExitBlock) {
- for (Instruction &I : *PreHeader) {
- if (I.isTerminator())
- continue;
- Value *CloneI = VMap.count(&I) ? VMap[&I] : nullptr;
- if (!CloneI)
- continue;
- SmallVector<Use *, 4> ExitUses;
- for (Use &U : I.uses()) {
- auto *UI = dyn_cast<Instruction>(U.getUser());
- BasicBlock *UserBlock = UI->getParent();
- if (UI && UserBlock != PreHeader && !L->contains(UserBlock) &&
- !VerLoop->contains(UserBlock))
- ExitUses.push_back(&U);
- }
- if (ExitUses.empty())
- continue;
- PHINode *MergePhi = PHINode::Create(
- I.getType(), 2, I.getName() + ".merge", OrigExitBlock->begin());
- MergePhi->addIncoming(&I, NewExitBlock);
- MergePhi->addIncoming(CloneI, VerNewExitBlock);
- for (Use *U : ExitUses)
- U->set(MergePhi);
- }
- }
-
- // Generate runtime checks on the versioned loop using LAA.
- LoopAccessInfoManager NewLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
- const LoopAccessInfo &VerLAI = NewLAIs.getInfo(*VerLoop);
-
- SCEVExpander MemExp(*VerLAI.getRuntimePointerChecking()->getSE(),
- "ivbound.mem.check");
- Value *MemRTC = nullptr;
- if (VerLAI.getRuntimePointerChecking()->Need)
- MemRTC = addRuntimeChecks(TmpTerm, VerLoop,
- VerLAI.getRuntimePointerChecking()->getChecks(),
- MemExp);
-
- SCEVExpander SCEVEx(SE, "ivbound.scev.check");
- const SCEVPredicate &Preds = VerLAI.getPSE().getPredicate();
- Value *SCEVrtc = SCEVEx.expandCodeForPredicate(&Preds, TmpTerm);
-
- // Check whether BoundPtrOl falls within any write group's [Low, High) byte
- // range at runtime. If it does, the speculative hoist is unsafe (a store
- // in the loop would overwrite *BoundPtr), so fall back to the scalar loop.
+ HoistLoad->insertBefore(CheckTerm->getIterator());
+ BoundLoad->replaceAllUsesWith(HoistLoad);
+
+ // Add the write-range check: verify at runtime that BoundPtrOl does not
+ // fall within any write group's [Low, High) byte range. If it does, the
+ // speculative hoist is unsafe (a store in the loop would overwrite
+ // *BoundPtr), so we must fall back to the original loop.
+ LLVMContext &Ctx = CheckBB->getContext();
SCEVExpander PreExp(SE, "ivbound.wg.check");
Value *BoundInWriteRange = nullptr;
for (const RuntimeCheckingPtrGroup &Group : RtPtrChecking.CheckingGroups) {
@@ -474,14 +372,14 @@ Loop *llvm::versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
if (!HasWrite)
continue;
Type *PtrArithTy = PointerType::get(Ctx, Group.AddressSpace);
- Value *GroupLow = PreExp.expandCodeFor(Group.Low, PtrArithTy, TmpTerm);
- Value *GroupHigh = PreExp.expandCodeFor(Group.High, PtrArithTy, TmpTerm);
+ Value *GroupLow = PreExp.expandCodeFor(Group.Low, PtrArithTy, CheckTerm);
+ Value *GroupHigh = PreExp.expandCodeFor(Group.High, PtrArithTy, CheckTerm);
if (Group.NeedsFreeze) {
- IRBuilder<> FreezeB(TmpTerm);
+ IRBuilder<> FreezeB(CheckTerm);
GroupLow = FreezeB.CreateFreeze(GroupLow, "ivbound.wg.low.fr");
GroupHigh = FreezeB.CreateFreeze(GroupHigh, "ivbound.wg.high.fr");
}
- IRBuilder<> Builder(TmpTerm);
+ IRBuilder<> Builder(CheckTerm);
Value *Cmp0 =
Builder.CreateICmpULE(GroupLow, BoundPtrOl, "ivbound.wg.low.chk");
Value *Cmp1 =
@@ -489,60 +387,42 @@ Loop *llvm::versionLoopForInvariantBoundLoad(Loop *L, LoadInst *BoundLoad,
Value *InRange = Builder.CreateAnd(Cmp0, Cmp1, "ivbound.in.write.range");
BoundInWriteRange =
BoundInWriteRange
- ? Builder.CreateOr(BoundInWriteRange, InRange, "ivbound.wg.rdx")
+ ? IRBuilder<>(CheckTerm).CreateOr(BoundInWriteRange, InRange,
+ "ivbound.wg.rdx")
: InRange;
}
- // Combine all runtime check conditions: alias checks, SCEV predicates, and
- // the bound-pointer write-range check.
- Value *RtCheckCond = nullptr;
- for (Value *Cond : {MemRTC, SCEVrtc, BoundInWriteRange}) {
- if (!Cond)
- continue;
- if (!RtCheckCond) {
- RtCheckCond = Cond;
- } else {
- IRBuilder<> Builder(TmpTerm);
- RtCheckCond = Builder.CreateOr(RtCheckCond, Cond, "ivbound.safe");
- }
+ // Fold the write-range check into the existing conditional branch condition.
+ // versionLoop() emits: br (lver.safe), fallback, fastpath
+ // We want: br (lver.safe | BoundInWriteRange), fallback, fastpath
+ if (BoundInWriteRange) {
+ auto *BI = cast<CondBrInst>(CheckTerm);
+ IRBuilder<> Builder(CheckBB, BI->getIterator());
+ Value *NewCond = Builder.CreateOr(BI->getCondition(), BoundInWriteRange,
+ "ivbound.safe");
+ BI->setCondition(NewCond);
}
- TmpTerm->eraseFromParent();
- IRBuilder<> Builder(RtCheckBB);
-
- // If no runtime checks are needed the versioned loop is unconditionally safe;
- // use constant false so the original (fallback) path is never taken.
- if (!RtCheckCond)
- RtCheckCond = ConstantInt::getFalse(Ctx);
-
- // Branch: If conflict is detected then take the original loop path, else take
- // the versioned loop path.
- Builder.CreateCondBr(RtCheckCond, PreHeader, VerPreHeader);
- DT.insertEdge(RtCheckBB, PreHeader);
- DT.insertEdge(RtCheckBB, VerPreHeader);
-
- assert(VerLatch &&
- "Versioned loop must have a single latch; loop not in canonical form");
-
- // Emit metadata on the versioned latch to prevent repeated re-versioning of
- // the same loop if processLoop is called again on the versioned clone.
- Instruction *VerLatchTerm = VerLatch->getTerminator();
+ // Emit metadata on L's latch to prevent re-versioning if processLoop is
+ // called again on the fast-path loop.
+ BasicBlock *Latch = L->getLoopLatch();
+ assert(Latch && "Loop must have a single latch");
+ Instruction *LatchTerm = Latch->getTerminator();
MDNode *Marker = MDNode::get(
Ctx, {(Metadata *)MDString::get(
- Ctx, "llvm.loop.speculative.bound.hoist.versioned")});
- MDNode *ExistingMD = VerLatchTerm->getMetadata(LLVMContext::MD_loop);
- SmallVector<Metadata *, 4> Mds;
- if (ExistingMD)
- Mds.append(ExistingMD->op_begin(), ExistingMD->op_end());
- else
- Mds.push_back(nullptr);
+ Ctx, "llvm.loop.speculative.bound.hoist.versioned")});
+ MDNode *ExistingMD = LatchTerm->getMetadata(LLVMContext::MD_loop);
+ SmallVector<Metadata *, 4> Mds = {nullptr};
+ if (ExistingMD) {
+ for (unsigned I = 1, E = ExistingMD->getNumOperands(); I < E; ++I)
+ Mds.push_back(ExistingMD->getOperand(I));
+ }
Mds.push_back(Marker);
- MDNode *NewLoopMD = MDNode::getDistinct(Ctx, Mds);
- if (!ExistingMD)
- NewLoopMD->replaceOperandWith(0, NewLoopMD);
- VerLatchTerm->setMetadata(LLVMContext::MD_loop, NewLoopMD);
+ MDNode *NewLoopMD = MDNode::get(Ctx, Mds);
+ NewLoopMD->replaceOperandWith(0, NewLoopMD);
+ LatchTerm->setMetadata(LLVMContext::MD_loop, NewLoopMD);
- return VerLoop;
+ return L;
}
namespace {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0978ee9d41cf9..db035712c33ca 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8189,15 +8189,24 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
//
// Versioning is rejected when the known trip count is below
// max(MinTC2, MinTC_load).
+// Returns true if it is profitable to emit a runtime-check-dominated versioned
+// loop with the bound load speculatively hoisted.
+//
+// Uses a conservative heuristic based on the number of existing runtime
+// pointer checks. The full LV cost model runs on the versioned clone inside
+// processLoop, so this is only a pre-screen to avoid obviously unprofitable
+// cases.
static bool isSpeculativeBoundVersioningProfitable(
const LoopAccessInfo &LAI, Loop *L, LoadInst *BoundLoad,
PredicatedScalarEvolution &PSE, DominatorTree *DT, LoopInfo *LI,
- TargetTransformInfo *TTI, OptimizationRemarkEmitter *ORE) {
-
- // We expect this function to only be called when we have identified a candidate load for speculative bound versioning, Should this be an assertion instead?
+ TargetTransformInfo *TTI,
+ OptimizationRemarkEmitter *ORE) {
+
+ // We expect this function to only be called when we have identified a
+ // candidate load for speculative bound versioning. Should this be an assert?
if (!BoundLoad)
return false;
-
+
// Hard cap: too many existing checks means too much overhead.
if (LAI.getNumRuntimePointerChecks() > VectorizeMemoryCheckThreshold) {
LLVM_DEBUG(dbgs() << "LV: Skipping speculative bound versioning: too many "
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
index 95b87c0b17c5c..9c9180cf2c0e0 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -82,46 +82,64 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK-LABEL: define dso_local void @_Z4foo2PiS_S_S_(
; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) local_unnamed_addr #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C9:%.*]] = ptrtoaddr ptr [[C]] to i64
-; CHECK-NEXT: [[B8:%.*]] = ptrtoaddr ptr [[B]] to i64
-; CHECK-NEXT: [[A7:%.*]] = ptrtoaddr ptr [[A]] to i64
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 8
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP11]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[IVBOUND_RTCHECK]]:
-; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 12
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[ARRAYIDX]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
-; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT: [[BOUND011:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND112:%.*]] = icmp ult ptr [[C]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT13:%.*]] = and i1 [[BOUND011]], [[BOUND112]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT13]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[ARRAYIDX]]
; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[ARRAYIDX]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
-; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], false
; CHECK-NEXT: [[IVBOUND_SAFE7:%.*]] = or i1 [[IVBOUND_SAFE]], [[IVBOUND_IN_WRITE_RANGE]]
; CHECK-NEXT: br i1 [[IVBOUND_SAFE7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
-; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY_LVER_ORIG:.*]]
+; CHECK: [[FOR_BODY_LVER_ORIG]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], %[[FOR_BODY_LVER_ORIG]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX1_LVER_ORIG:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX1_LVER_ORIG]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX3_LVER_ORIG:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX3_LVER_ORIG]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ADD_LVER_ORIG:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX5_LVER_ORIG:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: store i32 [[ADD_LVER_ORIG]], ptr [[ARRAYIDX5_LVER_ORIG]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 8
+; CHECK-NEXT: [[CMP_LVER_ORIG:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[TMP3]]
+; CHECK-NEXT: br i1 [[CMP_LVER_ORIG]], label %[[FOR_BODY_LVER_ORIG]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[A7]], [[B8]]
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 32
-; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[A7]], [[C9]]
-; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = icmp ult i64 [[TMP5]], 32
+; CHECK-NEXT: [[SCEVGEP16:%.*]] = getelementptr i8, ptr [[B]], i64 12
+; CHECK-NEXT: [[BOUND017:%.*]] = icmp ult ptr [[A]], [[SCEVGEP16]]
+; CHECK-NEXT: [[BOUND118:%.*]] = icmp ult ptr [[ARRAYIDX]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = and i1 [[BOUND017]], [[BOUND118]]
+; CHECK-NEXT: [[BOUND020:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND121:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = and i1 [[BOUND020]], [[BOUND121]]
; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK10]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[BOUND024:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND125:%.*]] = icmp ult ptr [[C]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT26:%.*]] = and i1 [[BOUND024]], [[BOUND125]]
+; CHECK-NEXT: [[CONFLICT_RDX27:%.*]] = or i1 [[CONFLICT_RDX11]], [[FOUND_CONFLICT26]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX27]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
@@ -130,43 +148,28 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META12:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META12]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META15:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META15]]
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD14]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD15]], [[WIDE_LOAD13]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP12]], i64 4
-; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META17:![0-9]+]], !noalias [[META19:![0-9]+]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META17]], !noalias [[META19]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
-; CHECK: [[FOR_BODY_LVER]]:
-; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX1_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1_LVER]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[ARRAYIDX3_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX3_LVER]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
-; CHECK-NEXT: [[ARRAYIDX5_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX5_LVER]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
-; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
@@ -175,7 +178,7 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV]]
@@ -184,10 +187,10 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP25:![0-9]+]]
;
entry:
%arrayidx = getelementptr inbounds nuw i8, ptr %B, i64 8
@@ -239,10 +242,19 @@ attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: r
; CHECK: [[META8]] = !{!"Simple C++ TBAA"}
; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]]}
; CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META12:![0-9]+]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
-; CHECK: [[META12]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
-; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META12]], [[META13]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META10]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META10]]}
+; CHECK: [[META12]] = !{[[META13:![0-9]+]]}
+; CHECK: [[META13]] = distinct !{[[META13]], [[META14:![0-9]+]]}
+; CHECK: [[META14]] = distinct !{[[META14]], !"LVerDomain"}
+; CHECK: [[META15]] = !{[[META16:![0-9]+]]}
+; CHECK: [[META16]] = distinct !{[[META16]], [[META14]]}
+; CHECK: [[META17]] = !{[[META18:![0-9]+]]}
+; CHECK: [[META18]] = distinct !{[[META18]], [[META14]]}
+; CHECK: [[META19]] = !{[[META20:![0-9]+]], [[META13]], [[META16]]}
+; CHECK: [[META20]] = distinct !{[[META20]], [[META14]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META10]], [[META22:![0-9]+]], [[META23:![0-9]+]], [[META24:![0-9]+]]}
+; CHECK: [[META22]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META23]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META24]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META10]], [[META22]], [[META23]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
index 95b87c0b17c5c..9c9180cf2c0e0 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-range-ld.ll
@@ -82,46 +82,64 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK-LABEL: define dso_local void @_Z4foo2PiS_S_S_(
; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) local_unnamed_addr #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C9:%.*]] = ptrtoaddr ptr [[C]] to i64
-; CHECK-NEXT: [[B8:%.*]] = ptrtoaddr ptr [[B]] to i64
-; CHECK-NEXT: [[A7:%.*]] = ptrtoaddr ptr [[A]] to i64
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 8
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP11]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[IVBOUND_RTCHECK]]:
-; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 12
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[ARRAYIDX]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
-; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT: [[BOUND011:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND112:%.*]] = icmp ult ptr [[C]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT13:%.*]] = and i1 [[BOUND011]], [[BOUND112]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT13]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[ARRAYIDX]]
; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[ARRAYIDX]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
-; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], false
; CHECK-NEXT: [[IVBOUND_SAFE7:%.*]] = or i1 [[IVBOUND_SAFE]], [[IVBOUND_IN_WRITE_RANGE]]
; CHECK-NEXT: br i1 [[IVBOUND_SAFE7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
-; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY_LVER_ORIG:.*]]
+; CHECK: [[FOR_BODY_LVER_ORIG]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], %[[FOR_BODY_LVER_ORIG]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX1_LVER_ORIG:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX1_LVER_ORIG]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX3_LVER_ORIG:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX3_LVER_ORIG]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[ADD_LVER_ORIG:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX5_LVER_ORIG:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: store i32 [[ADD_LVER_ORIG]], ptr [[ARRAYIDX5_LVER_ORIG]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 8
+; CHECK-NEXT: [[CMP_LVER_ORIG:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[TMP3]]
+; CHECK-NEXT: br i1 [[CMP_LVER_ORIG]], label %[[FOR_BODY_LVER_ORIG]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[A7]], [[B8]]
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 32
-; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[A7]], [[C9]]
-; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = icmp ult i64 [[TMP5]], 32
+; CHECK-NEXT: [[SCEVGEP16:%.*]] = getelementptr i8, ptr [[B]], i64 12
+; CHECK-NEXT: [[BOUND017:%.*]] = icmp ult ptr [[A]], [[SCEVGEP16]]
+; CHECK-NEXT: [[BOUND118:%.*]] = icmp ult ptr [[ARRAYIDX]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = and i1 [[BOUND017]], [[BOUND118]]
+; CHECK-NEXT: [[BOUND020:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND121:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = and i1 [[BOUND020]], [[BOUND121]]
; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK10]]
-; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[BOUND024:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND125:%.*]] = icmp ult ptr [[C]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT26:%.*]] = and i1 [[BOUND024]], [[BOUND125]]
+; CHECK-NEXT: [[CONFLICT_RDX27:%.*]] = or i1 [[CONFLICT_RDX11]], [[FOUND_CONFLICT26]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX27]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
@@ -130,43 +148,28 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META12:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META12]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META15:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META15]]
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD14]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD15]], [[WIDE_LOAD13]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP12]], i64 4
-; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META17:![0-9]+]], !noalias [[META19:![0-9]+]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[INT_TBAA5]], !alias.scope [[META17]], !noalias [[META19]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
-; CHECK: [[FOR_BODY_LVER]]:
-; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX1_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1_LVER]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[ARRAYIDX3_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX3_LVER]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
-; CHECK-NEXT: [[ARRAYIDX5_LVER:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX5_LVER]], align 4, !tbaa [[INT_TBAA5]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
-; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
@@ -175,7 +178,7 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV]]
@@ -184,10 +187,10 @@ define dso_local void @_Z4foo2PiS_S_S_(ptr noundef writeonly captures(none) %A,
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA5]]
; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP25:![0-9]+]]
;
entry:
%arrayidx = getelementptr inbounds nuw i8, ptr %B, i64 8
@@ -239,10 +242,19 @@ attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: r
; CHECK: [[META8]] = !{!"Simple C++ TBAA"}
; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]]}
; CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META12:![0-9]+]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
-; CHECK: [[META12]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
-; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META12]], [[META13]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META10]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META10]]}
+; CHECK: [[META12]] = !{[[META13:![0-9]+]]}
+; CHECK: [[META13]] = distinct !{[[META13]], [[META14:![0-9]+]]}
+; CHECK: [[META14]] = distinct !{[[META14]], !"LVerDomain"}
+; CHECK: [[META15]] = !{[[META16:![0-9]+]]}
+; CHECK: [[META16]] = distinct !{[[META16]], [[META14]]}
+; CHECK: [[META17]] = !{[[META18:![0-9]+]]}
+; CHECK: [[META18]] = distinct !{[[META18]], [[META14]]}
+; CHECK: [[META19]] = !{[[META20:![0-9]+]], [[META13]], [[META16]]}
+; CHECK: [[META20]] = distinct !{[[META20]], [[META14]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META10]], [[META22:![0-9]+]], [[META23:![0-9]+]], [[META24:![0-9]+]]}
+; CHECK: [[META22]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META23]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META24]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META10]], [[META22]], [[META23]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
index 63a554c91d11f..4e81d16870e42 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
@@ -12,15 +12,19 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP6]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[IVBOUND_RTCHECK]]:
-; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[LOOPBOUND]], i64 4
+; CHECK-NEXT: [[BOUND2:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND3:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT1:%.*]] = and i1 [[BOUND2]], [[BOUND3]]
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT1]], [[FOUND_CONFLICT]]
+; CHECK-NEXT: [[BOUND011:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND112:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT13:%.*]] = and i1 [[BOUND011]], [[BOUND112]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT13]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[LOOPBOUND]]
; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
@@ -28,72 +32,76 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK4:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE5:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK3]], [[IVBOUND_WG_HIGH_CHK4]]
; CHECK-NEXT: [[IVBOUND_WG_RDX:%.*]] = or i1 [[IVBOUND_IN_WRITE_RANGE]], [[IVBOUND_IN_WRITE_RANGE5]]
-; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[FOUND_CONFLICT]], false
; CHECK-NEXT: [[IVBOUND_SAFE6:%.*]] = or i1 [[IVBOUND_SAFE]], [[IVBOUND_WG_RDX]]
; CHECK-NEXT: br i1 [[IVBOUND_SAFE6]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
-; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY_LVER_ORIG:.*]]
+; CHECK: [[FOR_BODY_LVER_ORIG]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], %[[FOR_BODY_LVER_ORIG]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX_LVER_ORIG]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER_ORIG:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX2_LVER_ORIG]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD_LVER_ORIG:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: store i32 [[ADD_LVER_ORIG]], ptr [[ARRAYIDX2_LVER_ORIG]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2_LVER_ORIG:%.*]] = add nsw i32 [[ADD_LVER_ORIG]], 1
+; CHECK-NEXT: store i32 [[ADD2_LVER_ORIG]], ptr [[ARRAYIDX_LVER_ORIG]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX9:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX9]], 8
+; CHECK-NEXT: [[CMP_LVER_ORIG:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[TMP3]]
+; CHECK-NEXT: br i1 [[CMP_LVER_ORIG]], label %[[FOR_BODY_LVER_ORIG]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX3]], 12
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[SMAX3]], 2
-; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
-; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[LOOPBOUND]], i64 4
+; CHECK-NEXT: [[BOUND020:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND121:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT22:%.*]] = and i1 [[BOUND020]], [[BOUND121]]
; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[A]], [[SCEVGEP5]]
-; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[B]], [[SCEVGEP4]]
-; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: [[BOUND124:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT25:%.*]] = and i1 [[BOUND06]], [[BOUND124]]
+; CHECK-NEXT: [[CONFLICT_RDX26:%.*]] = or i1 [[FOUND_CONFLICT22]], [[FOUND_CONFLICT25]]
+; CHECK-NEXT: [[BOUND027:%.*]] = icmp ult ptr [[B]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND128:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT29:%.*]] = and i1 [[BOUND027]], [[BOUND128]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = or i1 [[CONFLICT_RDX26]], [[FOUND_CONFLICT29]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX9]], 8
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX9]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX3]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX3]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8:![0-9]+]]
-; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META10]], !noalias [[META13]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
-; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META15:![0-9]+]], !noalias [[META17:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META15]], !noalias [[META17]]
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD11]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD12]], [[WIDE_LOAD10]]
-; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
-; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP9]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META15]], !noalias [[META17]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP9]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META15]], !noalias [[META17]]
; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP10]], splat (i32 1)
; CHECK-NEXT: [[TMP13:%.*]] = add nsw <4 x i32> [[TMP11]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
-; CHECK-NEXT: store <4 x i32> [[TMP13]], ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
+; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: store <4 x i32> [[TMP13]], ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META10]], !noalias [[META13]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX9]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
-; CHECK: [[FOR_BODY_LVER]]:
-; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
-; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[ADD2_LVER:%.*]] = add nsw i32 [[ADD_LVER]], 1
-; CHECK-NEXT: store i32 [[ADD2_LVER]], ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
-; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP17:![0-9]+]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
@@ -102,7 +110,7 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
@@ -112,10 +120,10 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD]], 1
; CHECK-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP22:![0-9]+]]
;
entry:
%0 = load i32, ptr %LoopBound, align 4, !tbaa !4
@@ -154,15 +162,19 @@ define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY_PREHEADER:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP17]], i64 1)
-; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[SMAX]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP18]]
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP18]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[LOOPBOUND]], i64 4
+; CHECK-NEXT: [[BOUND2:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND3:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT1:%.*]] = and i1 [[BOUND2]], [[BOUND3]]
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT1]], [[FOUND_CONFLICT]]
+; CHECK-NEXT: [[BOUND011:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND112:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT13:%.*]] = and i1 [[BOUND011]], [[BOUND112]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT13]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[LOOPBOUND]]
; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
@@ -170,79 +182,83 @@ define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK4:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE5:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK3]], [[IVBOUND_WG_HIGH_CHK4]]
; CHECK-NEXT: [[IVBOUND_WG_RDX:%.*]] = or i1 [[IVBOUND_IN_WRITE_RANGE]], [[IVBOUND_IN_WRITE_RANGE5]]
-; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[FOUND_CONFLICT]], false
; CHECK-NEXT: [[IVBOUND_SAFE6:%.*]] = or i1 [[IVBOUND_SAFE]], [[IVBOUND_WG_RDX]]
; CHECK-NEXT: br i1 [[IVBOUND_SAFE6]], label %[[FOR_BODY_PREHEADER1:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
-; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK: [[FOR_BODY_PREHEADER1]]:
+; CHECK-NEXT: br label %[[FOR_BODY_LVER_ORIG:.*]]
+; CHECK: [[FOR_BODY_LVER_ORIG]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], %[[FOR_BODY_LVER_ORIG]] ], [ 0, %[[FOR_BODY_PREHEADER1]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX_LVER_ORIG]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER_ORIG:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX2_LVER_ORIG]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD_LVER_ORIG:%.*]] = add nsw i32 [[TMP14]], [[TMP4]]
+; CHECK-NEXT: store i32 [[ADD_LVER_ORIG]], ptr [[ARRAYIDX2_LVER_ORIG]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2_LVER_ORIG:%.*]] = add nsw i32 [[ADD_LVER_ORIG]], 1
+; CHECK-NEXT: store i32 [[ADD2_LVER_ORIG]], ptr [[ARRAYIDX_LVER_ORIG]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX9:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP19]], i64 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX9]], 8
+; CHECK-NEXT: [[CMP_LVER_ORIG:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[TMP19]]
+; CHECK-NEXT: br i1 [[CMP_LVER_ORIG]], label %[[FOR_BODY_LVER_ORIG]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], !llvm.loop [[LOOP8]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP20:%.*]] = sext i32 [[TMP2]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP20]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX3]], 12
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[TMP20:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP20]], i64 1)
-; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[SMAX3]], 2
-; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[LOOPBOUND]], i64 4
+; CHECK-NEXT: [[BOUND020:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND121:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT22:%.*]] = and i1 [[BOUND020]], [[BOUND121]]
; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[A]], [[SCEVGEP5]]
-; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[B]], [[SCEVGEP4]]
-; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: [[BOUND124:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT25:%.*]] = and i1 [[BOUND06]], [[BOUND124]]
+; CHECK-NEXT: [[CONFLICT_RDX26:%.*]] = or i1 [[FOUND_CONFLICT22]], [[FOUND_CONFLICT25]]
+; CHECK-NEXT: [[BOUND027:%.*]] = icmp ult ptr [[B]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND128:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT29:%.*]] = and i1 [[BOUND027]], [[BOUND128]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = or i1 [[CONFLICT_RDX26]], [[FOUND_CONFLICT29]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX9]], 8
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX9]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX3]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX3]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20:![0-9]+]]
-; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23:![0-9]+]], !noalias [[META26:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META26]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23:![0-9]+]], !noalias [[META20]]
-; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
+; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META28:![0-9]+]], !noalias [[META30:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META28]], !noalias [[META30]]
; CHECK-NEXT: [[TMP9:%.*]] = add nsw <4 x i32> [[WIDE_LOAD11]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD12]], [[WIDE_LOAD10]]
-; CHECK-NEXT: store <4 x i32> [[TMP9]], ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
-; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
+; CHECK-NEXT: store <4 x i32> [[TMP9]], ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META28]], !noalias [[META30]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META28]], !noalias [[META30]]
; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[TMP9]], splat (i32 1)
; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP10]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP5]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
-; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP5]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META26]]
+; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META26]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; CHECK: [[FOR_COND_CLEANUP]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX9]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[FOR_COND_CLEANUP]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
-; CHECK: [[FOR_BODY_LVER]]:
-; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP15]], [[TMP14]]
-; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[ADD2_LVER:%.*]] = add nsw i32 [[ADD_LVER]], 1
-; CHECK-NEXT: store i32 [[ADD2_LVER]], ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
-; CHECK-NEXT: [[TMP16:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
-; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP16]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT1]], !llvm.loop [[LOOP26:![0-9]+]]
-; CHECK: [[FOR_BODY_PREHEADER1]]:
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP1:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT1]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP1]]
; CHECK: [[FOR_COND_CLEANUP1]]:
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY1]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY1]] ], [ 0, %[[FOR_BODY_PREHEADER1]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY1]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
@@ -252,10 +268,10 @@ define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD]], 1
; CHECK-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[TMP2]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP3]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY1]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP18]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY1]], label %[[FOR_COND_CLEANUP_LOOPEXIT1]], !llvm.loop [[LOOP32:![0-9]+]]
;
entry:
br label %for.body.preheader
@@ -304,23 +320,29 @@ attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: r
; CHECK: [[META5]] = !{!"int", [[META6:![0-9]+]], i64 0}
; CHECK: [[META6]] = !{!"omnipotent char", [[META7:![0-9]+]], i64 0}
; CHECK: [[META7]] = !{!"Simple C++ TBAA"}
-; CHECK: [[META8]] = !{[[META9:![0-9]+]]}
-; CHECK: [[META9]] = distinct !{[[META9]], [[META10:![0-9]+]]}
-; CHECK: [[META10]] = distinct !{[[META10]], !"LVerDomain"}
-; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{[[META12]], [[META10]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]], [[META16:![0-9]+]]}
-; CHECK: [[META14]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
-; CHECK: [[META15]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META16]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META14]], [[META15]]}
-; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META19:![0-9]+]]}
-; CHECK: [[META19]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META20]] = !{[[META21:![0-9]+]]}
-; CHECK: [[META21]] = distinct !{[[META21]], [[META22:![0-9]+]]}
-; CHECK: [[META22]] = distinct !{[[META22]], !"LVerDomain"}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]]}
+; CHECK: [[META9]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
+; CHECK: [[META11]] = distinct !{[[META11]], [[META12:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], !"LVerDomain"}
+; CHECK: [[META13]] = !{[[META14:![0-9]+]]}
+; CHECK: [[META14]] = distinct !{[[META14]], [[META12]]}
+; CHECK: [[META15]] = !{[[META16:![0-9]+]]}
+; CHECK: [[META16]] = distinct !{[[META16]], [[META12]]}
+; CHECK: [[META17]] = !{[[META11]], [[META14]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META9]], [[META19:![0-9]+]], [[META20:![0-9]+]], [[META21:![0-9]+]]}
+; CHECK: [[META19]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META20]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META21]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META9]], [[META19]], [[META20]]}
; CHECK: [[META23]] = !{[[META24:![0-9]+]]}
-; CHECK: [[META24]] = distinct !{[[META24]], [[META22]]}
-; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META14]], [[META15]], [[META16]]}
-; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META14]], [[META15]]}
+; CHECK: [[META24]] = distinct !{[[META24]], [[META25:![0-9]+]]}
+; CHECK: [[META25]] = distinct !{[[META25]], !"LVerDomain"}
+; CHECK: [[META26]] = !{[[META27:![0-9]+]]}
+; CHECK: [[META27]] = distinct !{[[META27]], [[META25]]}
+; CHECK: [[META28]] = !{[[META29:![0-9]+]]}
+; CHECK: [[META29]] = distinct !{[[META29]], [[META25]]}
+; CHECK: [[META30]] = !{[[META24]], [[META27]]}
+; CHECK: [[LOOP31]] = distinct !{[[LOOP31]], [[META9]], [[META19]], [[META20]], [[META21]]}
+; CHECK: [[LOOP32]] = distinct !{[[LOOP32]], [[META9]], [[META19]], [[META20]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
index 79a4ac0c6de6a..94bbe50234971 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -11,36 +11,50 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP6]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[IVBOUND_RTCHECK]]:
-; CHECK-NEXT: [[DOTRTCBLOCK_LOAD:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
-; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
-; CHECK-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX1]], 2
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[LOOPBOUND]], i64 4
+; CHECK-NEXT: [[BOUND4:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND5:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT2:%.*]] = and i1 [[BOUND4]], [[BOUND5]]
; CHECK-NEXT: [[BOUND2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
-; CHECK-NEXT: [[BOUND3:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND3:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[FOUND_CONFLICT1:%.*]] = and i1 [[BOUND2]], [[BOUND3]]
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[FOUND_CONFLICT2]], [[FOUND_CONFLICT1]]
+; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
; CHECK-NEXT: [[IVBOUND_WG_LOW_CHK:%.*]] = icmp ule ptr [[A]], [[LOOPBOUND]]
; CHECK-NEXT: [[IVBOUND_WG_HIGH_CHK:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[IVBOUND_IN_WRITE_RANGE:%.*]] = and i1 [[IVBOUND_WG_LOW_CHK]], [[IVBOUND_WG_HIGH_CHK]]
-; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[FOUND_CONFLICT1]], false
; CHECK-NEXT: [[IVBOUND_SAFE3:%.*]] = or i1 [[TMP9]], [[IVBOUND_IN_WRITE_RANGE]]
; CHECK-NEXT: br i1 [[IVBOUND_SAFE3]], label %[[FOR_BODY_PREHEADER_LVER:.*]], label %[[FOR_BODY_PREHEADER:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: br label %[[FOR_BODY_LVER_ORIG:.*]]
+; CHECK: [[FOR_BODY_LVER_ORIG]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], %[[FOR_BODY_LVER_ORIG]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX_LVER_ORIG]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER_ORIG:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER_ORIG]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX2_LVER_ORIG]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ADD_LVER_ORIG:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: store i32 [[ADD_LVER_ORIG]], ptr [[ARRAYIDX2_LVER_ORIG]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1
+; CHECK-NEXT: [[DOTRTCBLOCK_LOAD:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
-; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP10]], i64 1)
+; CHECK-NEXT: [[CMP_LVER_ORIG:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[TMP10]]
+; CHECK-NEXT: br i1 [[CMP_LVER_ORIG]], label %[[FOR_BODY_LVER_ORIG]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[TMP25]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX3]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP11]], i64 1)
-; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[SMAX]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]]
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[LOOPBOUND]], i64 4
+; CHECK-NEXT: [[BOUND09:%.*]] = icmp ult ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[BOUND110:%.*]] = icmp ult ptr [[B]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT: [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[LOOPBOUND]], inttoptr (i64 -1 to ptr)
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[CONFLICT_RDX15:%.*]] = or i1 [[FOUND_CONFLICT11]], [[FOUND_CONFLICT]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX15]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX3]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX3]], [[N_MOD_VF]]
@@ -49,40 +63,26 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP13]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META7:![0-9]+]]
-; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP14]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META7]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META9:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP14]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META9]]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP15]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP15]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10:![0-9]+]], !noalias [[META7]]
-; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP15]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META12:![0-9]+]], !noalias [[META14:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META12]], !noalias [[META14]]
; CHECK-NEXT: [[TMP17:%.*]] = add nsw <4 x i32> [[WIDE_LOAD5]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
-; CHECK-NEXT: store <4 x i32> [[TMP17]], ptr [[TMP15]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
-; CHECK-NEXT: store <4 x i32> [[TMP18]], ptr [[TMP16]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
+; CHECK-NEXT: store <4 x i32> [[TMP17]], ptr [[TMP15]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META12]], !noalias [[META14]]
+; CHECK-NEXT: store <4 x i32> [[TMP18]], ptr [[TMP16]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META12]], !noalias [[META14]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
-; CHECK: [[FOR_BODY_LVER]]:
-; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA3]]
-; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
-; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA3]]
-; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP21]], [[TMP20]]
-; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA3]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
-; CHECK-NEXT: [[TMP22:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
-; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP22]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP16:![0-9]+]]
-; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
@@ -91,7 +91,7 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA3]]
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
@@ -99,10 +99,10 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP24]], [[TMP23]]
; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA3]]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
; CHECK-NEXT: [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP26]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP20:![0-9]+]]
;
entry:
%0 = load i32, ptr %LoopBound, align 4, !tbaa !4
@@ -150,16 +150,18 @@ for.body: ; preds = %for.body.preheader,
; CHECK: [[META4]] = !{!"int", [[META5:![0-9]+]], i64 0}
; CHECK: [[META5]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
; CHECK: [[META6]] = !{!"Simple C++ TBAA"}
-; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]}
-; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"}
-; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
-; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META14:![0-9]+]], [[META15:![0-9]+]]}
-; CHECK: [[META13]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
-; CHECK: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META15]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META13]], [[META14]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META18:![0-9]+]]}
-; CHECK: [[META18]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
+; CHECK: [[META8]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META11:![0-9]+]]}
+; CHECK: [[META11]] = distinct !{[[META11]], !"LVerDomain"}
+; CHECK: [[META12]] = !{[[META13:![0-9]+]]}
+; CHECK: [[META13]] = distinct !{[[META13]], [[META11]]}
+; CHECK: [[META14]] = !{[[META10]], [[META15:![0-9]+]]}
+; CHECK: [[META15]] = distinct !{[[META15]], [[META11]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META8]], [[META17:![0-9]+]], [[META18:![0-9]+]], [[META19:![0-9]+]]}
+; CHECK: [[META17]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META18]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META19]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META8]], [[META17]], [[META18]]}
;.
More information about the llvm-commits
mailing list