[llvm] [LoopVectorize] Enable vectorization of uncountable loops with invariant bound loads via speculative hoisting and runtime versioning. (PR #195027)
Pawan Nirpal via llvm-commits
llvm-commits at lists.llvm.org
Mon May 4 01:12:34 PDT 2026
https://github.com/pawan-nirpal-031 updated https://github.com/llvm/llvm-project/pull/195027
>From ce6347d4e088fdc2941e1a2e267cf04b75d6276b Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Tue, 16 Dec 2025 11:07:23 +0530
Subject: [PATCH 1/6] [AArch64] - Allow for aggressive unrolling, with non-zero
LoopMicroOpBufferSize for Oryon
---
llvm/lib/Target/AArch64/AArch64SchedOryon.td | 2 +-
.../aarch64-mcpu-oryon-runtime-unroll.ll | 152 ++++++++++++++++++
2 files changed, 153 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/aarch64-mcpu-oryon-runtime-unroll.ll
diff --git a/llvm/lib/Target/AArch64/AArch64SchedOryon.td b/llvm/lib/Target/AArch64/AArch64SchedOryon.td
index 5b597b91e7459..435eaf99c6175 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedOryon.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedOryon.td
@@ -19,7 +19,7 @@ def OryonModel : SchedMachineModel {
let MicroOpBufferSize = 376;
let LoadLatency = 4;
let MispredictPenalty = 13; // 13 cycles for mispredicted branch.
- let LoopMicroOpBufferSize = 0; // Do not have a LoopMicroOpBuffer
+ let LoopMicroOpBufferSize = 16; // Oryon-1 does not have loop micro op buffer, we enable this pseudo value to allow for aggressive unrolling based on runtime TC.
let PostRAScheduler = 1; // Using PostRA sched.
let CompleteModel = 1;
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mcpu-oryon-runtime-unroll.ll b/llvm/test/CodeGen/AArch64/aarch64-mcpu-oryon-runtime-unroll.ll
new file mode 100644
index 0000000000000..79136cf71c005
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-mcpu-oryon-runtime-unroll.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='loop-unroll' -S %s | FileCheck %s --check-prefix=UNROLLED
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @foo(ptr %mat, ptr %sharr, ptr %barr, i16 %rows, i16 %dimout) #0 {
+; UNROLLED-LABEL: define void @foo(
+; UNROLLED-SAME: ptr [[MAT:%.*]], ptr [[SHARR:%.*]], ptr [[BARR:%.*]], i16 [[ROWS:%.*]], i16 [[DIMOUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; UNROLLED-NEXT: [[ENTRY:.*:]]
+; UNROLLED-NEXT: [[CMP33:%.*]] = icmp sgt i16 [[DIMOUT]], 0
+; UNROLLED-NEXT: br i1 [[CMP33]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_END22:.*]]
+; UNROLLED: [[FOR_BODY_LR_PH]]:
+; UNROLLED-NEXT: [[CMP631:%.*]] = icmp sgt i16 [[ROWS]], 0
+; UNROLLED-NEXT: br i1 [[CMP631]], label %[[FOR_BODY_US_PREHEADER:.*]], label %[[FOR_BODY_LR_PH_SPLIT:.*]]
+; UNROLLED: [[FOR_BODY_US_PREHEADER]]:
+; UNROLLED-NEXT: [[WIDE_TRIP_COUNT39:%.*]] = zext nneg i16 [[DIMOUT]] to i64
+; UNROLLED-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i16 [[ROWS]] to i64
+; UNROLLED-NEXT: [[TMP0:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
+; UNROLLED-NEXT: br label %[[FOR_BODY_US:.*]]
+; UNROLLED: [[FOR_BODY_US]]:
+; UNROLLED-NEXT: [[INDVARS_IV36:%.*]] = phi i64 [ 0, %[[FOR_BODY_US_PREHEADER]] ], [ [[INDVARS_IV_NEXT37:%.*]], %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US:.*]] ]
+; UNROLLED-NEXT: store i8 0, ptr [[BARR]], align 1
+; UNROLLED-NEXT: [[INVARIANT_GEP_US:%.*]] = getelementptr i8, ptr [[MAT]], i64 [[INDVARS_IV36]]
+; UNROLLED-NEXT: [[XTRAITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 1
+; UNROLLED-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; UNROLLED-NEXT: br i1 [[TMP1]], label %[[FOR_BODY8_US_EPIL_PREHEADER:.*]], label %[[FOR_BODY_US_NEW:.*]]
+; UNROLLED: [[FOR_BODY_US_NEW]]:
+; UNROLLED-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[XTRAITER]]
+; UNROLLED-NEXT: br label %[[FOR_BODY8_US:.*]]
+; UNROLLED: [[FOR_BODY8_US]]:
+; UNROLLED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_US_NEW]] ], [ [[INDVARS_IV_NEXT_1:%.*]], %[[FOR_INC_US_1:.*]] ]
+; UNROLLED-NEXT: [[TMP2:%.*]] = phi i8 [ 0, %[[FOR_BODY_US_NEW]] ], [ [[TMP8:%.*]], %[[FOR_INC_US_1]] ]
+; UNROLLED-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[FOR_BODY_US_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[FOR_INC_US_1]] ]
+; UNROLLED-NEXT: [[GEP_US:%.*]] = getelementptr [2 x i8], ptr [[INVARIANT_GEP_US]], i64 [[INDVARS_IV]]
+; UNROLLED-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_US]], align 1
+; UNROLLED-NEXT: [[TOBOOL_NOT_US:%.*]] = icmp eq i8 [[TMP3]], 0
+; UNROLLED-NEXT: br i1 [[TOBOOL_NOT_US]], label %[[FOR_INC_US:.*]], label %[[IF_THEN_US:.*]]
+; UNROLLED: [[IF_THEN_US]]:
+; UNROLLED-NEXT: [[ARRAYIDX14_US:%.*]] = getelementptr inbounds nuw i8, ptr [[SHARR]], i64 [[INDVARS_IV]]
+; UNROLLED-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX14_US]], align 1
+; UNROLLED-NEXT: [[XOR30_US:%.*]] = xor i8 [[TMP2]], [[TMP4]]
+; UNROLLED-NEXT: store i8 [[XOR30_US]], ptr [[BARR]], align 1
+; UNROLLED-NEXT: br label %[[FOR_INC_US]]
+; UNROLLED: [[FOR_INC_US]]:
+; UNROLLED-NEXT: [[TMP5:%.*]] = phi i8 [ [[TMP2]], %[[FOR_BODY8_US]] ], [ [[XOR30_US]], %[[IF_THEN_US]] ]
+; UNROLLED-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; UNROLLED-NEXT: [[GEP_US_1:%.*]] = getelementptr [2 x i8], ptr [[INVARIANT_GEP_US]], i64 [[INDVARS_IV_NEXT]]
+; UNROLLED-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_US_1]], align 1
+; UNROLLED-NEXT: [[TOBOOL_NOT_US_1:%.*]] = icmp eq i8 [[TMP6]], 0
+; UNROLLED-NEXT: br i1 [[TOBOOL_NOT_US_1]], label %[[FOR_INC_US_1]], label %[[IF_THEN_US_1:.*]]
+; UNROLLED: [[IF_THEN_US_1]]:
+; UNROLLED-NEXT: [[ARRAYIDX14_US_1:%.*]] = getelementptr inbounds nuw i8, ptr [[SHARR]], i64 [[INDVARS_IV_NEXT]]
+; UNROLLED-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX14_US_1]], align 1
+; UNROLLED-NEXT: [[XOR30_US_1:%.*]] = xor i8 [[TMP5]], [[TMP7]]
+; UNROLLED-NEXT: store i8 [[XOR30_US_1]], ptr [[BARR]], align 1
+; UNROLLED-NEXT: br label %[[FOR_INC_US_1]]
+; UNROLLED: [[FOR_INC_US_1]]:
+; UNROLLED-NEXT: [[TMP8]] = phi i8 [ [[TMP5]], %[[FOR_INC_US]] ], [ [[XOR30_US_1]], %[[IF_THEN_US_1]] ]
+; UNROLLED-NEXT: [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; UNROLLED-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; UNROLLED-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; UNROLLED-NEXT: br i1 [[NITER_NCMP_1]], label %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US_UNR_LCSSA:.*]], label %[[FOR_BODY8_US]]
+; UNROLLED: [[FOR_COND3_FOR_INC20_CRIT_EDGE_US_UNR_LCSSA]]:
+; UNROLLED-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1]], %[[FOR_INC_US_1]] ]
+; UNROLLED-NEXT: [[DOTUNR:%.*]] = phi i8 [ [[TMP8]], %[[FOR_INC_US_1]] ]
+; UNROLLED-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; UNROLLED-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_BODY8_US_EPIL_PREHEADER]], label %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US]]
+; UNROLLED: [[FOR_BODY8_US_EPIL_PREHEADER]]:
+; UNROLLED-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[FOR_BODY_US]] ], [ [[INDVARS_IV_UNR]], %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US_UNR_LCSSA]] ]
+; UNROLLED-NEXT: [[DOTEPIL_INIT:%.*]] = phi i8 [ 0, %[[FOR_BODY_US]] ], [ [[DOTUNR]], %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US_UNR_LCSSA]] ]
+; UNROLLED-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; UNROLLED-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]])
+; UNROLLED-NEXT: br label %[[FOR_BODY8_US_EPIL:.*]]
+; UNROLLED: [[FOR_BODY8_US_EPIL]]:
+; UNROLLED-NEXT: [[GEP_US_EPIL:%.*]] = getelementptr [2 x i8], ptr [[INVARIANT_GEP_US]], i64 [[INDVARS_IV_EPIL_INIT]]
+; UNROLLED-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_US_EPIL]], align 1
+; UNROLLED-NEXT: [[TOBOOL_NOT_US_EPIL:%.*]] = icmp eq i8 [[TMP9]], 0
+; UNROLLED-NEXT: br i1 [[TOBOOL_NOT_US_EPIL]], label %[[FOR_INC_US_EPIL:.*]], label %[[IF_THEN_US_EPIL:.*]]
+; UNROLLED: [[IF_THEN_US_EPIL]]:
+; UNROLLED-NEXT: [[ARRAYIDX14_US_EPIL:%.*]] = getelementptr inbounds nuw i8, ptr [[SHARR]], i64 [[INDVARS_IV_EPIL_INIT]]
+; UNROLLED-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX14_US_EPIL]], align 1
+; UNROLLED-NEXT: [[XOR30_US_EPIL:%.*]] = xor i8 [[DOTEPIL_INIT]], [[TMP10]]
+; UNROLLED-NEXT: store i8 [[XOR30_US_EPIL]], ptr [[BARR]], align 1
+; UNROLLED-NEXT: br label %[[FOR_INC_US_EPIL]]
+; UNROLLED: [[FOR_INC_US_EPIL]]:
+; UNROLLED-NEXT: br label %[[FOR_COND3_FOR_INC20_CRIT_EDGE_US]]
+; UNROLLED: [[FOR_COND3_FOR_INC20_CRIT_EDGE_US]]:
+; UNROLLED-NEXT: [[INDVARS_IV_NEXT37]] = add nuw nsw i64 [[INDVARS_IV36]], 1
+; UNROLLED-NEXT: [[EXITCOND40_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT37]], [[WIDE_TRIP_COUNT39]]
+; UNROLLED-NEXT: br i1 [[EXITCOND40_NOT]], label %[[FOR_END22_LOOPEXIT:.*]], label %[[FOR_BODY_US]]
+; UNROLLED: [[FOR_BODY_LR_PH_SPLIT]]:
+; UNROLLED-NEXT: store i8 0, ptr [[BARR]], align 1
+; UNROLLED-NEXT: br label %[[FOR_END22]]
+; UNROLLED: [[FOR_END22_LOOPEXIT]]:
+; UNROLLED-NEXT: br label %[[FOR_END22]]
+; UNROLLED: [[FOR_END22]]:
+; UNROLLED-NEXT: ret void
+;
+entry:
+ %cmp33 = icmp sgt i16 %dimout, 0
+ br i1 %cmp33, label %for.body.lr.ph, label %for.end22
+
+for.body.lr.ph: ; preds = %entry
+ %cmp631 = icmp sgt i16 %rows, 0
+ br i1 %cmp631, label %for.body.us.preheader, label %for.body.lr.ph.split
+
+for.body.us.preheader: ; preds = %for.body.lr.ph
+ %wide.trip.count39 = zext nneg i16 %dimout to i64
+ %wide.trip.count = zext nneg i16 %rows to i64
+ br label %for.body.us
+
+for.body.us: ; preds = %for.body.us.preheader, %for.cond3.for.inc20_crit_edge.us
+ %indvars.iv36 = phi i64 [ 0, %for.body.us.preheader ], [ %indvars.iv.next37, %for.cond3.for.inc20_crit_edge.us ]
+ store i8 0, ptr %barr, align 1
+ %invariant.gep.us = getelementptr i8, ptr %mat, i64 %indvars.iv36
+ br label %for.body8.us
+
+for.body8.us: ; preds = %for.body.us, %for.inc.us
+ %indvars.iv = phi i64 [ 0, %for.body.us ], [ %indvars.iv.next, %for.inc.us ]
+ %0 = phi i8 [ 0, %for.body.us ], [ %3, %for.inc.us ]
+ %gep.us = getelementptr [2 x i8], ptr %invariant.gep.us, i64 %indvars.iv
+ %1 = load i8, ptr %gep.us, align 1
+ %tobool.not.us = icmp eq i8 %1, 0
+ br i1 %tobool.not.us, label %for.inc.us, label %if.then.us
+
+if.then.us: ; preds = %for.body8.us
+ %arrayidx14.us = getelementptr inbounds nuw i8, ptr %sharr, i64 %indvars.iv
+ %2 = load i8, ptr %arrayidx14.us, align 1
+ %xor30.us = xor i8 %0, %2
+ store i8 %xor30.us, ptr %barr, align 1
+ br label %for.inc.us
+
+for.inc.us: ; preds = %if.then.us, %for.body8.us
+ %3 = phi i8 [ %0, %for.body8.us ], [ %xor30.us, %if.then.us ]
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond3.for.inc20_crit_edge.us, label %for.body8.us
+
+for.cond3.for.inc20_crit_edge.us: ; preds = %for.inc.us
+ %indvars.iv.next37 = add nuw nsw i64 %indvars.iv36, 1
+ %exitcond40.not = icmp eq i64 %indvars.iv.next37, %wide.trip.count39
+ br i1 %exitcond40.not, label %for.end22, label %for.body.us
+
+for.body.lr.ph.split: ; preds = %for.body.lr.ph
+ store i8 0, ptr %barr, align 1
+ br label %for.end22
+
+for.end22: ; preds = %for.cond3.for.inc20_crit_edge.us, %for.body.lr.ph.split, %entry
+ ret void
+}
+
+attributes #0 = { "target-cpu"="oryon-1" "target-features"="+neon,+sve" }
>From 86f0e12d4478bb09207f0fe749358e1e6a491d3a Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Thu, 30 Apr 2026 00:31:17 -0700
Subject: [PATCH 2/6] [LoopVectorize] Enable vectorization of uncountable loops
with invariant bound loads via speculative hoisting and runtime versioning
---
.../Vectorize/LoopVectorizationLegality.h | 2 +
.../Vectorize/LoopVectorizationLegality.cpp | 117 +++++++++
.../Transforms/Vectorize/LoopVectorize.cpp | 194 +++++++++++++++
.../dynamic-bound-array-element.ll | 177 ++++++++++++++
.../dynamic-bound-chained-loads.ll | 99 ++++++++
.../LoopVectorize/dynamic-bound-fcmp-exit.ll | 56 +++++
.../dynamic-bound-loop-varying.ll | 79 ++++++
.../LoopVectorize/dynamic-bound-n-stores.ll | 226 ++++++++++++++++++
.../LoopVectorize/dynamic-bound-simple.ll | 160 +++++++++++++
.../dynamic-bound-unsafe-calls.ll | 150 ++++++++++++
10 files changed, 1260 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 49e8bb1e85526..7cc5f037ff90f 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -350,6 +350,8 @@ class LoopVectorizationLegality {
/// reductions found in the loop.
bool isInvariantStoreOfReduction(StoreInst *SI);
+ LoadInst* tryToFindDyanmicBoundLoadCandidate(Loop *L, AAResults &AA);
+
/// Returns True if given address is invariant and is used to store recurrent
/// expression
bool isInvariantAddressOfReduction(Value *V);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index bee08eeba9927..4c141782007e8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -579,6 +579,123 @@ class SCEVAddRecForUniformityRewriter
} // namespace
+static Value *stripSimpleCasts(Value *V) {
+ while (isa<SExtInst>(V) || isa<ZExtInst>(V) || isa<TruncInst>(V))
+ V = cast<Instruction>(V)->getOperand(0);
+ return V;
+}
+
+static LoadInst *getLoopBoundLoadCandidate(Loop *L) {
+ Instruction *Icmp = L->getLatchCmpInst();
+ if (!Icmp)
+ return nullptr;
+
+ Value *Op0 = stripSimpleCasts(Icmp->getOperand(0));
+ Value *Op1 = stripSimpleCasts(Icmp->getOperand(1));
+
+ if (!isa<LoadInst>(Op0) && !isa<LoadInst>(Op1))
+ return nullptr;
+
+ LoadInst *BoundLoad =
+ isa<LoadInst>(Op0) ? dyn_cast<LoadInst>(Op0) : dyn_cast<LoadInst>(Op1);
+
+ // Reject non int load types.
+ if (!BoundLoad->getType()->isIntegerTy())
+ return nullptr;
+
+ return BoundLoad;
+}
+
+static bool isLoadSpeculativelyHoistable(Loop *L, LoadInst *LoadCandidate,
+ PredicatedScalarEvolution &PSE,
+ AAResults &AA) {
+
+ if (!L || !LoadCandidate)
+ return false;
+
+ if (!L->contains(LoadCandidate))
+ return false;
+
+ Value *Ptr = LoadCandidate->getPointerOperand();
+ if (!L->isLoopInvariant(Ptr))
+ return false;
+
+ // We don't want to deal with global pointers for now.
+ if (isa<GlobalValue>(Ptr))
+ return false;
+
+ if (!LoadCandidate->isSimple() || LoadCandidate->isVolatile())
+ return false;
+
+ const SCEV *PtrScev = PSE.getSCEV(Ptr);
+ if (!PtrScev || !PSE.getSE()->isLoopInvariant(PtrScev, L))
+ return false;
+
+ for (auto *User : Ptr->users()) {
+ Instruction *UserI = dyn_cast<Instruction>(User);
+ if (!UserI || UserI == dyn_cast<Instruction>(LoadCandidate))
+ continue;
+
+ if (!L->contains(UserI))
+ continue;
+
+ if (UserI->mayWriteToMemory())
+ return false;
+ }
+
+ MemoryLocation LoadLoc = MemoryLocation::get(LoadCandidate);
+ // No call in loop may write to Ptr via aliasing.
+ for (BasicBlock *BB : L->blocks()) {
+ for (Instruction &I : *BB) {
+ auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB)
+ continue;
+
+ MemoryEffects ME = AA.getMemoryEffects(CB);
+ // If call doesn't write anything it's safe.
+ if (!isModSet(ME.getModRef()))
+ continue;
+
+ // If call writes to escaped or global memory reject.
+ if (isModSet(ME.getModRef(IRMemLocation::Other)))
+ return false;
+
+ // Check if any args alias with loadloc.
+ for (Value *Arg : CB->args()) {
+ if (!Arg->getType()->isPointerTy())
+ continue;
+ if (AA.alias(MemoryLocation::getBeforeOrAfter(Arg), LoadLoc) !=
+ AliasResult::NoAlias)
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+LoadInst *
+LoopVectorizationLegality::tryToFindDyanmicBoundLoadCandidate(Loop *L,
+ AAResults &AA) {
+ if (!L->isInnermost())
+ return nullptr;
+
+ LoadInst *LoadCandidate = getLoopBoundLoadCandidate(L);
+ if (!LoadCandidate)
+ return nullptr;
+
+ // Reject if loop body may have some non-vectorizable calls, like pow/ instrs
+ // with side-effects, here even if the load were hoisted it would not allow
+ // the loop to vectorize anyway.
+ if (!canVectorizeInstrs())
+ return nullptr;
+
+ if (!isLoadSpeculativelyHoistable(L, LoadCandidate, PSE, AA))
+ return nullptr;
+
+ return LoadCandidate;
+}
+
bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const {
if (isInvariant(V))
return true;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6c9298a2cf98d..0614b13066f7f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -144,6 +144,7 @@
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
+#include "llvm/Transforms/Utils/Cloning.h"
#include <algorithm>
#include <cassert>
#include <cmath>
@@ -378,6 +379,8 @@ static cl::opt<bool> EnableEarlyExitVectorization(
// after prolog. See `emitIterationCountCheck`.
static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
+static bool AllowInvariantBoundRetry = true;
+
/// A helper function that returns true if the given type is irregular. The
/// type is irregular if its allocated size doesn't equal the store size of an
/// element of the corresponding vector type.
@@ -8163,6 +8166,179 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi.eraseFromParent();
}
+static bool isSpeculativeBoundVersioningProfitable() { return true; }
+
+
+/*
+
+ preheader
+ |
+ rtcheck-block
+ / \
+ / \
+ for.preheader ver.preheader
+ | |
+ for.body ver.for.body
+
+*/
+static Loop *tryToVersionLoopForInvariantBoundLoad(
+ Loop *L, LoadInst *BoundLoad, const DataLayout &DL, DominatorTree &DT,
+ LoopInfo &LI, AAResults &AA, PredicatedScalarEvolution &PSE,
+ AssumptionCache *AC) {
+ LoadInst *InnerLoad = BoundLoad;
+ if (!InnerLoad)
+ return nullptr;
+ auto *PreHeader = L->getLoopPreheader();
+ if (!PreHeader)
+ return nullptr;
+
+ Value *BoundPtrOl = InnerLoad->getPointerOperand();
+ if (auto *BoundPtrI = dyn_cast<Instruction>(BoundPtrOl))
+ if (!DT.properlyDominates(BoundPtrI->getParent(), PreHeader))
+ return nullptr;
+
+ // All the relevent base pointers of participating stores should dominate both
+ // loops.
+ LoopAccessInfoManager PreFlightLAIs(*PSE.getSE(), AA, DT, LI, nullptr,
+ nullptr, AC);
+ const LoopAccessInfo &PreLAI = PreFlightLAIs.getInfo(*L);
+ const auto &RtPtrChecking = *PreLAI.getRuntimePointerChecking();
+
+ for (const auto &Check : RtPtrChecking.getChecks()) {
+ for (const RuntimeCheckingPtrGroup *Group : {Check.first, Check.second}) {
+ for (unsigned Idx : Group->Members) {
+ Value *Ptr = RtPtrChecking.Pointers[Idx].PointerValue;
+ Value *Base = getUnderlyingObject(Ptr);
+ if (auto *BaseI = dyn_cast<Instruction>(Base))
+ if (!DT.properlyDominates(BaseI->getParent(), PreHeader)) {
+ return nullptr;
+ }
+ }
+ }
+ }
+
+ Function *ParentF = PreHeader->getParent();
+ LLVMContext &Ctx = ParentF->getContext();
+
+ BasicBlock *RtCheckBB =
+ BasicBlock::Create(Ctx, "ivbound.rtcheck", ParentF, PreHeader);
+
+ // Temporary terminator for SCEVExapnder to have valid insert point.
+ IRBuilder<> TmpBuilder(RtCheckBB);
+ Instruction *TmpTerm = TmpBuilder.CreateUnreachable();
+
+ // Re-wire the CFG.
+ // Redirect the predecessors of preheader the new rtcheck block.
+ SmallVector<BasicBlock *, 4> PreHeaderPreds(predecessors(PreHeader));
+ for (auto *Pred : PreHeaderPreds)
+ Pred->getTerminator()->replaceUsesOfWith(PreHeader, RtCheckBB);
+
+ BasicBlock *OldPreheadIdom = DT.getNode(PreHeader)->getIDom()->getBlock();
+ DT.addNewBlock(RtCheckBB, OldPreheadIdom);
+ DT.changeImmediateDominator(PreHeader, RtCheckBB);
+
+ // Fix the preheader phis via rtcblock phis.
+ for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
+ // Create the rtcblock phi.
+ PHINode *RtcPN =
+ PHINode::Create(PN.getType(), PreHeaderPreds.size(),
+ PN.getName() + ".rtcphi", &RtCheckBB->front());
+ for (unsigned i = 0; i < PN.getNumIncomingValues(); i++) {
+ BasicBlock *IBB = PN.getIncomingBlock(i);
+ assert(is_contained(PreHeaderPreds, IBB) &&
+ "Incoming Basic Block not part of predecessors list!");
+ RtcPN->addIncoming(PN.getIncomingValue(i), IBB);
+ }
+ PN.replaceAllUsesWith(RtcPN);
+ PN.eraseFromParent();
+ }
+
+ // Clone the loop including preheader.
+ ValueToValueMapTy VMap;
+ SmallVector<BasicBlock *, 4> NewBlocks;
+ Loop *VerLoop = cloneLoopWithPreheader(PreHeader, RtCheckBB, L, VMap, ".lver",
+ &LI, &DT, NewBlocks);
+
+ remapInstructionsInBlocks(NewBlocks, VMap);
+
+ BasicBlock *VerPreHeader = cast<BasicBlock>(VMap[PreHeader]);
+ BasicBlock *VerLatch = cast<BasicBlock>(VMap[L->getLoopLatch()]);
+
+ if (Loop *ParLoop = L->getParentLoop())
+ ParLoop->addBasicBlockToLoop(RtCheckBB, LI);
+
+ VerLatch->getTerminator()->setMetadata(LLVMContext::MD_loop, nullptr);
+
+ LoadInst *HoistLoad = cast<LoadInst>(InnerLoad->clone());
+ HoistLoad->setName(InnerLoad->getName() + ".speculatively.hoisted");
+ // Put this hoisted load in RtcheckBB, since SCEVExapnder needs it.
+ HoistLoad->insertBefore(TmpTerm);
+ Instruction *ClonedInnerLoad = cast<Instruction>(VMap[InnerLoad]);
+ ClonedInnerLoad->replaceAllUsesWith(HoistLoad);
+ ClonedInnerLoad->eraseFromParent();
+
+ // fix lcssa phis.
+ SmallVector<BasicBlock *, 4> ExitBlocks;
+ L->getExitBlocks(ExitBlocks);
+ for (BasicBlock *ExBB : ExitBlocks) {
+ DT.changeImmediateDominator(ExBB, RtCheckBB);
+ for (PHINode &Phi : ExBB->phis()) {
+ int idx = Phi.getBasicBlockIndex(L->getLoopLatch());
+ if (idx < 0)
+ continue;
+ Value *OrigVal = Phi.getIncomingValue(idx);
+ Value *NewVal =
+ VMap.count(OrigVal) ? cast<Value>(VMap[OrigVal]) : OrigVal;
+ Phi.addIncoming(NewVal, VerLatch);
+ }
+ }
+
+ formDedicatedExitBlocks(L, &DT, &LI, nullptr, true);
+ formDedicatedExitBlocks(VerLoop, &DT, &LI, nullptr, true);
+
+ // RTC condition generation using SCEV.
+ auto &SE = *PSE.getSE();
+ LoopAccessInfoManager NewLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
+ const LoopAccessInfo &VerLAI = NewLAIs.getInfo(*VerLoop);
+
+ // Generate LAA based ptr runtime checks.
+ SCEVExpander MemExp(*VerLAI.getRuntimePointerChecking()->getSE(),
+ "ivbound.mem.check");
+ Value *MemRTC = nullptr;
+ if (VerLAI.getRuntimePointerChecking()->Need)
+ MemRTC = addRuntimeChecks(TmpTerm, VerLoop,
+ VerLAI.getRuntimePointerChecking()->getChecks(),
+ MemExp);
+
+ // Generate SCEV predicate runtime checks.
+ SCEVExpander SCEVEx(SE, "ivbound.scev.check");
+ const SCEVPredicate &Preds = VerLAI.getPSE().getPredicate();
+ Value *SCEVrtc = SCEVEx.expandCodeForPredicate(&Preds, TmpTerm);
+
+ Value *RtCheckCond = nullptr;
+ if (MemRTC && SCEVrtc) {
+ IRBuilder<> Builder(TmpTerm);
+ RtCheckCond = Builder.CreateOr(MemRTC, SCEVrtc, "ivbound.safe");
+ } else {
+ RtCheckCond = MemRTC ? MemRTC : SCEVrtc;
+ }
+
+ TmpTerm->eraseFromParent();
+ IRBuilder<> Builder(RtCheckBB);
+
+ // If No runtime checks needed then ver-loop is unconditionally safe.
+ // Use constant false to disregard orignal loop path.
+ if (!RtCheckCond)
+ RtCheckCond = ConstantInt::getFalse(Ctx);
+
+ Builder.CreateCondBr(RtCheckCond, PreHeader, VerPreHeader);
+ // RTCheck now idom's both preheaders.
+ DT.insertEdge(RtCheckBB, PreHeader);
+ DT.insertEdge(RtCheckBB, VerPreHeader);
+
+ return VerLoop;
+}
+
bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&
"VPlan-native path is not enabled. Only process inner loops.");
@@ -8215,6 +8391,24 @@ bool LoopVectorizePass::processLoop(Loop *L) {
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
+ if (AllowInvariantBoundRetry) {
+ // There has to be a better way to do this? Trying to avoid changing processLoop Signature.
+ AllowInvariantBoundRetry = false;
+ const DataLayout &DL = F->getDataLayout();
+ // 1) Check if the loop qualifies for loop invariant bound load.
+ // 2) Generate the versioned loop with runtime checks using the dynamic loop bound.
+ // load. Return this versioned loop to attempt vectorization
+ // again. 3) If the versioned loop is generated successfully, re-try
+ // vectorization on the versioned loop.
+ if (LoadInst *BoundLoad = LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
+ if (isSpeculativeBoundVersioningProfitable())
+ if (Loop *Cand = tryToVersionLoopForInvariantBoundLoad(
+ L, BoundLoad, DL, *DT, *LI, *AA, PSE, AC)) {
+ SE->forgetLoop(L);
+ return processLoop(Cand);
+ }
+ }
+ }
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
Hints.emitRemarkWithHints();
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
new file mode 100644
index 0000000000000..78d4431437138
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; TEST10: Loop bound loaded from an array element A[2], where A is also
+; written inside the loop. The load and store use different indices, so
+; pointer identity does not flag a conflict, but runtime checks will
+; guard correctness.
+; Verify the load is hoisted and runtime checks are generated.
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < A[2]; i++)
+; A[i] = B[i] + C[i];
+; }
+;
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) #0 {
+; CHECK-LABEL: define dso_local void @foo(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C9:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: [[B8:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[A7:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[IVBOUND_RTCHECK]]:
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], false
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[A7]], [[B8]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 32
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[A7]], [[C9]]
+; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = icmp ult i64 [[TMP5]], 32
+; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK10]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD14]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD15]], [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP12]], i64 4
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX1_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1_LVER]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX3_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX3_LVER]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NEXT: [[ARRAYIDX5_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX5_LVER]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP19]], [[TMP18]]
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+;
+entry:
+ %arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
+ %0 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %cmp11 = icmp sgt i32 %0, 0
+ br i1 %cmp11, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx1 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx1, align 4, !tbaa !5
+ %arrayidx3 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx3, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx5, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
+; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]], [[META10:![0-9]+]]}
+; CHECK: [[META9]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META10]] = !{!"llvm.loop.unroll.disable"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
new file mode 100644
index 0000000000000..ffe3d2511b71b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; Negative test: the upper bound is **PtrLen (a chained / indirect load).
+; The worklist correctly traces both loads, but LoopAccessAnalysis cannot
+; compute SCEV bounds for the indirect load (load i32, ptr %ptr) because
+; %ptr itself is loaded inside the loop. The pass must reject the loop.
+;
+; C source:
+; void foo(int *A, int *B, int *C, int **PtrLen) {
+; for (int i = 0; i < **PtrLen; i++)
+; A[i] = B[i] + C[i];
+; }
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %PtrLen) #0 {
+; CHECK-LABEL: define dso_local void @foo(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readonly captures(none) [[PTRLEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[PTR0:%.*]] = load ptr, ptr [[PTRLEN]], align 8, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[VAL0:%.*]] = load i32, ptr [[PTR0]], align 4, !tbaa [[TBAA5:![0-9]+]]
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[VAL0]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[PTR:%.*]] = load ptr, ptr [[PTRLEN]], align 8, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR]], align 4, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[VAL]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP2]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+;
+entry:
+ %ptr0 = load ptr, ptr %PtrLen, align 8, !tbaa !5
+ %val0 = load i32, ptr %ptr0, align 4, !tbaa !10
+ %cmp9 = icmp sgt i32 %val0, 0
+ br i1 %cmp9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !10
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !10
+ %add = add nsw i32 %2, %1
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx4, align 4, !tbaa !10
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %ptr = load ptr, ptr %PtrLen, align 8, !tbaa !5
+ %val = load i32, ptr %ptr, align 4, !tbaa !10
+ %3 = sext i32 %val to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %3
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !12
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 8}
+!6 = !{!7, i64 8, !"p1 int"}
+!7 = !{!8, i64 8, !"any pointer"}
+!8 = !{!9, i64 1, !"omnipotent char"}
+!9 = !{!"Simple C++ TBAA"}
+!10 = !{!11, !11, i64 0, i64 4}
+!11 = !{!8, i64 4, !"int"}
+!12 = distinct !{!12, !13, !14}
+!13 = !{!"llvm.loop.mustprogress"}
+!14 = !{!"llvm.loop.unroll.disable"}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 8}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 8, !"p1 int"}
+; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 8, !"any pointer"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]], i64 1, !"omnipotent char"}
+; CHECK: [[META4]] = !{!"Simple C++ TBAA"}
+; CHECK: [[TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0, i64 4}
+; CHECK: [[META6]] = !{[[META3]], i64 4, !"int"}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
+; CHECK: [[META8]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META9]] = !{!"llvm.loop.unroll.disable"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
new file mode 100644
index 0000000000000..915d728090a8e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; LoopVectorize speculative load hoist shouldn't kick in as SCEV cannot compute the trip count of a float-bounded loop, so the
+; pass enters the dynamic-bound path. It then rejects because the
+; branch condition is an FCmpInst, not an ICmpInst.
+;
+; void foo(float *A, float *B, float *Limit) {
+; for (float x = 0.0f; x < *Limit; x += 1.0f)
+; A[(int)x] = B[(int)x];
+; }
+
+define dso_local void @foo(ptr noundef %A, ptr noundef readonly %B, ptr noundef readonly %Limit) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[FIV:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[FIV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[IV]]
+; CHECK-NEXT: [[B:%.*]] = load i32, ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[B]], ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[FIV_NEXT]] = fadd float [[FIV]], 1.000000e+00
+; CHECK-NEXT: [[LIM:%.*]] = load float, ptr [[LIMIT:%.*]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[FIV_NEXT]], [[LIM]]
+; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_EXIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: for.exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %fiv = phi float [ 0.0, %entry ], [ %fiv.next, %for.body ]
+ %gep.b = getelementptr inbounds i32, ptr %B, i64 %iv
+ %b = load i32, ptr %gep.b, align 4
+ %gep.a = getelementptr inbounds i32, ptr %A, i64 %iv
+ store i32 %b, ptr %gep.a, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %fiv.next = fadd float %fiv, 1.0
+ %lim = load float, ptr %Limit, align 4
+ %cmp = fcmp olt float %fiv.next, %lim
+ br i1 %cmp, label %for.body, label %for.exit, !llvm.loop !0
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
new file mode 100644
index 0000000000000..bdb4ae8b96b5f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; LoopVectorize speculative load hoist shouldn't kick in at all if the bound is Len[i], which depends on the
+; induction variable hence not invariant. The dependency chain traces back to a loop header
+; PHI, so the pass will reject this loop instead of versioning it.
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < Len[i]; i++)
+; A[i] = B[i] + C[i];
+; }
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %Len) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[CMP12:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP12]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.cond.cleanup.loopexit:
+; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: ret void
+; CHECK: for.body:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i32, ptr [[C:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX6]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[LEN]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP4]]
+; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
+;
+entry:
+ %0 = load i32, ptr %Len, align 4, !tbaa !5
+ %cmp12 = icmp sgt i32 %0, 0
+ br i1 %cmp12, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx2, align 4, !tbaa !5
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx4, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx6 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx6, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %arrayidx = getelementptr inbounds nuw i32, ptr %Len, i64 %indvars.iv.next
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
new file mode 100644
index 0000000000000..c313f217d4393
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
@@ -0,0 +1,226 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64"
+
+define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %LoopBound) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @_Z3fooPiS_S_(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[LOOPBOUND:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4:![0-9]+]]
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[IVBOUND_RTCHECK]]:
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[FOUND_CONFLICT]], false
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX9:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX9]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[SMAX3]], 2
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
+; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[A]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[B]], [[SCEVGEP4]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX9]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX9]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD11]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD12]], [[WIDE_LOAD10]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP9]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP10]], splat (i32 1)
+; CHECK-NEXT: [[TMP13:%.*]] = add nsw <4 x i32> [[TMP11]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
+; CHECK-NEXT: store <4 x i32> [[TMP13]], ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META8]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX9]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2_LVER:%.*]] = add nsw i32 [[ADD_LVER]], 1
+; CHECK-NEXT: store i32 [[ADD2_LVER]], ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP19]], [[TMP18]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD]], 1
+; CHECK-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+;
+entry:
+ %0 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %cmp6 = icmp sgt i32 %0, 0
+ br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !4
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !4
+ %add = add nsw i32 %2, %1
+ store i32 %add, ptr %arrayidx2, align 4, !tbaa !4
+ %add2 = add nsw i32 %add, 1
+ store i32 %add2, ptr %arrayidx, align 4, !tbaa !4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !8
+}
+
+define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %LoopBound) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local void @bar(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[LOOPBOUND:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD]], 1
+; CHECK-NEXT: store i32 [[ADD2]], ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[TMP2]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP3]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP17]]
+;
+entry:
+ br label %for.body.preheader
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.cond.cleanup, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !4
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !4
+ %add = add nsw i32 %2, %1
+ store i32 %add, ptr %arrayidx2, align 4, !tbaa !4
+ %add2 = add nsw i32 %add, 1
+ store i32 %add2, ptr %arrayidx, align 4, !tbaa !4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !8
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+aes,+crc,+fp-armv8,+neon,+perfmon,+sha2,+v8a,-fmv" }
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!3}
+!llvm.errno.tbaa = !{!4}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"uwtable", i32 2}
+!2 = !{i32 7, !"frame-pointer", i32 4}
+!3 = !{!"clang version 23.0.0git (https://github.com/pawan-nirpal-031/llvm-project.git 93914ac0a0b80824d385c817cdc2d74ec6436b43)"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C++ TBAA"}
+!8 = distinct !{!8, !9}
+!9 = !{!"llvm.loop.mustprogress"}
+;.
+; CHECK: [[INT_TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
+; CHECK: [[META5]] = !{!"int", [[META6:![0-9]+]], i64 0}
+; CHECK: [[META6]] = !{!"omnipotent char", [[META7:![0-9]+]], i64 0}
+; CHECK: [[META7]] = !{!"Simple C++ TBAA"}
+; CHECK: [[META8]] = !{[[META9:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]], [[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], !"LVerDomain"}
+; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META10]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]]}
+; CHECK: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META15]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META14]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META18:![0-9]+]]}
+; CHECK: [[META18]] = !{!"llvm.loop.mustprogress"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
new file mode 100644
index 0000000000000..1ab4ef27079ca
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -0,0 +1,160 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+
+define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
+; CHECK-LABEL: define void @_Z3fooPiS_S_(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[LOOPBOUND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3:![0-9]+]]
+; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP6]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[IVBOUND_RTCHECK]]:
+; CHECK-NEXT: [[DOTRTCBLOCK_LOAD:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
+; CHECK-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX1]], 2
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND2:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND3:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
+; CHECK-NEXT: [[FOUND_CONFLICT1:%.*]] = and i1 [[BOUND2]], [[BOUND3]]
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[FOUND_CONFLICT1]], false
+; CHECK-NEXT: br i1 [[TMP9]], label %[[FOR_BODY_PREHEADER_LVER:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP10]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX3]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP11]], i64 1)
+; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP12]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX3]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX3]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP13]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP14]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META7]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP15]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP15]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10:![0-9]+]], !noalias [[META7]]
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP16]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
+; CHECK-NEXT: [[TMP17:%.*]] = add nsw <4 x i32> [[WIDE_LOAD5]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP18:%.*]] = add nsw <4 x i32> [[WIDE_LOAD6]], [[WIDE_LOAD4]]
+; CHECK-NEXT: store <4 x i32> [[TMP17]], ptr [[TMP15]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
+; CHECK-NEXT: store <4 x i32> [[TMP18]], ptr [[TMP16]], align 4, !tbaa [[INT_TBAA3]], !alias.scope [[META10]], !noalias [[META7]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX3]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP21]], [[TMP20]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP22:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP22]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP24]], [[TMP23]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
+; CHECK-NEXT: [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP26]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+;
+entry:
+ %0 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %cmp6 = icmp sgt i32 %0, 0
+ br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !4
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !4
+ %add = add nsw i32 %2, %1
+ store i32 %add, ptr %arrayidx2, align 4, !tbaa !4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %LoopBound, align 4, !tbaa !4
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !8
+}
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.errno.tbaa = !{!4}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"uwtable", i32 2}
+!2 = !{i32 7, !"frame-pointer", i32 4}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C++ TBAA"}
+!8 = distinct !{!8, !9}
+!9 = !{!"llvm.loop.mustprogress"}
+;.
+; CHECK: [[INT_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
+; CHECK: [[META4]] = !{!"int", [[META5:![0-9]+]], i64 0}
+; CHECK: [[META5]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
+; CHECK: [[META6]] = !{!"Simple C++ TBAA"}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"}
+; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
+; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
+; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META13]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]]}
+; CHECK: [[META17]] = !{!"llvm.loop.mustprogress"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
new file mode 100644
index 0000000000000..72468a2fe233e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+$warypen = comdat any
+
+; Function Attrs: uwtable
+define weak_odr dso_local noundef double @baz(ptr noundef nonnull align 8 dereferenceable(88) %arg, double noundef %arg1) local_unnamed_addr #0 comdat($warypen) {
+; CHECK-LABEL: define weak_odr dso_local noundef double @baz(
+; CHECK-SAME: ptr noundef nonnull align 8 dereferenceable(88) [[ARG:%.*]], double noundef [[ARG1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] comdat($warypen) {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 72
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GETELEMENTPTR]], align 8
+; CHECK-NEXT: [[ICMP:%.*]] = icmp eq i32 [[LOAD]], 0
+; CHECK-NEXT: br i1 [[ICMP]], label %[[BB6:.*]], label %[[BB2:.*]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: [[GETELEMENTPTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[ARG]], i64 80
+; CHECK-NEXT: [[LOAD4:%.*]] = load ptr, ptr [[GETELEMENTPTR3]], align 8, !tbaa [[DOUBLEPTR_TBAA14:![0-9]+]]
+; CHECK-NEXT: br label %[[BB8:.*]]
+; CHECK: [[BB5:.*]]:
+; CHECK-NEXT: [[PHI:%.*]] = phi double [ [[FADD:%.*]], %[[BB8]] ]
+; CHECK-NEXT: br label %[[BB6]]
+; CHECK: [[BB6]]:
+; CHECK-NEXT: [[PHI7:%.*]] = phi double [ 0.000000e+00, %[[BB]] ], [ [[PHI]], %[[BB5]] ]
+; CHECK-NEXT: [[FDIV:%.*]] = fdiv double 1.000000e+00, [[ARG1]]
+; CHECK-NEXT: [[CALL:%.*]] = tail call double @pow(double noundef [[PHI7]], double noundef [[FDIV]]) #[[ATTR3:[0-9]+]], !tbaa [[INT_TBAA10:![0-9]+]]
+; CHECK-NEXT: ret double [[CALL]]
+; CHECK: [[BB8]]:
+; CHECK-NEXT: [[PHI9:%.*]] = phi i64 [ 0, %[[BB2]] ], [ [[ADD:%.*]], %[[BB8]] ]
+; CHECK-NEXT: [[PHI10:%.*]] = phi double [ 0.000000e+00, %[[BB2]] ], [ [[FADD]], %[[BB8]] ]
+; CHECK-NEXT: [[GETELEMENTPTR11:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[LOAD4]], i64 [[PHI9]]
+; CHECK-NEXT: [[LOAD12:%.*]] = load double, ptr [[GETELEMENTPTR11]], align 8
+; CHECK-NEXT: [[CALL13:%.*]] = tail call noundef double @llvm.fabs.f64(double [[LOAD12]])
+; CHECK-NEXT: [[CALL14:%.*]] = tail call double @pow(double noundef [[CALL13]], double noundef [[ARG1]]) #[[ATTR3]], !tbaa [[INT_TBAA10]]
+; CHECK-NEXT: [[FADD]] = fadd double [[PHI10]], [[CALL14]]
+; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[PHI9]], 1
+; CHECK-NEXT: [[LOAD15:%.*]] = load i32, ptr [[GETELEMENTPTR]], align 8
+; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[LOAD15]] to i64
+; CHECK-NEXT: [[ICMP16:%.*]] = icmp samesign ult i64 [[ADD]], [[ZEXT]]
+; CHECK-NEXT: br i1 [[ICMP16]], label %[[BB8]], label %[[BB5]], !llvm.loop [[LOOP30:![0-9]+]]
+;
+bb:
+ %getelementptr = getelementptr inbounds nuw i8, ptr %arg, i64 72
+ %load = load i32, ptr %getelementptr, align 8
+ %icmp = icmp eq i32 %load, 0
+ br i1 %icmp, label %bb6, label %bb2
+
+bb2: ; preds = %bb
+ %getelementptr3 = getelementptr inbounds nuw i8, ptr %arg, i64 80
+ %load4 = load ptr, ptr %getelementptr3, align 8, !tbaa !14
+ br label %bb8
+
+bb5: ; preds = %bb8
+ %phi = phi double [ %fadd, %bb8 ]
+ br label %bb6
+
+bb6: ; preds = %bb5, %bb
+ %phi7 = phi double [ 0.000000e+00, %bb ], [ %phi, %bb5 ]
+ %fdiv = fdiv double 1.000000e+00, %arg1
+ %call = tail call double @pow(double noundef %phi7, double noundef %fdiv) #3, !tbaa !10
+ ret double %call
+
+bb8: ; preds = %bb8, %bb2
+ %phi9 = phi i64 [ 0, %bb2 ], [ %add, %bb8 ]
+ %phi10 = phi double [ 0.000000e+00, %bb2 ], [ %fadd, %bb8 ]
+ %getelementptr11 = getelementptr inbounds nuw [8 x i8], ptr %load4, i64 %phi9
+ %load12 = load double, ptr %getelementptr11, align 8
+ %call13 = tail call noundef double @llvm.fabs.f64(double %load12)
+ %call14 = tail call double @pow(double noundef %call13, double noundef %arg1) #3, !tbaa !10
+ %fadd = fadd double %phi10, %call14
+ %add = add nuw nsw i64 %phi9, 1
+ %load15 = load i32, ptr %getelementptr, align 8
+ %zext = zext i32 %load15 to i64
+ %icmp16 = icmp samesign ult i64 %add, %zext
+ br i1 %icmp16, label %bb8, label %bb5, !llvm.loop !30
+}
+
+; Function Attrs: mustprogress nocallback nofree nounwind willreturn memory(errnomem: write)
+declare double @pow(double noundef, double noundef) local_unnamed_addr #1
+
+; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare double @llvm.fabs.f64(double) #2
+
+attributes #0 = { uwtable "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+aes,+crc,+fp-armv8,+neon,+outline-atomics,+perfmon,+sha2,+v8a,-fmv" }
+attributes #1 = { mustprogress nocallback nofree nounwind willreturn memory(errnomem: write) "frame-pointer"="non-leaf-no-reserve" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+aes,+crc,+fp-armv8,+neon,+outline-atomics,+perfmon,+sha2,+v8a,-fmv" }
+attributes #2 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8}
+!llvm.ident = !{!9}
+!llvm.errno.tbaa = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 23.0.0git (ssh://git-hexagon-lv.qualcomm.com:29418/llvm/llvm-project fa6bb24a9808ef39e1ecebfc7d42c885fca2261f)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cc", directory: "well")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 8, !"PIC Level", i32 2}
+!4 = !{i32 7, !"PIE Level", i32 2}
+!5 = !{i32 7, !"uwtable", i32 2}
+!6 = !{i32 7, !"frame-pointer", i32 4}
+!7 = !{i32 1, !"ThinLTO", i32 0}
+!8 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!9 = !{!"clang version 23.0.0git (ssh://git-hexagon-lv.qualcomm.com:29418/llvm/llvm-project fa6bb24a9808ef39e1ecebfc7d42c885fca2261f)"}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"Simple C++ TBAA"}
+!14 = !{!15, !29, i64 80}
+!15 = !{!"sym1", !16, i64 0, !11, i64 72, !11, i64 76, !29, i64 80}
+!16 = !{!"sym2", !11, i64 8, !17, i64 16, !28, i64 64}
+!17 = !{!"sym3", !18, i64 0}
+!18 = !{!"sym4", !19, i64 0}
+!19 = !{!"sym5", !20, i64 0, !22, i64 8}
+!20 = !{!"sym6", !21, i64 0}
+!21 = !{!"sym7"}
+!22 = !{!"sym8", !23, i64 0, !27, i64 32}
+!23 = !{!"sym9", !24, i64 0, !25, i64 8, !25, i64 16, !25, i64 24}
+!24 = !{!"sym10", !12, i64 0}
+!25 = !{!"p1 sym9", !26, i64 0}
+!26 = !{!"any pointer", !12, i64 0}
+!27 = !{!"long", !12, i64 0}
+!28 = !{!"p1 sym11", !26, i64 0}
+!29 = !{!"p1 double", !26, i64 0}
+!30 = distinct !{!30}
+;.
+; CHECK: [[INT_TBAA10]] = !{[[META11:![0-9]+]], [[META11]], i64 0}
+; CHECK: [[META11]] = !{!"int", [[META12:![0-9]+]], i64 0}
+; CHECK: [[META12]] = !{!"omnipotent char", [[META13:![0-9]+]], i64 0}
+; CHECK: [[META13]] = !{!"Simple C++ TBAA"}
+; CHECK: [[DOUBLEPTR_TBAA14]] = !{[[META15:![0-9]+]], [[META29:![0-9]+]], i64 80}
+; CHECK: [[META15]] = !{!"sym1", [[META16:![0-9]+]], i64 0, [[META11]], i64 72, [[META11]], i64 76, [[META29]], i64 80}
+; CHECK: [[META16]] = !{!"sym2", [[META11]], i64 8, [[META17:![0-9]+]], i64 16, [[META28:![0-9]+]], i64 64}
+; CHECK: [[META17]] = !{!"sym3", [[META18:![0-9]+]], i64 0}
+; CHECK: [[META18]] = !{!"sym4", [[META19:![0-9]+]], i64 0}
+; CHECK: [[META19]] = !{!"sym5", [[META20:![0-9]+]], i64 0, [[META22:![0-9]+]], i64 8}
+; CHECK: [[META20]] = !{!"sym6", [[META21:![0-9]+]], i64 0}
+; CHECK: [[META21]] = !{!"sym7"}
+; CHECK: [[META22]] = !{!"sym8", [[META23:![0-9]+]], i64 0, [[META27:![0-9]+]], i64 32}
+; CHECK: [[META23]] = !{!"sym9", [[META24:![0-9]+]], i64 0, [[META25:![0-9]+]], i64 8, [[META25]], i64 16, [[META25]], i64 24}
+; CHECK: [[META24]] = !{!"sym10", [[META12]], i64 0}
+; CHECK: [[META25]] = !{!"p1 sym9", [[META26:![0-9]+]], i64 0}
+; CHECK: [[META26]] = !{!"any pointer", [[META12]], i64 0}
+; CHECK: [[META27]] = !{!"long", [[META12]], i64 0}
+; CHECK: [[META28]] = !{!"p1 sym11", [[META26]], i64 0}
+; CHECK: [[META29]] = !{!"p1 double", [[META26]], i64 0}
+; CHECK: [[LOOP30]] = distinct !{[[LOOP30]]}
+;.
>From ec754166819516a776e92cbbe307523aa1796949 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Thu, 30 Apr 2026 01:03:18 -0700
Subject: [PATCH 3/6] clang-format and comments
---
.../Vectorize/LoopVectorizationLegality.h | 2 +-
.../Vectorize/LoopVectorizationLegality.cpp | 2 +-
.../lib/Transforms/Vectorize/LoopVectorize.cpp | 18 ++++++++++--------
3 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 7cc5f037ff90f..21a3b11cad04d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -350,7 +350,7 @@ class LoopVectorizationLegality {
/// reductions found in the loop.
bool isInvariantStoreOfReduction(StoreInst *SI);
- LoadInst* tryToFindDyanmicBoundLoadCandidate(Loop *L, AAResults &AA);
+ LoadInst *tryToFindDyanmicBoundLoadCandidate(Loop *L, AAResults &AA);
/// Returns True if given address is invariant and is used to store recurrent
/// expression
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 4c141782007e8..894279c08cc0c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -676,7 +676,7 @@ static bool isLoadSpeculativelyHoistable(Loop *L, LoadInst *LoadCandidate,
LoadInst *
LoopVectorizationLegality::tryToFindDyanmicBoundLoadCandidate(Loop *L,
- AAResults &AA) {
+ AAResults &AA) {
if (!L->isInnermost())
return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0614b13066f7f..bb8b23b8947ec 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -136,6 +136,7 @@
#include "llvm/Support/NativeFormatting.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
@@ -144,7 +145,6 @@
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
-#include "llvm/Transforms/Utils/Cloning.h"
#include <algorithm>
#include <cassert>
#include <cmath>
@@ -8166,9 +8166,10 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi.eraseFromParent();
}
+// Is profitable to generate runtime check dominated version of the loop? which
+// has speculatively hoisted the loop bound load.
static bool isSpeculativeBoundVersioningProfitable() { return true; }
-
/*
preheader
@@ -8239,7 +8240,6 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
// Fix the preheader phis via rtcblock phis.
for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
- // Create the rtcblock phi.
PHINode *RtcPN =
PHINode::Create(PN.getType(), PreHeaderPreds.size(),
PN.getName() + ".rtcphi", &RtCheckBB->front());
@@ -8391,16 +8391,18 @@ bool LoopVectorizePass::processLoop(Loop *L) {
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
- if (AllowInvariantBoundRetry) {
- // There has to be a better way to do this? Trying to avoid changing processLoop Signature.
+ if (AllowInvariantBoundRetry) {
+ // There has to be a better way to do this? Trying to avoid changing
+ // processLoop Signature.
AllowInvariantBoundRetry = false;
const DataLayout &DL = F->getDataLayout();
// 1) Check if the loop qualifies for loop invariant bound load.
- // 2) Generate the versioned loop with runtime checks using the dynamic loop bound.
- // load. Return this versioned loop to attempt vectorization
+ // 2) Generate the versioned loop with runtime checks using the dynamic
+ // loop bound. load. Return this versioned loop to attempt vectorization
// again. 3) If the versioned loop is generated successfully, re-try
// vectorization on the versioned loop.
- if (LoadInst *BoundLoad = LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
+ if (LoadInst *BoundLoad =
+ LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
if (isSpeculativeBoundVersioningProfitable())
if (Loop *Cand = tryToVersionLoopForInvariantBoundLoad(
L, BoundLoad, DL, *DT, *LI, *AA, PSE, AC)) {
>From 74191cee9dc10c34aed2a975a688618f6dfb6e1c Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Thu, 30 Apr 2026 01:33:35 -0700
Subject: [PATCH 4/6] use iteartors instead of raw pointers for creating instrs
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb8b23b8947ec..41117a13aef58 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8242,7 +8242,7 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
PHINode *RtcPN =
PHINode::Create(PN.getType(), PreHeaderPreds.size(),
- PN.getName() + ".rtcphi", &RtCheckBB->front());
+ PN.getName() + ".rtcphi", RtCheckBB->begin());
for (unsigned i = 0; i < PN.getNumIncomingValues(); i++) {
BasicBlock *IBB = PN.getIncomingBlock(i);
assert(is_contained(PreHeaderPreds, IBB) &&
@@ -8272,7 +8272,7 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
LoadInst *HoistLoad = cast<LoadInst>(InnerLoad->clone());
HoistLoad->setName(InnerLoad->getName() + ".speculatively.hoisted");
// Put this hoisted load in RtcheckBB, since SCEVExapnder needs it.
- HoistLoad->insertBefore(TmpTerm);
+ HoistLoad->insertBefore(TmpTerm->getIterator());
Instruction *ClonedInnerLoad = cast<Instruction>(VMap[InnerLoad]);
ClonedInnerLoad->replaceAllUsesWith(HoistLoad);
ClonedInnerLoad->eraseFromParent();
>From 4137209acc2f0afccaf3fa1a0f1446c13c622dfe Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Fri, 1 May 2026 01:37:44 -0700
Subject: [PATCH 5/6] use metadata to re-attempt vectoriztion
---
.../Transforms/Vectorize/LoopVectorize.cpp | 44 ++++++++++++++++---
1 file changed, 38 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 41117a13aef58..03b6b0bc2decf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -379,8 +379,6 @@ static cl::opt<bool> EnableEarlyExitVectorization(
// after prolog. See `emitIterationCountCheck`.
static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
-static bool AllowInvariantBoundRetry = true;
-
/// A helper function that returns true if the given type is irregular. The
/// type is irregular if its allocated size doesn't equal the store size of an
/// element of the corresponding vector type.
@@ -8336,6 +8334,28 @@ static Loop *tryToVersionLoopForInvariantBoundLoad(
DT.insertEdge(RtCheckBB, PreHeader);
DT.insertEdge(RtCheckBB, VerPreHeader);
+ assert(VerLatch &&
+ "Versioned loop must have a single latch, loop not in canonical form");
+
+ // Emit metadata llvm.loop.speculative.bound.hoist.versioned to prevent
+ // repeted recursive calls with same loop to processLoop.
+ Instruction *VerLatchTerm = VerLatch->getTerminator();
+ LLVMContext &ctx = VerLatchTerm->getContext();
+ MDNode *Marker = MDNode::get(
+ ctx, {(Metadata *)MDString::get(
+ Ctx, "llvm.loop.speculative.bound.hoist.versioned")});
+ MDNode *ExistingMD = VerLatchTerm->getMetadata(LLVMContext::MD_loop);
+ SmallVector<Metadata *, 4> Mds;
+ if (ExistingMD)
+ Mds.append(ExistingMD->op_begin(), ExistingMD->op_end());
+ else
+ Mds.push_back(nullptr);
+ Mds.push_back(Marker);
+ MDNode *NewLoopMD = MDNode::getDistinct(ctx, Mds);
+ if (!ExistingMD)
+ NewLoopMD->replaceOperandWith(0, NewLoopMD);
+ VerLatchTerm->setMetadata(LLVMContext::MD_loop, NewLoopMD);
+
return VerLoop;
}
@@ -8391,10 +8411,22 @@ bool LoopVectorizePass::processLoop(Loop *L) {
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
- if (AllowInvariantBoundRetry) {
- // There has to be a better way to do this? Trying to avoid changing
- // processLoop Signature.
- AllowInvariantBoundRetry = false;
+ bool VisitedSpeculativeHoist = false;
+ if (BasicBlock *Latch = L->getLoopLatch()) {
+ if (MDNode *LoopMD =
+ Latch->getTerminator()->getMetadata(LLVMContext::MD_loop)) {
+ for (unsigned i = 1, e = LoopMD->getNumOperands(); i < e; ++i) {
+ if (auto *Op = dyn_cast<MDNode>(LoopMD->getOperand(i)))
+ if (Op->getNumOperands() > 0)
+ if (auto *S = dyn_cast<MDString>(Op->getOperand(0)))
+ if (S->getString() ==
+ "llvm.loop.speculative.bound.hoist.versioned")
+ VisitedSpeculativeHoist = true;
+ }
+ }
+ }
+
+ if (!VisitedSpeculativeHoist) {
const DataLayout &DL = F->getDataLayout();
// 1) Check if the loop qualifies for loop invariant bound load.
// 2) Generate the versioned loop with runtime checks using the dynamic
>From d8e04099ea52be95b9a7be549bd651aa7feb0ee2 Mon Sep 17 00:00:00 2001
From: Pawan Nirpal <pnirpal at qti.qualcomm.com>
Date: Mon, 4 May 2026 13:42:18 +0530
Subject: [PATCH 6/6] test fixes
---
.../dynamic-bound-array-element.ll | 19 +--
.../LoopVectorize/dynamic-bound-n-stores.ll | 108 ++++++++++++++++--
.../LoopVectorize/dynamic-bound-simple.ll | 17 +--
3 files changed, 115 insertions(+), 29 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
index 78d4431437138..2729e5d8c5cde 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -97,7 +97,7 @@ define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
@@ -121,7 +121,7 @@ define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]]
;
entry:
%arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
@@ -167,11 +167,12 @@ attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: r
; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
-; CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]], [[META10:![0-9]+]]}
-; CHECK: [[META9]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META10]] = !{!"llvm.loop.unroll.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; CHECK: [[META5]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]], [[META6]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]]}
+; CHECK: [[META10]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META11]] = !{!"llvm.loop.unroll.disable"}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
index c313f217d4393..0146089151411 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll
@@ -82,7 +82,7 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP17]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
@@ -107,7 +107,7 @@ define dso_local void @_Z3fooPiS_S_(ptr noundef captures(none) %A, ptr noundef r
; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP21]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]]
;
entry:
%0 = load i32, ptr %LoopBound, align 4, !tbaa !4
@@ -146,11 +146,87 @@ define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY_PREHEADER:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP17]], i64 1)
+; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP18]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP18]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[FOUND_CONFLICT]], false
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER1:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX9:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP19]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX9]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX3:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP20]], i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[SMAX3]], 2
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[A]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[B]], [[SCEVGEP4]]
+; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX9]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX9]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP:.*]]:
-; CHECK-NEXT: ret void
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP5]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23:![0-9]+]], !noalias [[META20]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
+; CHECK-NEXT: [[TMP9:%.*]] = add nsw <4 x i32> [[WIDE_LOAD11]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD12]], [[WIDE_LOAD10]]
+; CHECK-NEXT: store <4 x i32> [[TMP9]], ptr [[TMP7]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP8]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META23]], !noalias [[META20]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[TMP9]], splat (i32 1)
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP10]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP5]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
+; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP6]], align 4, !tbaa [[INT_TBAA4]], !alias.scope [[META20]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX9]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[FOR_COND_CLEANUP]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ARRAYIDX2_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD_LVER:%.*]] = add nsw i32 [[TMP15]], [[TMP14]]
+; CHECK-NEXT: store i32 [[ADD_LVER]], ptr [[ARRAYIDX2_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[ADD2_LVER:%.*]] = add nsw i32 [[ADD_LVER]], 1
+; CHECK-NEXT: store i32 [[ADD2_LVER]], ptr [[ARRAYIDX_LVER]], align 4, !tbaa [[INT_TBAA4]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP16]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT1]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK: [[FOR_BODY_PREHEADER1]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP1:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP1]]
+; CHECK: [[FOR_COND_CLEANUP1]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY1]] ], [ 0, %[[FOR_BODY_PREHEADER1]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
@@ -163,7 +239,7 @@ define dso_local void @bar(ptr noundef captures(none) %A, ptr noundef readonly c
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[TMP2]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP3]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP17]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY1]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP18]]
;
entry:
br label %for.body.preheader
@@ -217,10 +293,18 @@ attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: r
; CHECK: [[META10]] = distinct !{[[META10]], !"LVerDomain"}
; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
; CHECK: [[META12]] = distinct !{[[META12]], [[META10]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]]}
-; CHECK: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META15]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META14]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META18:![0-9]+]]}
-; CHECK: [[META18]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]], [[META16:![0-9]+]]}
+; CHECK: [[META14]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META15]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META16]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META14]], [[META15]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META19:![0-9]+]]}
+; CHECK: [[META19]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META20]] = !{[[META21:![0-9]+]]}
+; CHECK: [[META21]] = distinct !{[[META21]], [[META22:![0-9]+]]}
+; CHECK: [[META22]] = distinct !{[[META22]], !"LVerDomain"}
+; CHECK: [[META23]] = !{[[META24:![0-9]+]]}
+; CHECK: [[META24]] = distinct !{[[META24]], [[META22]]}
+; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META14]], [[META15]], [[META16]]}
+; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META14]], [[META15]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
index 1ab4ef27079ca..537dd105f5d67 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -75,7 +75,7 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK-NEXT: [[INDVARS_IV_NEXT_LVER]] = add nuw nsw i64 [[INDVARS_IV_LVER]], 1
; CHECK-NEXT: [[TMP22:%.*]] = sext i32 [[DOTRTCBLOCK_LOAD]] to i64
; CHECK-NEXT: [[CMP_LVER:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_LVER]], [[TMP22]]
-; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP_LVER]], label %[[FOR_BODY_LVER]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT:.*]]:
@@ -98,7 +98,7 @@ define void @_Z3fooPiS_S_(ptr %A, ptr %B, ptr %LoopBound) {
; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[LOOPBOUND]], align 4, !tbaa [[INT_TBAA3]]
; CHECK-NEXT: [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP26]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP17:![0-9]+]]
;
entry:
%0 = load i32, ptr %LoopBound, align 4, !tbaa !4
@@ -151,10 +151,11 @@ for.body: ; preds = %for.body.preheader,
; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"}
; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META14:![0-9]+]]}
-; CHECK: [[META13]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META14]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META13]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META17:![0-9]+]]}
-; CHECK: [[META17]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META13:![0-9]+]], [[META14:![0-9]+]], [[META15:![0-9]+]]}
+; CHECK: [[META13]] = !{!"llvm.loop.speculative.bound.hoist.versioned"}
+; CHECK: [[META14]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META15]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META13]], [[META14]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META18:![0-9]+]]}
+; CHECK: [[META18]] = !{!"llvm.loop.mustprogress"}
;.
More information about the llvm-commits
mailing list