[llvm] [LoopVectorize] Enable vectorization of uncountable loops with invariant bound loads via speculative hoisting and runtime versioning. (PR #195027)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 00:36:10 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Pawan Nirpal (pawan-nirpal-031)
<details>
<summary>Changes</summary>
Loops whose trip count depends on a value loaded through a pointer are a common pattern:
void foo(int *A, int *B, int *LoopBound) {
for (int k = 0; k < *LoopBound; k++)
A[k] += B[k];
}
Because *LoopBound may alias A or B, the compiler cannot prove the bound is loop-invariant and must treat the loop as uncountable, preventing vectorization entirely.
This patch extends LoopVectorizer to handle this pattern via versioning. When the bound load is identified as a candidate non-volatile, its pointer dominating the loop preheader, and all store base pointers dominating both loop versions LV emits a versioned CFG. A new ivbound.rtcheck block speculatively hoists the bound load and uses LoopAccessAnalysis to generate SCEV-based runtime alias checks between the bound pointer and all store ranges. If no conflict is detected at runtime, control transfers to a fully vectorized loop using the hoisted bound value; if a conflict is detected, execution falls back to the original scalar loop which re-reads the bound on every iteration.
---
Patch is 70.79 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/195027.diff
10 Files Affected:
- (modified) llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h (+2)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+117)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+194)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll (+177)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll (+99)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll (+56)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll (+79)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-n-stores.ll (+226)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll (+160)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-unsafe-calls.ll (+150)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 49e8bb1e85526..7cc5f037ff90f 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -350,6 +350,8 @@ class LoopVectorizationLegality {
/// reductions found in the loop.
bool isInvariantStoreOfReduction(StoreInst *SI);
+ LoadInst* tryToFindDyanmicBoundLoadCandidate(Loop *L, AAResults &AA);
+
/// Returns True if given address is invariant and is used to store recurrent
/// expression
bool isInvariantAddressOfReduction(Value *V);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index bee08eeba9927..4c141782007e8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -579,6 +579,123 @@ class SCEVAddRecForUniformityRewriter
} // namespace
+static Value *stripSimpleCasts(Value *V) {
+ while (isa<SExtInst>(V) || isa<ZExtInst>(V) || isa<TruncInst>(V))
+ V = cast<Instruction>(V)->getOperand(0);
+ return V;
+}
+
+static LoadInst *getLoopBoundLoadCandidate(Loop *L) {
+ Instruction *Icmp = L->getLatchCmpInst();
+ if (!Icmp)
+ return nullptr;
+
+ Value *Op0 = stripSimpleCasts(Icmp->getOperand(0));
+ Value *Op1 = stripSimpleCasts(Icmp->getOperand(1));
+
+ if (!isa<LoadInst>(Op0) && !isa<LoadInst>(Op1))
+ return nullptr;
+
+ LoadInst *BoundLoad =
+ isa<LoadInst>(Op0) ? dyn_cast<LoadInst>(Op0) : dyn_cast<LoadInst>(Op1);
+
+ // Reject non int load types.
+ if (!BoundLoad->getType()->isIntegerTy())
+ return nullptr;
+
+ return BoundLoad;
+}
+
+static bool isLoadSpeculativelyHoistable(Loop *L, LoadInst *LoadCandidate,
+ PredicatedScalarEvolution &PSE,
+ AAResults &AA) {
+
+ if (!L || !LoadCandidate)
+ return false;
+
+ if (!L->contains(LoadCandidate))
+ return false;
+
+ Value *Ptr = LoadCandidate->getPointerOperand();
+ if (!L->isLoopInvariant(Ptr))
+ return false;
+
+ // We don't want to deal with global pointers for now.
+ if (isa<GlobalValue>(Ptr))
+ return false;
+
+ if (!LoadCandidate->isSimple() || LoadCandidate->isVolatile())
+ return false;
+
+ const SCEV *PtrScev = PSE.getSCEV(Ptr);
+ if (!PtrScev || !PSE.getSE()->isLoopInvariant(PtrScev, L))
+ return false;
+
+ for (auto *User : Ptr->users()) {
+ Instruction *UserI = dyn_cast<Instruction>(User);
+ if (!UserI || UserI == dyn_cast<Instruction>(LoadCandidate))
+ continue;
+
+ if (!L->contains(UserI))
+ continue;
+
+ if (UserI->mayWriteToMemory())
+ return false;
+ }
+
+ MemoryLocation LoadLoc = MemoryLocation::get(LoadCandidate);
+ // No call in loop may write to Ptr via aliasing.
+ for (BasicBlock *BB : L->blocks()) {
+ for (Instruction &I : *BB) {
+ auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB)
+ continue;
+
+ MemoryEffects ME = AA.getMemoryEffects(CB);
+ // If call doesn't write anything it's safe.
+ if (!isModSet(ME.getModRef()))
+ continue;
+
+ // If call writes to escaped or global memory reject.
+ if (isModSet(ME.getModRef(IRMemLocation::Other)))
+ return false;
+
+ // Check if any args alias with loadloc.
+ for (Value *Arg : CB->args()) {
+ if (!Arg->getType()->isPointerTy())
+ continue;
+ if (AA.alias(MemoryLocation::getBeforeOrAfter(Arg), LoadLoc) !=
+ AliasResult::NoAlias)
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+LoadInst *
+LoopVectorizationLegality::tryToFindDyanmicBoundLoadCandidate(Loop *L,
+ AAResults &AA) {
+ if (!L->isInnermost())
+ return nullptr;
+
+ LoadInst *LoadCandidate = getLoopBoundLoadCandidate(L);
+ if (!LoadCandidate)
+ return nullptr;
+
+ // Reject if loop body may have some non-vectorizable calls, like pow/ instrs
+ // with side-effects, here even if the load were hoisted it would not allow
+ // the loop to vectorize anyway.
+ if (!canVectorizeInstrs())
+ return nullptr;
+
+ if (!isLoadSpeculativelyHoistable(L, LoadCandidate, PSE, AA))
+ return nullptr;
+
+ return LoadCandidate;
+}
+
bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const {
if (isInvariant(V))
return true;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6c9298a2cf98d..0614b13066f7f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -144,6 +144,7 @@
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
+#include "llvm/Transforms/Utils/Cloning.h"
#include <algorithm>
#include <cassert>
#include <cmath>
@@ -378,6 +379,8 @@ static cl::opt<bool> EnableEarlyExitVectorization(
// after prolog. See `emitIterationCountCheck`.
static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
+static bool AllowInvariantBoundRetry = true;
+
/// A helper function that returns true if the given type is irregular. The
/// type is irregular if its allocated size doesn't equal the store size of an
/// element of the corresponding vector type.
@@ -8163,6 +8166,179 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi.eraseFromParent();
}
+static bool isSpeculativeBoundVersioningProfitable() { return true; }
+
+
+/*
+
+ preheader
+ |
+ rtcheck-block
+ / \
+ / \
+ for.preheader ver.preheader
+ | |
+ for.body ver.for.body
+
+*/
+static Loop *tryToVersionLoopForInvariantBoundLoad(
+ Loop *L, LoadInst *BoundLoad, const DataLayout &DL, DominatorTree &DT,
+ LoopInfo &LI, AAResults &AA, PredicatedScalarEvolution &PSE,
+ AssumptionCache *AC) {
+ LoadInst *InnerLoad = BoundLoad;
+ if (!InnerLoad)
+ return nullptr;
+ auto *PreHeader = L->getLoopPreheader();
+ if (!PreHeader)
+ return nullptr;
+
+ Value *BoundPtrOl = InnerLoad->getPointerOperand();
+ if (auto *BoundPtrI = dyn_cast<Instruction>(BoundPtrOl))
+ if (!DT.properlyDominates(BoundPtrI->getParent(), PreHeader))
+ return nullptr;
+
+ // All the relevent base pointers of participating stores should dominate both
+ // loops.
+ LoopAccessInfoManager PreFlightLAIs(*PSE.getSE(), AA, DT, LI, nullptr,
+ nullptr, AC);
+ const LoopAccessInfo &PreLAI = PreFlightLAIs.getInfo(*L);
+ const auto &RtPtrChecking = *PreLAI.getRuntimePointerChecking();
+
+ for (const auto &Check : RtPtrChecking.getChecks()) {
+ for (const RuntimeCheckingPtrGroup *Group : {Check.first, Check.second}) {
+ for (unsigned Idx : Group->Members) {
+ Value *Ptr = RtPtrChecking.Pointers[Idx].PointerValue;
+ Value *Base = getUnderlyingObject(Ptr);
+ if (auto *BaseI = dyn_cast<Instruction>(Base))
+ if (!DT.properlyDominates(BaseI->getParent(), PreHeader)) {
+ return nullptr;
+ }
+ }
+ }
+ }
+
+ Function *ParentF = PreHeader->getParent();
+ LLVMContext &Ctx = ParentF->getContext();
+
+ BasicBlock *RtCheckBB =
+ BasicBlock::Create(Ctx, "ivbound.rtcheck", ParentF, PreHeader);
+
+ // Temporary terminator for SCEVExapnder to have valid insert point.
+ IRBuilder<> TmpBuilder(RtCheckBB);
+ Instruction *TmpTerm = TmpBuilder.CreateUnreachable();
+
+ // Re-wire the CFG.
+ // Redirect the predecessors of preheader the new rtcheck block.
+ SmallVector<BasicBlock *, 4> PreHeaderPreds(predecessors(PreHeader));
+ for (auto *Pred : PreHeaderPreds)
+ Pred->getTerminator()->replaceUsesOfWith(PreHeader, RtCheckBB);
+
+ BasicBlock *OldPreheadIdom = DT.getNode(PreHeader)->getIDom()->getBlock();
+ DT.addNewBlock(RtCheckBB, OldPreheadIdom);
+ DT.changeImmediateDominator(PreHeader, RtCheckBB);
+
+ // Fix the preheader phis via rtcblock phis.
+ for (PHINode &PN : make_early_inc_range(PreHeader->phis())) {
+ // Create the rtcblock phi.
+ PHINode *RtcPN =
+ PHINode::Create(PN.getType(), PreHeaderPreds.size(),
+ PN.getName() + ".rtcphi", &RtCheckBB->front());
+ for (unsigned i = 0; i < PN.getNumIncomingValues(); i++) {
+ BasicBlock *IBB = PN.getIncomingBlock(i);
+ assert(is_contained(PreHeaderPreds, IBB) &&
+ "Incoming Basic Block not part of predecessors list!");
+ RtcPN->addIncoming(PN.getIncomingValue(i), IBB);
+ }
+ PN.replaceAllUsesWith(RtcPN);
+ PN.eraseFromParent();
+ }
+
+ // Clone the loop including preheader.
+ ValueToValueMapTy VMap;
+ SmallVector<BasicBlock *, 4> NewBlocks;
+ Loop *VerLoop = cloneLoopWithPreheader(PreHeader, RtCheckBB, L, VMap, ".lver",
+ &LI, &DT, NewBlocks);
+
+ remapInstructionsInBlocks(NewBlocks, VMap);
+
+ BasicBlock *VerPreHeader = cast<BasicBlock>(VMap[PreHeader]);
+ BasicBlock *VerLatch = cast<BasicBlock>(VMap[L->getLoopLatch()]);
+
+ if (Loop *ParLoop = L->getParentLoop())
+ ParLoop->addBasicBlockToLoop(RtCheckBB, LI);
+
+ VerLatch->getTerminator()->setMetadata(LLVMContext::MD_loop, nullptr);
+
+ LoadInst *HoistLoad = cast<LoadInst>(InnerLoad->clone());
+ HoistLoad->setName(InnerLoad->getName() + ".speculatively.hoisted");
+ // Put this hoisted load in RtcheckBB, since SCEVExapnder needs it.
+ HoistLoad->insertBefore(TmpTerm);
+ Instruction *ClonedInnerLoad = cast<Instruction>(VMap[InnerLoad]);
+ ClonedInnerLoad->replaceAllUsesWith(HoistLoad);
+ ClonedInnerLoad->eraseFromParent();
+
+ // fix lcssa phis.
+ SmallVector<BasicBlock *, 4> ExitBlocks;
+ L->getExitBlocks(ExitBlocks);
+ for (BasicBlock *ExBB : ExitBlocks) {
+ DT.changeImmediateDominator(ExBB, RtCheckBB);
+ for (PHINode &Phi : ExBB->phis()) {
+ int idx = Phi.getBasicBlockIndex(L->getLoopLatch());
+ if (idx < 0)
+ continue;
+ Value *OrigVal = Phi.getIncomingValue(idx);
+ Value *NewVal =
+ VMap.count(OrigVal) ? cast<Value>(VMap[OrigVal]) : OrigVal;
+ Phi.addIncoming(NewVal, VerLatch);
+ }
+ }
+
+ formDedicatedExitBlocks(L, &DT, &LI, nullptr, true);
+ formDedicatedExitBlocks(VerLoop, &DT, &LI, nullptr, true);
+
+ // RTC condition generation using SCEV.
+ auto &SE = *PSE.getSE();
+ LoopAccessInfoManager NewLAIs(SE, AA, DT, LI, nullptr, nullptr, AC);
+ const LoopAccessInfo &VerLAI = NewLAIs.getInfo(*VerLoop);
+
+ // Generate LAA based ptr runtime checks.
+ SCEVExpander MemExp(*VerLAI.getRuntimePointerChecking()->getSE(),
+ "ivbound.mem.check");
+ Value *MemRTC = nullptr;
+ if (VerLAI.getRuntimePointerChecking()->Need)
+ MemRTC = addRuntimeChecks(TmpTerm, VerLoop,
+ VerLAI.getRuntimePointerChecking()->getChecks(),
+ MemExp);
+
+ // Generate SCEV predicate runtime checks.
+ SCEVExpander SCEVEx(SE, "ivbound.scev.check");
+ const SCEVPredicate &Preds = VerLAI.getPSE().getPredicate();
+ Value *SCEVrtc = SCEVEx.expandCodeForPredicate(&Preds, TmpTerm);
+
+ Value *RtCheckCond = nullptr;
+ if (MemRTC && SCEVrtc) {
+ IRBuilder<> Builder(TmpTerm);
+ RtCheckCond = Builder.CreateOr(MemRTC, SCEVrtc, "ivbound.safe");
+ } else {
+ RtCheckCond = MemRTC ? MemRTC : SCEVrtc;
+ }
+
+ TmpTerm->eraseFromParent();
+ IRBuilder<> Builder(RtCheckBB);
+
+ // If No runtime checks needed then ver-loop is unconditionally safe.
+ // Use constant false to disregard orignal loop path.
+ if (!RtCheckCond)
+ RtCheckCond = ConstantInt::getFalse(Ctx);
+
+ Builder.CreateCondBr(RtCheckCond, PreHeader, VerPreHeader);
+ // RTCheck now idom's both preheaders.
+ DT.insertEdge(RtCheckBB, PreHeader);
+ DT.insertEdge(RtCheckBB, VerPreHeader);
+
+ return VerLoop;
+}
+
bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&
"VPlan-native path is not enabled. Only process inner loops.");
@@ -8215,6 +8391,24 @@ bool LoopVectorizePass::processLoop(Loop *L) {
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
+ if (AllowInvariantBoundRetry) {
+ // There has to be a better way to do this? Trying to avoid changing processLoop Signature.
+ AllowInvariantBoundRetry = false;
+ const DataLayout &DL = F->getDataLayout();
+ // 1) Check if the loop qualifies for loop invariant bound load.
+ // 2) Generate the versioned loop with runtime checks using the dynamic loop bound.
+ // load. Return this versioned loop to attempt vectorization
+ // again. 3) If the versioned loop is generated successfully, re-try
+ // vectorization on the versioned loop.
+ if (LoadInst *BoundLoad = LVL.tryToFindDyanmicBoundLoadCandidate(L, *AA)) {
+ if (isSpeculativeBoundVersioningProfitable())
+ if (Loop *Cand = tryToVersionLoopForInvariantBoundLoad(
+ L, BoundLoad, DL, *DT, *LI, *AA, PSE, AC)) {
+ SE->forgetLoop(L);
+ return processLoop(Cand);
+ }
+ }
+ }
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
Hints.emitRemarkWithHints();
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
new file mode 100644
index 0000000000000..78d4431437138
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes='loop-vectorize' | FileCheck %s
+;
+; TEST10: Loop bound loaded from an array element A[2], where A is also
+; written inside the loop. The load and store use different indices, so
+; pointer identity does not flag a conflict, but runtime checks will
+; guard correctness.
+; Verify the load is hoisted and runtime checks are generated.
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < A[2]; i++)
+; A[i] = B[i] + C[i];
+; }
+;
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) #0 {
+; CHECK-LABEL: define dso_local void @foo(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C9:%.*]] = ptrtoaddr ptr [[C]] to i64
+; CHECK-NEXT: [[B8:%.*]] = ptrtoaddr ptr [[B]] to i64
+; CHECK-NEXT: [[A7:%.*]] = ptrtoaddr ptr [[A]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[IVBOUND_RTCHECK:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[IVBOUND_RTCHECK]]:
+; CHECK-NEXT: [[DOTSPECULATIVELY_HOISTED:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT: [[IVBOUND_SAFE:%.*]] = or i1 [[CONFLICT_RDX]], false
+; CHECK-NEXT: br i1 [[IVBOUND_SAFE]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY_PREHEADER_LVER:.*]]
+; CHECK: [[FOR_BODY_PREHEADER_LVER]]:
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTSPECULATIVELY_HOISTED]] to i64
+; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP3]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[A7]], [[B8]]
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 32
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[A7]], [[C9]]
+; CHECK-NEXT: [[DIFF_CHECK10:%.*]] = icmp ult i64 [[TMP5]], 32
+; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK10]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP6]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP8]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD14]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD15]], [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP12]], i64 4
+; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT_LOOPEXIT1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER_LVER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_LVER:.*]]
+; CHECK: [[FOR_BODY_LVER]]:
+; CHECK-NEXT: [[INDVARS_IV_LVER:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_LVER:%.*]], %[[FOR_BODY_LVER]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX1_LVER:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_LVER]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1_LVER]], align 4, !tbaa [[TBAA...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/195027
More information about the llvm-commits
mailing list