[llvm] [LV][NFC] Emit remarks when failing to vectorise early exit loops (PR #225078)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 02:33:56 PDT 2026
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/225078
>From c4361f6ed5a2358f69e47c0e74220da6251a47ff Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Mon, 21 Sep 2026 12:53:43 +0000
Subject: [PATCH 1/3] [LV][NFC] Emit remarks when failing to vectorise early
exit loops
I spotted recently that when attempting to compile a function with
an early exit loop we don't emit any remarks when we fail to
vectorise due to possible faulting loads. This PR fixes that and
other missing remarks, plus I've also added missing tests for some
cases where we fail to vectorise early exit loops.
For the remarks, I've used debug information to uniquely identify
the remark and associate it with a particular function.
Tests assisted by Codex (GPT 5.5)
---
.../Transforms/Vectorize/LoopVectorize.cpp | 5 +-
.../Vectorize/VPlanConstruction.cpp | 3 -
.../Transforms/Vectorize/VPlanTransforms.cpp | 59 +-
.../Transforms/Vectorize/VPlanTransforms.h | 7 +-
.../LoopVectorize/early_exit_legality.ll | 299 ++++++----
.../early_exit_store_legality.ll | 548 ++++++++++++------
6 files changed, 600 insertions(+), 321 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5c4e20ee66b65a..a6bdb6a15ea8ea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6439,9 +6439,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
? UncountableExitStyle::MaskedHandleExitInScalarLoop
: UncountableExitStyle::ReadOnly;
if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
- OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
- EEStyle))
+ ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
+ EEStyle)) {
return nullptr;
+ }
} else {
RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4cd455adb7a902..dac58befe7e521 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1251,9 +1251,6 @@ bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
TheLoop, SE, DT, AC, &Preds))
continue;
- LLVM_DEBUG(
- dbgs() << "LV: Not vectorizing: Auto-vectorization of loops with "
- "potentially faulting load is not supported.\n");
return false;
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d78a3d3b1e3443..a8a92de5e64cdd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -41,6 +41,7 @@
#include "llvm/Transforms/Utils/LoopUtils.h"
using namespace llvm;
+using namespace LoopVectorizationUtils;
using namespace VPlanPatternMatch;
using namespace SCEVPatternMatch;
@@ -3187,8 +3188,8 @@ struct EarlyExitInfo {
static bool handleUncountableExitsWithSideEffects(
VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
- Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT,
- AssumptionCache *AC) {
+ OptimizationRemarkEmitter *ORE, Loop *TheLoop,
+ PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC) {
// Disconnect early exiting blocks from successors, remove branches. We
// currently don't support multiple uses for recipes involved in creating
@@ -3211,8 +3212,13 @@ static bool handleUncountableExitsWithSideEffects(
SmallVector<VPInstruction *, 8> ConditionRecipes;
VPValue *Cond = getRecipesForUncountableExit(ConditionRecipes, LatchVPBB);
- if (!Cond)
+ if (!Cond) {
+ reportVectorizationFailure(
+ "Unable to determine early exit condition for loop with side effects.\n",
+ "EarlyExitSideEffectsCond",
+ ORE, TheLoop);
return false;
+ }
// Find load contributing to condition.
// At the moment LoopVectorizationLegality only supports a single
@@ -3246,8 +3252,13 @@ static bool handleUncountableExitsWithSideEffects(
if (!isDereferenceableAndAlignedInLoop(
PtrSCEV, cast<LoadInst>(Load->getUnderlyingInstr())->getAlign(),
PSE.getSE()->getConstant(EltSize), TheLoop, *PSE.getSE(), DT, AC,
- &Predicates))
+ &Predicates)) {
+ reportVectorizationFailure("Early exit loop with side effects contains "
+ "critical load that may fault.\n",
+ "EarlyExitSideEffectsFaultingLoad",
+ ORE, TheLoop);
return false;
+ }
}
// Check for a single GEP for the condition load to see if we can link it to
@@ -3257,9 +3268,14 @@ static bool handleUncountableExitsWithSideEffects(
if (!match(IV->getStartValue(), m_SpecificInt(0)) ||
!match(IV->getStepValue(), m_SpecificInt(1)))
return false;
- if (!match(Ptr, m_VPInstruction<Instruction::GetElementPtr>(m_LiveIn(),
- m_Specific(IV))))
+ if (!match(Ptr, m_VPInstruction<Instruction::GetElementPtr>(
+ m_LiveIn(), m_Specific(IV)))) {
+ reportVectorizationFailure(
+ "Early exit loop with side effects contains unsupported critical load.\n",
+ "EarlyExitSideEffectsBadCriticalLoad",
+ ORE, TheLoop);
return false;
+ }
// We want to guarantee that the uncountable exit condition (and the mask
// we will generate from it) are available for all operations in the loop
@@ -3291,8 +3307,13 @@ static bool handleUncountableExitsWithSideEffects(
for (VPRecipeBase &R : *VPBB)
if (R.mayReadOrWriteMemory() && &R != Load) {
// TODO: Handle conditional memory operations in the loop.
- if (!VPDT.dominates(R.getParent(), LatchVPBB))
+ if (!VPDT.dominates(R.getParent(), LatchVPBB)) {
+ reportVectorizationFailure("Early exit loop with side effects "
+ "contains unsupported memory operations.\n",
+ "EarlyExitSideEffectsUnsupportedMemOps",
+ ORE, TheLoop);
return false;
+ }
cast<VPInstruction>(&R)->addMask(Mask);
}
@@ -3315,8 +3336,13 @@ static bool handleUncountableExitsWithSideEffects(
auto Phis = ScalarPH->phis();
// TODO: Handle more than one Phi; re-derive from IV.
// TODO: Handle reductions.
- if (range_size(Phis) != 1)
+ if (range_size(Phis) != 1) {
+ reportVectorizationFailure("Early exit loop with side effects contains "
+ "unsupported reductions or recurrences.\n",
+ "EarlyExitEffectsSideReductions",
+ ORE, TheLoop);
return false;
+ }
VPPhi *ContinueIV = cast<VPPhi>(Phis.begin());
// Make sure we're referring to the same IV.
assert(
@@ -3328,8 +3354,9 @@ static bool handleUncountableExitsWithSideEffects(
}
bool VPlanTransforms::handleUncountableEarlyExits(
- VPlan &Plan, Loop *TheLoop, PredicatedScalarEvolution &PSE,
- DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) {
+ VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
+ PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
+ UncountableExitStyle Style) {
#ifndef NDEBUG
VPDominatorTree VPDT(Plan);
#endif
@@ -3341,8 +3368,13 @@ bool VPlanTransforms::handleUncountableEarlyExits(
// stores, as only the loads contributing to the exit condition need to
// be checked.
if (Style == UncountableExitStyle::ReadOnly &&
- !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
+ !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) {
+ reportVectorizationFailure(
+ "Auto-vectorization of early exit loops with potentially "
+ "faulting loads is not supported.\n.",
+ "EarlyExitFaultingLoads", ORE, TheLoop);
return false;
+ }
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
SmallVector<EarlyExitInfo> Exits;
@@ -3431,8 +3463,9 @@ bool VPlanTransforms::handleUncountableEarlyExits(
LatchVPBB->setSuccessors({MiddleVPBB, MiddleVPBB, HeaderVPBB});
MiddleVPBB->clearPredecessors();
MiddleVPBB->setPredecessors({LatchVPBB, LatchVPBB});
- return handleUncountableExitsWithSideEffects(
- Plan, Exits, HeaderVPBB, LatchVPBB, MiddleVPBB, TheLoop, PSE, DT, AC);
+ return handleUncountableExitsWithSideEffects(Plan, Exits, HeaderVPBB,
+ LatchVPBB, MiddleVPBB, ORE,
+ TheLoop, PSE, DT, AC);
}
// Create the vector.early.exit blocks.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 1c7b17942b7952..a837d3fd4c266f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -383,9 +383,10 @@ struct VPlanTransforms {
/// latch exit condition. Multiple exits are handled with a dispatch block
/// that determines which exit to take based on lane-by-lane semantics.
LLVM_ABI_FOR_TEST static bool
- handleUncountableEarlyExits(VPlan &Plan, Loop *TheLoop,
- PredicatedScalarEvolution &PSE, DominatorTree &DT,
- AssumptionCache *AC, UncountableExitStyle Style);
+ handleUncountableEarlyExits(VPlan &Plan, OptimizationRemarkEmitter *ORE,
+ Loop *TheLoop, PredicatedScalarEvolution &PSE,
+ DominatorTree &DT, AssumptionCache *AC,
+ UncountableExitStyle Style);
/// Disconnect countable early exits from the loop.
LLVM_ABI_FOR_TEST static void handleCountableEarlyExits(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index 7768aac2ec309d..ae7cce71167048 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -1,6 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize,vplan -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize,vplan -force-vector-width=4 \
+; RUN: -disable-output 2>&1 | FileCheck %s --check-prefix=CHECK-DEBUG
+; RUN: opt -S < %s -p loop-vectorize -pass-remarks-analysis='loop-vectorize' \
+; RUN: -force-vector-width=4 -disable-output 2>&1 | FileCheck %s --check-prefix=CHECK-REMARK
declare void @init_mem(ptr, i64) nofree
@@ -8,10 +11,10 @@ declare void @init_mem(ptr, i64) nofree
; The form of the induction variables requires SCEV predicates.
define i32 @diff_exit_block_needs_scev_check(i32 %end) {
-; CHECK-LABEL: LV: Checking a loop in 'diff_exit_block_needs_scev_check'
-; CHECK: Found an early exit loop with symbolic max backedge taken count: (-1 + (1 umax (zext i10 (trunc i32 %end to i10) to i32)))<nsw>
-; CHECK-NEXT: LV: We can vectorize this loop!
-; CHECK-NOT: LV: Not vectorizing:
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'diff_exit_block_needs_scev_check'
+; CHECK-DEBUG: Found an early exit loop with symbolic max backedge taken count: (-1 + (1 umax (zext i10 (trunc i32 %end to i10) to i32)))<nsw>
+; CHECK-DEBUG-NEXT: LV: We can vectorize this loop!
+; CHECK-DEBUG-NOT: LV: Not vectorizing:
entry:
%p1 = alloca [1024 x i32]
%p2 = alloca [1024 x i32]
@@ -46,10 +49,10 @@ exit:
define i64 @same_exit_block_pre_inc_use1() {
-; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1'
-; CHECK: LV: Found an early exit loop with symbolic max backedge taken count: 63
-; CHECK-NEXT: LV: We can vectorize this loop!
-; CHECK-NOT: LV: Not vectorizing
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1'
+; CHECK-DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
+; CHECK-DEBUG-NEXT: LV: We can vectorize this loop!
+; CHECK-DEBUG-NOT: LV: Not vectorizing
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
@@ -78,9 +81,9 @@ loop.end:
define i64 @loop_contains_safe_call() {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_safe_call'
-; CHECK: LV: Found an early exit loop with symbolic max backedge taken count: 63
-; CHECK-NEXT: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_safe_call'
+; CHECK-DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
+; CHECK-DEBUG-NEXT: LV: We can vectorize this loop!
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
@@ -108,9 +111,9 @@ loop.end:
define i64 @loop_contains_safe_div() {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_safe_div'
-; CHECK: LV: Found an early exit loop with symbolic max backedge taken count: 63
-; CHECK-NEXT: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_safe_div'
+; CHECK-DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
+; CHECK-DEBUG-NEXT: LV: We can vectorize this loop!
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
@@ -138,10 +141,10 @@ loop.end:
define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(8) %p2) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_load_after_early_exit'
-; CHECK: LV: Found an early exit loop with symbolic max backedge taken count: 63
-; CHECK-NEXT: LV: We can vectorize this loop!
-; CHECK-NOT: LV: Not vectorizing
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_load_after_early_exit'
+; CHECK-DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
+; CHECK-DEBUG-NEXT: LV: We can vectorize this loop!
+; CHECK-DEBUG-NOT: LV: Not vectorizing
entry:
%p1 = alloca [1024 x i8]
call void @init_mem(ptr %p1, i64 1024)
@@ -167,22 +170,23 @@ loop.end:
}
-define i64 @one_uncountable_two_countable_same_exit_phi_of_consts() {
-; CHECK-LABEL: LV: Checking a loop in 'one_uncountable_two_countable_same_exit_phi_of_consts'
-; CHECK: LV: Found an early exit loop with symbolic max backedge taken count: 61
-; CHECK-NEXT: LV: We can vectorize this loop!
-; CHECK-NEXT: LV: Not vectorizing: Auto-vectorization of early exit loops requiring a scalar epilogue is unsupported.
+define i64 @one_uncountable_two_countable_same_exit_phi_of_consts() !dbg !39 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'one_uncountable_two_countable_same_exit_phi_of_consts'
+; CHECK-DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 61
+; CHECK-DEBUG-NEXT: LV: We can vectorize this loop!
+; CHECK-DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of early exit loops requiring a scalar epilogue is unsupported.
+; CHECK-REMARK: foo.c:60:3: loop not vectorized: Auto-vectorization of early exit loops requiring a scalar epilogue is unsupported
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
call void @init_mem(ptr %p1, i64 1024)
call void @init_mem(ptr %p2, i64 1024)
- br label %loop
+ br label %loop, !dbg !40
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
%cmp1 = icmp ne i64 %index, 64
- br i1 %cmp1, label %search, label %loop.end
+ br i1 %cmp1, label %search, label %loop.end, !dbg !40
search:
%arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
@@ -190,12 +194,12 @@ search:
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
%ld2 = load i8, ptr %arrayidx1, align 1
%cmp3 = icmp eq i8 %ld1, %ld2
- br i1 %cmp3, label %loop.end, label %loop.inc
+ br i1 %cmp3, label %loop.end, label %loop.inc, !dbg !40
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 128
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !40
loop.end:
%retval = phi i64 [ 0, %loop ], [ 1, %search ], [ 0, %loop.inc ]
@@ -206,29 +210,30 @@ loop.end:
; == SOME ILLEGAL EXAMPLES ==
-define i64 @same_exit_block_pre_inc_use1_too_small_allocas() {
-; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_too_small_allocas'
-; CHECK: LV: Not vectorizing: Auto-vectorization of loops with potentially faulting load is not supported.
+define i64 @same_exit_block_pre_inc_use1_too_small_allocas() !dbg !4 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_too_small_allocas'
+; CHECK-DEBUG: LV: Not vectorizing: Auto-vectorization of early exit loops with potentially faulting loads is not supported.
+; CHECK-REMARK: foo.c:5:3: loop not vectorized: Auto-vectorization of early exit loops with potentially faulting loads is not supported.
entry:
%p1 = alloca [42 x i8]
%p2 = alloca [42 x i8]
call void @init_mem(ptr %p1, i64 1024)
call void @init_mem(ptr %p2, i64 1024)
- br label %loop
+ br label %loop, !dbg !8
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
%arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
- %ld1 = load i8, ptr %arrayidx, align 1
+ %ld1 = load i8, ptr %arrayidx, align 1, !dbg !8
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
- %ld2 = load i8, ptr %arrayidx1, align 1
- %cmp3 = icmp eq i8 %ld1, %ld2
- br i1 %cmp3, label %loop.inc, label %loop.end
+ %ld2 = load i8, ptr %arrayidx1, align 1, !dbg !8
+ %cmp3 = icmp eq i8 %ld1, %ld2, !dbg !8
+ br i1 %cmp3, label %loop.inc, label %loop.end, !dbg !8
loop.inc:
- %index.next = add i64 %index, 1
- %exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
+ %index.next = add i64 %index, 1, !dbg !8
+ %exitcond = icmp ne i64 %index.next, 67, !dbg !8
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !8
loop.end:
%retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
@@ -236,11 +241,12 @@ loop.end:
}
-define i64 @same_exit_block_pre_inc_use1_too_small_deref_ptrs(ptr dereferenceable(42) %p1, ptr dereferenceable(42) %p2) {
-; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_too_small_deref_ptrs'
-; CHECK: LV: Not vectorizing: Auto-vectorization of loops with potentially faulting load is not supported.
+define i64 @same_exit_block_pre_inc_use1_too_small_deref_ptrs(ptr dereferenceable(42) %p1, ptr dereferenceable(42) %p2) !dbg !41 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_too_small_deref_ptrs'
+; CHECK-DEBUG: LV: Not vectorizing: Auto-vectorization of early exit loops with potentially faulting loads is not supported.
+; CHECK-REMARK: foo.c:70:3: loop not vectorized: Auto-vectorization of early exit loops with potentially faulting loads is not supported.
entry:
- br label %loop
+ br label %loop, !dbg !42
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
@@ -249,12 +255,12 @@ loop:
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
%ld2 = load i8, ptr %arrayidx1, align 1
%cmp3 = icmp eq i8 %ld1, %ld2
- br i1 %cmp3, label %loop.inc, label %loop.end
+ br i1 %cmp3, label %loop.inc, label %loop.end, !dbg !42
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !42
loop.end:
%retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
@@ -262,11 +268,12 @@ loop.end:
}
-define i64 @same_exit_block_pre_inc_use1_unknown_ptrs(ptr %p1, ptr %p2) {
-; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_unknown_ptrs'
-; CHECK: LV: Not vectorizing: Auto-vectorization of loops with potentially faulting load is not supported.
+define i64 @same_exit_block_pre_inc_use1_unknown_ptrs(ptr %p1, ptr %p2) !dbg !43 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_unknown_ptrs'
+; CHECK-DEBUG: LV: Not vectorizing: Auto-vectorization of early exit loops with potentially faulting loads is not supported.
+; CHECK-REMARK: foo.c:80:3: loop not vectorized: Auto-vectorization of early exit loops with potentially faulting loads is not supported.
entry:
- br label %loop
+ br label %loop, !dbg !44
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
@@ -275,39 +282,40 @@ loop:
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
%ld2 = load i8, ptr %arrayidx1, align 1
%cmp3 = icmp eq i8 %ld1, %ld2
- br i1 %cmp3, label %loop.inc, label %loop.end
+ br i1 %cmp3, label %loop.inc, label %loop.end, !dbg !44
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !44
loop.end:
%retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
ret i64 %retval
}
-define ptr @same_exit_block_strided_unknown_ptr(ptr %first, ptr %last, i32 %value) {
-; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_strided_unknown_ptr'
-; CHECK: LV: Not vectorizing: Loop contains potentially faulting strided load.
+define ptr @same_exit_block_strided_unknown_ptr(ptr %first, ptr %last, i32 %value) !dbg !45 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_strided_unknown_ptr'
+; CHECK-DEBUG: LV: Not vectorizing: Loop contains potentially faulting strided load.
+; CHECK-REMARK: foo.c:90:3: loop not vectorized: Cannot vectorize early exit loop with strided fault-only-first load
entry:
%cond = icmp eq ptr %first, %last
- br i1 %cond, label %return, label %for.body
+ br i1 %cond, label %return, label %for.body, !dbg !46
for.body:
%first.addr = phi ptr [ %first, %entry ], [ %first.next, %for.inc ]
%1 = load i32, ptr %first.addr, align 4
%cond2 = icmp eq i32 %1, %value
- br i1 %cond2, label %for.end, label %for.inc
+ br i1 %cond2, label %for.end, label %for.inc, !dbg !46
for.inc:
%first.next = getelementptr inbounds i32, ptr %first.addr, i64 2
%cond3 = icmp eq ptr %first.next, %last
- br i1 %cond3, label %for.end, label %for.body
+ br i1 %cond3, label %for.end, label %for.body, !dbg !46
for.end:
%retval.ph = phi ptr [ %first.addr, %for.body ], [ %last, %for.inc ]
- br label %return
+ br label %return, !dbg !46
return:
%retval = phi ptr [ %first, %entry ], [ %retval.ph, %for.end ]
@@ -316,21 +324,22 @@ return:
; The early exit (i.e. unknown exit-not-taken count) is the latch - we don't
; support this yet.
-define i64 @uncountable_exit_on_last_block() {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_on_last_block'
-; CHECK: LV: Not vectorizing: Cannot determine exact exit count for latch block.
+define i64 @uncountable_exit_on_last_block() !dbg !47 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_on_last_block'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot determine exact exit count for latch block.
+; CHECK-REMARK: foo.c:100:3: loop not vectorized: Cannot vectorize early exit loop
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
call void @init_mem(ptr %p1, i64 1024)
call void @init_mem(ptr %p2, i64 1024)
- br label %loop
+ br label %loop, !dbg !48
loop:
%index = phi i64 [ %index.next, %search ], [ 3, %entry ]
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %search, label %loop.end
+ br i1 %exitcond, label %search, label %loop.end, !dbg !48
search:
%arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
@@ -338,7 +347,7 @@ search:
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
%ld2 = load i8, ptr %arrayidx1, align 1
%cmp3 = icmp eq i8 %ld1, %ld2
- br i1 %cmp3, label %loop.end, label %loop
+ br i1 %cmp3, label %loop.end, label %loop, !dbg !48
loop.end:
%retval = phi i64 [ 64, %loop ], [ %index, %search ]
@@ -348,8 +357,8 @@ loop.end:
; Multiple uncountable early exits are now supported.
define i64 @multiple_uncountable_exits() {
-; CHECK-LABEL: LV: Checking a loop in 'multiple_uncountable_exits'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'multiple_uncountable_exits'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
@@ -381,15 +390,17 @@ loop.end:
}
-define i64 @uncountable_exit_infinite_loop() {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_infinite_loop'
-; CHECK: LV: Not vectorizing: Cannot vectorize uncountable loop.
+define i64 @uncountable_exit_infinite_loop() !dbg !51 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_infinite_loop'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot vectorize uncountable loop.
+; CHECK-REMARK: foo.c:120:3: loop not vectorized: Cannot vectorize uncountable loop
+; CHECK-REMARK-NEXT: foo.c:120:3: loop not vectorized: could not determine number of loop iterations
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
call void @init_mem(ptr %p1, i64 1024)
call void @init_mem(ptr %p2, i64 1024)
- br label %loop
+ br label %loop, !dbg !52
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
@@ -398,12 +409,12 @@ loop:
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
%ld2 = load i8, ptr %arrayidx1, align 1
%cmp3 = icmp eq i8 %ld1, %ld2
- br i1 %cmp3, label %loop.inc, label %loop.end
+ br i1 %cmp3, label %loop.inc, label %loop.end, !dbg !52
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br label %loop
+ br label %loop, !dbg !52
loop.end:
%retval = phi i64 [ %index, %loop ]
@@ -411,15 +422,16 @@ loop.end:
}
-define i64 @loop_contains_unsafe_call() {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_unsafe_call'
-; CHECK: LV: Not vectorizing: Early exit loop contains operations that cannot be speculatively executed.
+define i64 @loop_contains_unsafe_call() !dbg !53 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_unsafe_call'
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop contains operations that cannot be speculatively executed.
+; CHECK-REMARK: foo.c:130:3: loop not vectorized: Early exit loop contains operations that cannot be speculatively executed
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
call void @init_mem(ptr %p1, i64 1024)
call void @init_mem(ptr %p2, i64 1024)
- br label %loop
+ br label %loop, !dbg !54
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
@@ -427,12 +439,12 @@ loop:
%ld1 = load i32, ptr %arrayidx, align 1
%bad_call = call i32 @foo(i32 %ld1) #0
%cmp = icmp eq i32 %bad_call, 34
- br i1 %cmp, label %loop.inc, label %loop.end
+ br i1 %cmp, label %loop.inc, label %loop.end, !dbg !54
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !54
loop.end:
%retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
@@ -440,15 +452,16 @@ loop.end:
}
-define i64 @loop_contains_unsafe_div() {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_unsafe_div'
-; CHECK: LV: Not vectorizing: Early exit loop contains operations that cannot be speculatively executed.
+define i64 @loop_contains_unsafe_div() !dbg !55 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_unsafe_div'
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop contains operations that cannot be speculatively executed.
+; CHECK-REMARK: foo.c:140:3: loop not vectorized: Early exit loop contains operations that cannot be speculatively executed
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
call void @init_mem(ptr %p1, i64 1024)
call void @init_mem(ptr %p2, i64 1024)
- br label %loop
+ br label %loop, !dbg !56
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
@@ -456,23 +469,24 @@ loop:
%ld1 = load i32, ptr %arrayidx, align 1
%div = udiv i32 20000, %ld1
%cmp = icmp eq i32 %div, 1
- br i1 %cmp, label %loop.inc, label %loop.end
+ br i1 %cmp, label %loop.inc, label %loop.end, !dbg !56
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !56
loop.end:
%retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
ret i64 %retval
}
-define void @exit_conditions_combined_in_single_branch(ptr noalias dereferenceable(40) %array, ptr readonly align 2 dereferenceable(40) %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'exit_conditions_combined_in_single_branch'
-; CHECK: LV: Not vectorizing: Cannot vectorize uncountable loop.
+define void @exit_conditions_combined_in_single_branch(ptr noalias dereferenceable(40) %array, ptr readonly align 2 dereferenceable(40) %pred) !dbg !57 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'exit_conditions_combined_in_single_branch'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot vectorize uncountable loop.
+; CHECK-REMARK: foo.c:150:3: loop not vectorized: Cannot vectorize uncountable loop
entry:
- br label %for.body
+ br label %for.body, !dbg !58
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
@@ -486,7 +500,7 @@ for.body:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
%or.cond = select i1 %ee.cond, i1 true, i1 %counted.cond
- br i1 %or.cond, label %exit, label %for.body
+ br i1 %or.cond, label %exit, label %for.body, !dbg !58
exit:
ret void
@@ -495,13 +509,14 @@ exit:
; %prev is a fixed-order recurrence and, under a SCEV predicate, also the only
; canonical {0,+,1} induction candidate.
; the early-exit loop is still rejected here.
-define i64 @same_exit_block_with_recurrence_that_is_also_an_induction() {
-; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_with_recurrence_that_is_also_an_induction'
-; CHECK: LV: Not vectorizing: Found reductions or recurrences in early-exit loop.
+define i64 @same_exit_block_with_recurrence_that_is_also_an_induction() !dbg !59 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_with_recurrence_that_is_also_an_induction'
+; CHECK-DEBUG: LV: Not vectorizing: Found reductions or recurrences in early-exit loop.
+; CHECK-REMARK: foo.c:160:3: loop not vectorized: Cannot vectorize early exit loop with reductions or recurrences
entry:
%p1 = alloca [4096 x i8]
call void @init_mem(ptr %p1, i64 4096)
- br label %loop
+ br label %loop, !dbg !60
loop:
%prev = phi i64 [ %ext, %loop.inc ], [ 0, %entry ]
@@ -513,27 +528,28 @@ loop:
%ld1 = load i8, ptr %arrayidx, align 1
%prev.trunc = trunc i64 %prev to i8
%cmp3 = icmp eq i8 %ld1, %prev.trunc
- br i1 %cmp3, label %loop.inc, label %loop.end
+ br i1 %cmp3, label %loop.inc, label %loop.end, !dbg !60
loop.inc:
%index.next = add i32 %index, 1
%exitcond = icmp ne i32 %index.next, 4000
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !60
loop.end:
%retval = phi i64 [ %ext, %loop ], [ 67, %loop.inc ]
ret i64 %retval
}
-define i64 @same_exit_block_pre_inc_use1_with_reduction() {
-; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_with_reduction'
-; CHECK: LV: Not vectorizing: Found reductions or recurrences in early-exit loop.
+define i64 @same_exit_block_pre_inc_use1_with_reduction() !dbg !61 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_with_reduction'
+; CHECK-DEBUG: LV: Not vectorizing: Found reductions or recurrences in early-exit loop.
+; CHECK-REMARK: foo.c:170:3: loop not vectorized: Cannot vectorize early exit loop with reductions or recurrences
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
call void @init_mem(ptr %p1, i64 1024)
call void @init_mem(ptr %p2, i64 1024)
- br label %loop
+ br label %loop, !dbg !62
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
@@ -545,12 +561,12 @@ loop:
%ld2.zext = zext i8 %ld2 to i64
%red.next = add i64 %red, %ld2.zext
%cmp3 = icmp eq i8 %ld1, %ld2
- br i1 %cmp3, label %loop.inc, label %loop.end
+ br i1 %cmp3, label %loop.inc, label %loop.end, !dbg !62
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !62
loop.end:
%final.ind = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
@@ -559,13 +575,15 @@ loop.end:
}
-define i64 @uncountable_exit_has_multiple_outside_successors() {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_has_multiple_outside_successors'
-; CHECK: LV: Not vectorizing: Loop contains an unsupported switch
+define i64 @uncountable_exit_has_multiple_outside_successors() !dbg !63 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_has_multiple_outside_successors'
+; CHECK-DEBUG: LV: Not vectorizing: Loop contains an unsupported switch
+; CHECK-REMARK: foo.c:180:3: loop not vectorized: Loop contains an unsupported switch
+; CHECK-REMARK-NEXT: foo.c:180:3: loop not vectorized: Incorrect number of successors from early exiting block
entry:
%p1 = alloca [1024 x i8]
call void @init_mem(ptr %p1, i64 1024)
- br label %loop
+ br label %loop, !dbg !64
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
@@ -574,12 +592,12 @@ loop:
switch i8 %ld1, label %loop.inc [
i8 2, label %loop.end
i8 3, label %loop.surprise
- ]
+ ], !dbg !64
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !64
loop.surprise:
ret i64 3
@@ -593,8 +611,8 @@ loop.end:
; Two early exits on parallel branches (neither dominates the other).
; This is now supported with predicated early exits.
define i64 @uncountable_exits_on_parallel_branches() {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exits_on_parallel_branches'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exits_on_parallel_branches'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
@@ -633,31 +651,33 @@ loop.end:
; Parallel uncountable exits with loop-invariant conditions.
; Note: This loop cannot be vectorized because the latch has no determinate
; exit count (loop is infinite without early exits).
-define void @uncountable_exits_invariant_conditions(ptr %p, i1 %cond1, i1 %cond2, i1 %cond3) {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exits_invariant_conditions'
-; CHECK: LV: Not vectorizing: Cannot determine exact exit count for latch block.
+define void @uncountable_exits_invariant_conditions(ptr %p, i1 %cond1, i1 %cond2, i1 %cond3) !dbg !67 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exits_invariant_conditions'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot determine exact exit count for latch block.
+; CHECK-REMARK: foo.c:200:3: loop not vectorized: Cannot vectorize early exit loop
+; CHECK-REMARK-NEXT: foo.c:200:3: loop not vectorized: could not determine number of loop iterations
entry:
- br label %loop.header
+ br label %loop.header, !dbg !68
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
- br i1 %cond1, label %then.1, label %else.1
+ br i1 %cond1, label %then.1, label %else.1, !dbg !68
else.1:
- br i1 %cond2, label %then.2, label %exit
+ br i1 %cond2, label %then.2, label %exit, !dbg !68
then.2:
%getelementptr.i = getelementptr i16, ptr %p, i64 %iv
%l = load i16, ptr %getelementptr.i, align 2
%ec = icmp ugt i16 %l, -25
- br i1 %ec, label %exit, label %loop.latch
+ br i1 %ec, label %exit, label %loop.latch, !dbg !68
then.1:
- br i1 %cond3, label %exit, label %loop.latch
+ br i1 %cond3, label %exit, label %loop.latch, !dbg !68
loop.latch:
%iv.next = add i64 %iv, 1
- br label %loop.header
+ br label %loop.header, !dbg !68
exit:
ret void
@@ -667,3 +687,46 @@ declare i32 @foo(i32) readonly
declare <vscale x 4 x i32> @foo_vec(<vscale x 4 x i32>)
attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }
+
+!llvm.dbg.cu = !{!28}
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 1, !"PIC Level", i32 2}
+!3 = !DILocation(line: 5, column: 20, scope: !4)
+!4 = distinct !DISubprogram(name: "foo", scope: !5, file: !5, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!5 = !DIFile(filename: "foo.c", directory: "")
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 5, column: 3, scope: !4)
+!9 = !DILocation(line: 6, column: 14, scope: !4)
+!10 = !DILocation(line: 9, column: 3, scope: !4)
+!15 = !DILocation(line: 6, column: 19, scope: !4)
+!16 = !DILocation(line: 6, column: 11, scope: !4)
+!39 = distinct !DISubprogram(name: "one_uncountable_two_countable_same_exit_phi_of_consts", scope: !5, file: !5, line: 60, type: !6, isLocal: false, isDefinition: true, scopeLine: 60, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!40 = !DILocation(line: 60, column: 3, scope: !39)
+!41 = distinct !DISubprogram(name: "same_exit_block_pre_inc_use1_too_small_deref_ptrs", scope: !5, file: !5, line: 70, type: !6, isLocal: false, isDefinition: true, scopeLine: 70, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!42 = !DILocation(line: 70, column: 3, scope: !41)
+!43 = distinct !DISubprogram(name: "same_exit_block_pre_inc_use1_unknown_ptrs", scope: !5, file: !5, line: 80, type: !6, isLocal: false, isDefinition: true, scopeLine: 80, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!44 = !DILocation(line: 80, column: 3, scope: !43)
+!45 = distinct !DISubprogram(name: "same_exit_block_strided_unknown_ptr", scope: !5, file: !5, line: 90, type: !6, isLocal: false, isDefinition: true, scopeLine: 90, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!46 = !DILocation(line: 90, column: 3, scope: !45)
+!47 = distinct !DISubprogram(name: "uncountable_exit_on_last_block", scope: !5, file: !5, line: 100, type: !6, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!48 = !DILocation(line: 100, column: 3, scope: !47)
+!51 = distinct !DISubprogram(name: "uncountable_exit_infinite_loop", scope: !5, file: !5, line: 120, type: !6, isLocal: false, isDefinition: true, scopeLine: 120, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!52 = !DILocation(line: 120, column: 3, scope: !51)
+!53 = distinct !DISubprogram(name: "loop_contains_unsafe_call", scope: !5, file: !5, line: 130, type: !6, isLocal: false, isDefinition: true, scopeLine: 130, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!54 = !DILocation(line: 130, column: 3, scope: !53)
+!55 = distinct !DISubprogram(name: "loop_contains_unsafe_div", scope: !5, file: !5, line: 140, type: !6, isLocal: false, isDefinition: true, scopeLine: 140, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!56 = !DILocation(line: 140, column: 3, scope: !55)
+!57 = distinct !DISubprogram(name: "exit_conditions_combined_in_single_branch", scope: !5, file: !5, line: 150, type: !6, isLocal: false, isDefinition: true, scopeLine: 150, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!58 = !DILocation(line: 150, column: 3, scope: !57)
+!59 = distinct !DISubprogram(name: "same_exit_block_with_recurrence_that_is_also_an_induction", scope: !5, file: !5, line: 160, type: !6, isLocal: false, isDefinition: true, scopeLine: 160, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!60 = !DILocation(line: 160, column: 3, scope: !59)
+!61 = distinct !DISubprogram(name: "same_exit_block_pre_inc_use1_with_reduction", scope: !5, file: !5, line: 170, type: !6, isLocal: false, isDefinition: true, scopeLine: 170, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!62 = !DILocation(line: 170, column: 3, scope: !61)
+!63 = distinct !DISubprogram(name: "uncountable_exit_has_multiple_outside_successors", scope: !5, file: !5, line: 180, type: !6, isLocal: false, isDefinition: true, scopeLine: 180, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!64 = !DILocation(line: 180, column: 3, scope: !63)
+!67 = distinct !DISubprogram(name: "uncountable_exits_invariant_conditions", scope: !5, file: !5, line: 200, type: !6, isLocal: false, isDefinition: true, scopeLine: 200, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!68 = !DILocation(line: 200, column: 3, scope: !67)
+!28 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang", file: !5, isOptimized: true, flags: "-O2", splitDebugFilename: "abc.debug", emissionKind: 2)
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index 2ad0cfd0d744d6..f29f00590a08d6 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -1,13 +1,17 @@
; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-vectorization-with-side-effects -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-vectorization-with-side-effects \
+; RUN: -force-vector-width=4 -disable-output 2>&1 | FileCheck %s --check-prefix=CHECK-DEBUG
+; RUN: opt -S < %s -p loop-vectorize -pass-remarks-analysis='loop-vectorize' -enable-early-exit-vectorization-with-side-effects \
+; RUN: -force-vector-width=4 -disable-output 2>&1 | FileCheck %s --check-prefix=CHECK-REMARK
;; This currently doesn't vectorize because the load used to determine the
;; uncountable exit condition has a second user (the store).
-define i64 @loop_contains_store(ptr dereferenceable(1024) %p1, ptr noalias %dest) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store'
-; CHECK: LV: Not vectorizing: Early exit loop with store but no supported condition load.
+define i64 @loop_contains_store(ptr dereferenceable(1024) %p1, ptr noalias %dest) !dbg !10 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store'
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with store but no supported condition load.
+; CHECK-REMARK: foo.c:10:3: loop not vectorized: Early exit loop with store but no supported condition load
entry:
- br label %loop
+ br label %loop, !dbg !11
loop:
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
@@ -16,12 +20,12 @@ loop:
%arrayidx2 = getelementptr inbounds i32, ptr %dest, i64 %index
store i32 %ld1, ptr %arrayidx2, align 4
%cmp = icmp eq i32 %ld1, 1
- br i1 %cmp, label %loop.inc, label %loop.end
+ br i1 %cmp, label %loop.inc, label %loop.end, !dbg !11
loop.inc:
%index.next = add i64 %index, 1
%exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %loop, label %loop.end
+ br i1 %exitcond, label %loop, label %loop.end, !dbg !11
loop.end:
%retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
@@ -29,8 +33,8 @@ loop.end:
}
define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_has_single_user'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_has_single_user'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -54,10 +58,37 @@ exit:
ret void
}
+define void @loop_contains_store_multidim_condition_load(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) readonly %pred) !dbg !12 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_multidim_condition_load'
+; CHECK-DEBUG: LV: Not vectorizing: Unable to determine early exit condition for loop with side effects.
+; CHECK-REMARK: foo.c:20:3: loop not vectorized: Unable to determine early exit condition for loop with side effects.
+entry:
+ br label %for.body, !dbg !13
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw [2 x i16], ptr %pred, i64 %iv, i64 0
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !13
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !13
+
+exit:
+ ret void
+}
+
;; Exit-condition load on the RHS of the icmp must still be accepted.
define void @swapped_cmp_operands(ptr noalias %array, ptr %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'swapped_cmp_operands'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'swapped_cmp_operands'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
br label %loop
@@ -82,11 +113,12 @@ exit:
;; Avoid vectorization because we will either exit on the first iteration, or
;; never exit early.
;; We shouldn't see IR like this if LV-LICM has done its job.
-define void @novec_loop_contains_store_ee_condition_is_invariant(ptr dereferenceable(40) noalias %array, i16 %ee.val) {
-; CHECK-LABEL: LV: Checking a loop in 'novec_loop_contains_store_ee_condition_is_invariant'
-; CHECK: LV: Not vectorizing: Early exit loop with store but no supported condition load.
+define void @novec_loop_contains_store_ee_condition_is_invariant(ptr dereferenceable(40) noalias %array, i16 %ee.val) !dbg !16 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'novec_loop_contains_store_ee_condition_is_invariant'
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with store but no supported condition load.
+; CHECK-REMARK: foo.c:40:3: loop not vectorized: Early exit loop with store but no supported condition load
entry:
- br label %for.body
+ br label %for.body, !dbg !17
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -95,23 +127,24 @@ for.body:
%inc = add nsw i16 %data, 1
store i16 %inc, ptr %st.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !17
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !17
exit:
ret void
}
;; Vectorizeable, needs work on exit condition recipe collection.
-define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_fcmp_condition'
-; CHECK: LV: Not vectorizing: Early exit loop with store but no supported condition load.
+define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) !dbg !18 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_fcmp_condition'
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with store but no supported condition load.
+; CHECK-REMARK: foo.c:50:3: loop not vectorized: Early exit loop with store but no supported condition load
entry:
- br label %for.body
+ br label %for.body, !dbg !19
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -122,24 +155,25 @@ for.body:
%ee.addr = getelementptr inbounds nuw half, ptr %pred, i64 %iv
%ee.val = load half, ptr %ee.addr, align 2
%ee.cond = fcmp ugt half %ee.val, 500.0
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !19
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !19
exit:
ret void
}
;; Vectorizeable, needs work on alias checks for the exit condition load.
-define void @loop_contains_store_safe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(96) %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_safe_dependency'
-; CHECK: LV: Not vectorizing: Cannot determine whether critical uncountable exit load address does not alias with a memory write.
+define void @loop_contains_store_safe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(96) %pred) !dbg !20 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_safe_dependency'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot determine whether critical uncountable exit load address does not alias with a memory write.
+; CHECK-REMARK: foo.c:60:3: loop not vectorized: Cannot determine whether critical uncountable exit load address does not alias with a memory write
entry:
%pred.plus.8 = getelementptr inbounds nuw i16, ptr %pred, i64 8
- br label %for.body
+ br label %for.body, !dbg !21
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -152,27 +186,28 @@ for.body:
%ee.cond = icmp sgt i16 %ee.val, 500
%some.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
store i16 42, ptr %some.addr, align 2
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !21
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !21
exit:
ret void
}
;; Possibly vectorizeable, but would require some runtime checks.
-define void @loop_contains_store_unsafe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_unsafe_dependency'
-; CHECK: LV: Not vectorizing: Cannot determine whether critical uncountable exit load address does not alias with a memory write.
+define void @loop_contains_store_unsafe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) %pred) !dbg !22 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_unsafe_dependency'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot determine whether critical uncountable exit load address does not alias with a memory write.
+; CHECK-REMARK: foo.c:70:3: loop not vectorized: Cannot determine whether critical uncountable exit load address does not alias with a memory write
entry:
%unknown.offset = call i64 @get_an_unknown_offset()
%unknown.cmp = icmp ult i64 %unknown.offset, 20
%clamped.offset = select i1 %unknown.cmp, i64 %unknown.offset, i64 20
%unknown.base = getelementptr i16, ptr %pred, i64 %clamped.offset
- br label %for.body
+ br label %for.body, !dbg !23
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -185,12 +220,12 @@ for.body:
%ee.cond = icmp sgt i16 %ee.val, 500
%some.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
store i16 42, ptr %some.addr, align 2
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !23
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !23
exit:
ret void
@@ -200,9 +235,8 @@ exit:
;; might exceed known dereferenceable extents.
;; Alternatively, we could use masked.load.ff or vp.load.ff
define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i64 %n) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
-; CHECK: LV: We can vectorize this loop!
-; CHECK: LV: Vectorization is possible but not beneficial.
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
%n_bytes = mul nuw nsw i64 %n, 2
call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i64 %n_bytes) ]
@@ -229,8 +263,8 @@ exit:
}
define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 dereferenceable(40) readonly %load.array, ptr align 2 noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_pointer_with_no_deref_info'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_to_pointer_with_no_deref_info'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -257,9 +291,8 @@ exit:
;; Vectorizeable, requires runtime checks and/or ff loads.
define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100) noalias %array, ptr align 2 dereferenceable(100) readonly %pred, i64 %n) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_unknown_bounds'
-; CHECK: LV: We can vectorize this loop!
-; CHECK: LV: Vectorization is possible but not beneficial.
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_unknown_bounds'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -285,11 +318,13 @@ exit:
;; Avoid vectorization, volatile memory locations may have unexpected behaviour
;; if we try to vectorize.
-define void @novec_loop_contains_store_volatile(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'novec_loop_contains_store_volatile'
-; CHECK: LV: Not vectorizing: Complex writes to memory unsupported in early exit loops.
+define void @novec_loop_contains_store_volatile(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) !dbg !30 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'novec_loop_contains_store_volatile'
+; CHECK-DEBUG: LV: Not vectorizing: Complex writes to memory unsupported in early exit loops.
+; CHECK-REMARK: foo.c:110:3: loop not vectorized: Cannot vectorize early exit loop with complex writes to memory
+; CHECK-REMARK-NEXT: foo.c:110:3: loop not vectorized: instruction cannot be vectorized
entry:
- br label %for.body
+ br label %for.body, !dbg !31
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -300,23 +335,24 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !31
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !31
exit:
ret void
}
;; Vectorizeable, but we really want LICM to sink the store out of the loop
-define void @loop_contains_store_to_invariant_location(ptr dereferenceable(40) readonly %array, ptr align 2 dereferenceable(40) readonly %pred, ptr noalias %store_addr) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_invariant_location'
-; CHECK: LV: Not vectorizing: Cannot vectorize early exit loops with stores to loop-invariant addresses.
+define void @loop_contains_store_to_invariant_location(ptr dereferenceable(40) readonly %array, ptr align 2 dereferenceable(40) readonly %pred, ptr noalias %store_addr) !dbg !32 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_to_invariant_location'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot vectorize early exit loops with stores to loop-invariant addresses.
+; CHECK-REMARK: foo.c:120:3: loop not vectorized: Cannot vectorize early exit loops with stores to loop-invariant addresses
entry:
- br label %for.body
+ br label %for.body, !dbg !33
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -327,20 +363,20 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !33
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !33
exit:
ret void
}
define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_in_latch_block'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_in_latch_block'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -365,11 +401,12 @@ exit:
}
;; Vectorizeable, requires runtime checks.
-define void @loop_contains_store_requiring_alias_check(ptr dereferenceable(40) %array, ptr align 2 dereferenceable(40) %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_requiring_alias_check'
-; CHECK: LV: Not vectorizing: Cannot determine whether critical uncountable exit load address does not alias with a memory write.
+define void @loop_contains_store_requiring_alias_check(ptr dereferenceable(40) %array, ptr align 2 dereferenceable(40) %pred) !dbg !36 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_requiring_alias_check'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot determine whether critical uncountable exit load address does not alias with a memory write.
+; CHECK-REMARK: foo.c:140:3: loop not vectorized: Cannot determine whether critical uncountable exit load address does not alias with a memory write
entry:
- br label %for.body
+ br label %for.body, !dbg !37
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -380,20 +417,20 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !37
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !37
exit:
ret void
}
define void @loop_contains_store_decrementing_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_decrementing_iv'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_decrementing_iv'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -419,11 +456,12 @@ exit:
;; Vectorizeable, requires improvements to exit condition recipe collection and
;; masked.gather.ff
-define void @loop_contains_store_condition_load_requires_gather(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(512) readonly %pred, ptr align 1 dereferenceable(20) readonly %offsets) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_requires_gather'
-; CHECK: LV: Not vectorizing: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop.
+define void @loop_contains_store_condition_load_requires_gather(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(512) readonly %pred, ptr align 1 dereferenceable(20) readonly %offsets) !dbg !40 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_requires_gather'
+; CHECK-DEBUG: LV: Not vectorizing: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop.
+; CHECK-REMARK: foo.c:160:3: loop not vectorized: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop
entry:
- br label %for.body
+ br label %for.body, !dbg !41
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -437,23 +475,25 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %offset.zext
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !41
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !41
exit:
ret void
}
;; Vectorizeable, requires improvements in handling switch instructions in LV.
-define void @loop_contains_store_uncounted_exit_is_a_switch(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_uncounted_exit_is_a_switch'
-; CHECK: LV: Not vectorizing: Loop contains an unsupported switch
+define void @loop_contains_store_uncounted_exit_is_a_switch(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) !dbg !42 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_uncounted_exit_is_a_switch'
+; CHECK-DEBUG: LV: Not vectorizing: Loop contains an unsupported switch
+; CHECK-REMARK: foo.c:170:3: loop not vectorized: Loop contains an unsupported switch
+; CHECK-REMARK-NEXT: foo.c:170:3: loop not vectorized: Early exit loop contains operations that cannot be speculatively executed
entry:
- br label %for.body
+ br label %for.body, !dbg !43
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -463,12 +503,12 @@ for.body:
store i16 %inc, ptr %st.addr, align 2
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
- switch i16 %ee.val, label %for.inc [ i16 500, label %exit ]
+ switch i16 %ee.val, label %for.inc [ i16 500, label %exit ], !dbg !43
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !43
exit:
ret void
@@ -476,11 +516,12 @@ exit:
;; Vectorizeable, needs to extend the predicated early exit work and improve
;; exit condition recipe collection.
-define void @loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute'
-; CHECK: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+define void @loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) !dbg !44 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute'
+; CHECK-DEBUG: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+; CHECK-REMARK: foo.c:180:3: loop not vectorized: Load for uncountable exit not guaranteed to execute
entry:
- br label %for.body
+ br label %for.body, !dbg !45
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -490,18 +531,18 @@ for.body:
store i16 %inc, ptr %st.addr, align 2
%rem = urem i64 %iv, 5
%skip.ee.cmp = icmp eq i64 %rem, 0
- br i1 %skip.ee.cmp, label %for.inc, label %ee.block
+ br i1 %skip.ee.cmp, label %for.inc, label %ee.block, !dbg !45
ee.block:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !45
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !45
exit:
ret void
@@ -510,11 +551,12 @@ exit:
;; Vectorizeable, requires better alias checking in legality. However, hopefully
;; we wouldn't get this as input IR since it stores to the same address that
;; we load from immediately afterwards.
-define void @test_nodep(ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'test_nodep'
-; CHECK: LV: Not vectorizing: Cannot determine whether critical uncountable exit load address does not alias with a memory write.
+define void @test_nodep(ptr align 2 dereferenceable(40) readonly %pred) !dbg !46 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'test_nodep'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot determine whether critical uncountable exit load address does not alias with a memory write.
+; CHECK-REMARK: foo.c:190:3: loop not vectorized: Cannot determine whether critical uncountable exit load address does not alias with a memory write
entry:
- br label %for.body
+ br label %for.body, !dbg !47
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -522,23 +564,26 @@ for.body:
store i16 0, ptr %st.addr, align 2
%ee.val = load i16, ptr %st.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !47
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !47
exit:
ret void
}
;; Vectorizeable, requires working with the existing histogram code.
-define void @histogram_with_uncountable_exit(ptr noalias %buckets, ptr readonly %indices, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'histogram_with_uncountable_exit'
-; CHECK: LV: Not vectorizing: Cannot vectorize unsafe dependencies in uncountable exit loop with side effects.
+define void @histogram_with_uncountable_exit(ptr noalias %buckets, ptr readonly %indices, ptr align 2 dereferenceable(40) readonly %pred) !dbg !48 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'histogram_with_uncountable_exit'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot vectorize unsafe dependencies in uncountable exit loop with side effects.
+; CHECK-REMARK: foo.c:200:3: loop not vectorized: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-REMARK-NEXT: Unsafe indirect dependence.
+; CHECK-REMARK-NEXT: foo.c:200:3: loop not vectorized: Cannot vectorize unsafe dependencies in uncountable exit loop with side effects
entry:
- br label %for.body
+ br label %for.body, !dbg !49
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -552,30 +597,31 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !49
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !49
exit:
ret void
}
;; Vectorizeable, requires processing more than one exit.
-define void @loop_contains_store_between_two_early_exits(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_between_two_early_exits'
-; CHECK: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+define void @loop_contains_store_between_two_early_exits(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) !dbg !50 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_between_two_early_exits'
+; CHECK-DEBUG: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+; CHECK-REMARK: foo.c:210:3: loop not vectorized: Load for uncountable exit not guaranteed to execute
entry:
- br label %for.body
+ br label %for.body, !dbg !51
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp slt i16 %ee.val, 250
- br i1 %ee.cond, label %exit, label %for.cont
+ br i1 %ee.cond, label %exit, label %for.cont, !dbg !51
for.cont:
%st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
@@ -585,23 +631,24 @@ for.cont:
%ee2.addr = getelementptr inbounds nuw i16, ptr %pred2, i64 %iv
%ee2.val = load i16, ptr %ee2.addr, align 2
%ee2.cond = icmp sgt i16 %ee2.val, 500
- br i1 %ee2.cond, label %exit, label %for.inc
+ br i1 %ee2.cond, label %exit, label %for.inc, !dbg !51
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !51
exit:
ret void
}
;; Vectorizeable, requires processing more than one exit.
-define void @loop_contains_store_before_two_early_exits(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_before_two_early_exits'
-; CHECK: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+define void @loop_contains_store_before_two_early_exits(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) !dbg !52 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_before_two_early_exits'
+; CHECK-DEBUG: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+; CHECK-REMARK: foo.c:220:3: loop not vectorized: Load for uncountable exit not guaranteed to execute
entry:
- br label %for.body
+ br label %for.body, !dbg !53
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -612,34 +659,35 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp slt i16 %ee.val, 250
- br i1 %ee.cond, label %exit, label %for.cont
+ br i1 %ee.cond, label %exit, label %for.cont, !dbg !53
for.cont:
%ee2.addr = getelementptr inbounds nuw i16, ptr %pred2, i64 %iv
%ee2.val = load i16, ptr %ee2.addr, align 2
%ee2.cond = icmp sgt i16 %ee2.val, 500
- br i1 %ee2.cond, label %exit, label %for.inc
+ br i1 %ee2.cond, label %exit, label %for.inc, !dbg !53
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !53
exit:
ret void
}
;; Vectorizeable, requires processing more than one exit.
-define void @one_uncountable_two_countable_exits(ptr dereferenceable(1024) noalias %array, ptr dereferenceable(1024) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'one_uncountable_two_countable_exits'
-; CHECK: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+define void @one_uncountable_two_countable_exits(ptr dereferenceable(1024) noalias %array, ptr dereferenceable(1024) readonly %pred) !dbg !54 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'one_uncountable_two_countable_exits'
+; CHECK-DEBUG: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+; CHECK-REMARK: foo.c:230:3: loop not vectorized: Load for uncountable exit not guaranteed to execute
entry:
- br label %loop
+ br label %loop, !dbg !55
loop:
%iv = phi i64 [ %iv.next, %loop.inc ], [ 3, %entry ]
%ce.ee.cmp = icmp ne i64 %iv, 64
- br i1 %ce.ee.cmp, label %update, label %loop.end
+ br i1 %ce.ee.cmp, label %update, label %loop.end, !dbg !55
update:
%st.addr = getelementptr inbounds i8, ptr %array, i64 %iv
@@ -649,23 +697,24 @@ update:
%ee.addr = getelementptr inbounds i8, ptr %pred, i64 %iv
%ee.val = load i8, ptr %ee.addr, align 1
%ee.cond = icmp eq i8 %ee.val, 37
- br i1 %ee.cond, label %loop.end, label %loop.inc
+ br i1 %ee.cond, label %loop.end, label %loop.inc, !dbg !55
loop.inc:
%iv.next = add i64 %iv, 1
%ce.latch.cmp = icmp ne i64 %iv.next, 128
- br i1 %ce.latch.cmp, label %loop, label %loop.end
+ br i1 %ce.latch.cmp, label %loop, label %loop.end, !dbg !55
loop.end:
ret void
}
;; Vectorizeable, need to handle reductions.
-define i16 @uncountable_exit_with_reduction(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_reduction'
-; CHECK: LV: Not vectorizing: Found an unidentified PHI %rdx = phi i16 [ 0, %entry ], [ %rdx.next, %for.inc ]
+define i16 @uncountable_exit_with_reduction(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) !dbg !56 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_reduction'
+; CHECK-DEBUG: LV: Not vectorizing: Found an unidentified PHI %rdx = phi i16 [ 0, %entry ], [ %rdx.next, %for.inc ]
+; CHECK-REMARK: foo.c:240:3: loop not vectorized: value that could not be identified as reduction is used outside the loop
entry:
- br label %for.body
+ br label %for.body, !dbg !57
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -677,13 +726,13 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !57
for.inc:
%rdx.next = add i16 %rdx, %data
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !57
exit:
%res = phi i16 [ %rdx, %for.body ], [ %rdx.next, %for.inc ]
@@ -691,8 +740,8 @@ exit:
}
define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_live_out'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_live_out'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -717,12 +766,13 @@ exit:
}
; Vectorizeable, requires improvements in dereferenceability checks
-define void @uncountable_exit_with_constant_nonunit_stride(ptr dereferenceable(4000) noalias %array, ptr align 2 dereferenceable(4000) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_constant_nonunit_stride'
-; CHECK: LV: We can vectorize this loop!
-; CHECK: LV: Not vectorizing: unable to calculate the loop count due to complex control flow.
+define void @uncountable_exit_with_constant_nonunit_stride(ptr dereferenceable(4000) noalias %array, ptr align 2 dereferenceable(4000) readonly %pred) !dbg !60 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_constant_nonunit_stride'
+; CHECK-DEBUG: LV: We can vectorize this loop!
+; CHECK-DEBUG: LV: Not vectorizing: unable to calculate the loop count due to complex control flow.
+; CHECK-REMARK: foo.c:260:3: loop not vectorized: unable to calculate the loop count due to complex control flow
entry:
- br label %for.body
+ br label %for.body, !dbg !61
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -733,23 +783,25 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !61
for.inc:
%iv.next = add nuw nsw i64 %iv, 20
%counted.cond = icmp slt i64 %iv.next, 2001
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !61
exit:
ret void
}
; Vectorizeable, requires improvements in dereferenceability checks
-define void @uncountable_exit_with_invariant_but_unknown_stride(ptr dereferenceable(4000) noalias %array, ptr align 2 dereferenceable(4000) readonly %pred, i64 %stride) {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_invariant_but_unknown_stride'
-; CHECK: LV: Not vectorizing: Cannot determine exact exit count for latch block.
+define void @uncountable_exit_with_invariant_but_unknown_stride(ptr dereferenceable(4000) noalias %array, ptr align 2 dereferenceable(4000) readonly %pred, i64 %stride) !dbg !62 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_invariant_but_unknown_stride'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot determine exact exit count for latch block.
+; CHECK-REMARK: foo.c:270:3: loop not vectorized: Cannot vectorize early exit loop
+; CHECK-REMARK-NEXT: foo.c:270:3: loop not vectorized: could not determine number of loop iterations
entry:
- br label %for.body
+ br label %for.body, !dbg !63
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -760,20 +812,20 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !63
for.inc:
%iv.next = add nuw nsw i64 %iv, %stride
%counted.cond = icmp slt i64 %iv.next, 2001
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !63
exit:
ret void
}
define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_separate_exit_block'
-; CHECK: LV: We can vectorize this loop!
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_separate_exit_block'
+; CHECK-DEBUG: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -800,37 +852,68 @@ exit.uncountable:
ret i32 1
}
-define i32 @uncountable_exit_with_masked_ldst_separate_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 readonly %st.pred) {
-; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_masked_ldst_separate_condition'
-; CHECK: LV: We can vectorize this loop!
-; CHECK: LV: Vectorization is possible but not beneficial.
+; This loop passes legality checks, but fails in vplan due to unsupported
+; getelementptr used for the critical load that feeds the exit condition.
+define void @uncountable_exit_condition_load_offset_from_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(42) readonly %pred) !dbg !64 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_condition_load_offset_from_iv'
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with side effects contains unsupported critical load.
+; CHECK-REMARK: foo.c:280:3: loop not vectorized: Early exit loop with side effects contains unsupported critical load.
entry:
- br label %for.body
+ br label %for.body, !dbg !65
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.offset = add nuw nsw i64 %iv, 1
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %ee.offset
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !65
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !65
+
+exit:
+ ret void
+}
+
+define i32 @uncountable_exit_with_masked_ldst_separate_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 readonly %st.pred) !dbg !66 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_masked_ldst_separate_condition'
+; CHECK-DEBUG: LV: We can vectorize this loop!
+; CHECK-DEBUG: LV: Vectorization is possible but not beneficial.
+; CHECK-REMARK: foo.c:290:3: loop not vectorized: Early exit loop with side effects contains unsupported memory operations.
+entry:
+ br label %for.body, !dbg !67
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
%stp.gep = getelementptr inbounds nuw i16, ptr %st.pred, i64 %iv
%stp.val = load i16, ptr %stp.gep, align 2
%stp.cond = icmp slt i16 %stp.val, 2345
- br i1 %stp.cond, label %ldst.block, label %ee.block
+ br i1 %stp.cond, label %ldst.block, label %ee.block, !dbg !67
ldst.block:
%st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
%data = load i16, ptr %st.addr, align 2
%inc = add nsw i16 %data, 1
store i16 %inc, ptr %st.addr, align 2
- br label %ee.block
+ br label %ee.block, !dbg !67
ee.block:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit.uncountable, label %for.inc
+ br i1 %ee.cond, label %exit.uncountable, label %for.inc, !dbg !67
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit.countable, label %for.body
+ br i1 %counted.cond, label %exit.countable, label %for.body, !dbg !67
exit.countable:
ret i32 0
@@ -842,11 +925,12 @@ exit.uncountable:
;; Avoid vectorization; similar to another invariant test above, we would either
;; exit immediately on the first lane or never take the early exit. Should be
;; versioned before reaching LV.
-define void @novec_uncountable_exit_condition_address_is_invariant(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(2) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'novec_uncountable_exit_condition_address_is_invariant'
-; CHECK: LV: Not vectorizing: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop.
+define void @novec_uncountable_exit_condition_address_is_invariant(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(2) readonly %pred) !dbg !68 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'novec_uncountable_exit_condition_address_is_invariant'
+; CHECK-DEBUG: LV: Not vectorizing: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop.
+; CHECK-REMARK: foo.c:300:3: loop not vectorized: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop
entry:
- br label %for.body
+ br label %for.body, !dbg !69
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
@@ -856,12 +940,12 @@ for.body:
store i16 %inc, ptr %st.addr, align 2
%ee.val = load i16, ptr %pred, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !69
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !69
exit:
ret void
@@ -870,16 +954,17 @@ exit:
;; Avoid vectorization; similar to the test above, we would either exit
;; immediately on the first lane or never take the early exit. Should be
;; versioned before reaching LV.
-define void @novec_uncountable_exit_condition_address_is_addrec_in_outer_loop(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(2) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'novec_uncountable_exit_condition_address_is_addrec_in_outer_loop'
-; CHECK: LV: Not vectorizing: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop.
+define void @novec_uncountable_exit_condition_address_is_addrec_in_outer_loop(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(2) readonly %pred) !dbg !70 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'novec_uncountable_exit_condition_address_is_addrec_in_outer_loop'
+; CHECK-DEBUG: LV: Not vectorizing: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop.
+; CHECK-REMARK: foo.c:310:3: loop not vectorized: Uncountable exit condition depends on load with an address that is not an add recurrence in the loop
entry:
- br label %outer.body
+ br label %outer.body, !dbg !71
outer.body:
%outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.inc ]
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %outer.iv
- br label %for.body
+ br label %for.body, !dbg !71
for.body:
%iv = phi i64 [ 0, %outer.body ], [ %iv.next, %for.inc ]
@@ -889,17 +974,17 @@ for.body:
store i16 %inc, ptr %st.addr, align 2
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !71
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
- br i1 %counted.cond, label %outer.inc, label %for.body
+ br i1 %counted.cond, label %outer.inc, label %for.body, !dbg !71
outer.inc:
%outer.iv.next = add nuw nsw i64 %outer.iv, 1
%outer.cond = icmp eq i64 %outer.iv.next, 2
- br i1 %outer.cond, label %exit, label %outer.body
+ br i1 %outer.cond, label %exit, label %outer.body, !dbg !71
exit:
ret void
@@ -908,63 +993,66 @@ exit:
;; ICE was caused by assert for the load used in the uncountable exit condition
;; being guaranteed to execute.
@ee.global = external global [4 x i8]
-define void @crash_conditional_load_for_uncountable_exit(ptr dereferenceable(40) noalias %store.area) {
-; CHECK-LABEL: LV: Checking a loop in 'crash_conditional_load_for_uncountable_exit'
-; CHECK: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+define void @crash_conditional_load_for_uncountable_exit(ptr dereferenceable(40) noalias %store.area) !dbg !72 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'crash_conditional_load_for_uncountable_exit'
+; CHECK-DEBUG: LV: Not vectorizing: Load for uncountable exit not guaranteed to execute.
+; CHECK-REMARK: foo.c:320:3: loop not vectorized: Load for uncountable exit not guaranteed to execute
entry:
- br label %for.body
+ br label %for.body, !dbg !73
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
%ee.addr = getelementptr i8, ptr @ee.global, i64 %iv
- br i1 false, label %ee.block, label %invalid.block
+ br i1 false, label %ee.block, label %invalid.block, !dbg !73
ee.block:
%ee.val = load i8, ptr %ee.addr, align 1
store i16 0, ptr %store.area, align 2
%ee.cmp = icmp eq i8 %ee.val, 0
- br i1 %ee.cmp, label %for.inc, label %invalid.block
+ br i1 %ee.cmp, label %for.inc, label %invalid.block, !dbg !73
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 10
- br i1 %counted.cond, label %invalid.block, label %for.body
+ br i1 %counted.cond, label %invalid.block, label %for.body, !dbg !73
invalid.block:
unreachable
}
-define void @crash_conditional_load_for_uncountable_exit_argptr(ptr dereferenceable(40) noalias %store.area, ptr dereferenceable(4) %load.area, i1 %skip.cond) {
-; CHECK-LABEL: LV: Checking a loop in 'crash_conditional_load_for_uncountable_exit_argptr'
-; CHECK: LV: Not vectorizing: Early exit loop with store but no supported condition load.
+define void @crash_conditional_load_for_uncountable_exit_argptr(ptr dereferenceable(40) noalias %store.area, ptr dereferenceable(4) %load.area, i1 %skip.cond) !dbg !74 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'crash_conditional_load_for_uncountable_exit_argptr'
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with store but no supported condition load.
+; CHECK-REMARK: foo.c:330:3: loop not vectorized: Early exit loop with store but no supported condition load
entry:
- br label %for.body
+ br label %for.body, !dbg !75
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
%ee.addr = getelementptr i8, ptr %load.area, i64 %iv
- br i1 %skip.cond, label %ee.block, label %invalid.block
+ br i1 %skip.cond, label %ee.block, label %invalid.block, !dbg !75
ee.block:
%ee.val = load i8, ptr %ee.addr, align 1
store i16 0, ptr %store.area, align 2
%ee.cmp = icmp eq i8 %ee.val, 0
- br i1 %ee.cmp, label %for.inc, label %invalid.block
+ br i1 %ee.cmp, label %for.inc, label %invalid.block, !dbg !75
for.inc:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 10
- br i1 %counted.cond, label %invalid.block, label %for.body
+ br i1 %counted.cond, label %invalid.block, label %for.body, !dbg !75
invalid.block:
unreachable
}
-define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %src, ptr align 4 dereferenceable(80) noalias %dst, ptr align 4 dereferenceable(80) readonly %pred) {
-; CHECK-LABEL: LV: Checking a loop in 'combined_exit_conditions'
-; CHECK: LV: Not vectorizing: Cannot vectorize uncountable loop.
+define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %src, ptr align 4 dereferenceable(80) noalias %dst, ptr align 4 dereferenceable(80) readonly %pred) !dbg !76 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'combined_exit_conditions'
+; CHECK-DEBUG: LV: Not vectorizing: Cannot vectorize uncountable loop.
+; CHECK-REMARK: foo.c:340:3: loop not vectorized: Cannot vectorize uncountable loop
entry:
- br label %for.body
+ br label %for.body, !dbg !77
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
@@ -979,11 +1067,107 @@ for.body:
%iv.next = add nuw nsw i64 %iv, 1
%counted.cmp = icmp eq i64 %iv.next, 20
%combined.cond = select i1 %ee.cmp, i1 true, i1 %counted.cmp
- br i1 %combined.cond, label %exit, label %for.body
+ br i1 %combined.cond, label %exit, label %for.body, !dbg !77
exit:
ret void
}
+define i64 @uncountable_exit_with_extra_induction(ptr dereferenceable(80) noalias %array, ptr align 4 dereferenceable(80) readonly %pred) !dbg !78 {
+; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_extra_induction'
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with side effects contains unsupported reductions or recurrences.
+; CHECK-REMARK: foo.c:350:3: loop not vectorized: Early exit loop with side effects contains unsupported reductions or recurrences.
+entry:
+ br label %for.body, !dbg !79
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %j = phi i64 [ 100, %entry ], [ %j.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i32, ptr %array, i64 %iv
+ %j.trunc = trunc i64 %j to i32
+ store i32 %j.trunc, ptr %st.addr, align 4
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i64 %iv
+ %ee.val = load i32, ptr %ee.addr, align 4
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !79
+
+for.inc:
+ %j.next = add nuw nsw i64 %j, 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !79
+
+exit:
+ %res = phi i64 [ %j, %for.body ], [ %j.next, %for.inc ]
+ ret i64 %res
+}
+
+
declare void @init_mem(ptr, i64);
declare i64 @get_an_unknown_offset();
+
+!llvm.dbg.cu = !{!1000}
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 1, !"PIC Level", i32 2}
+!2 = !DIFile(filename: "foo.c", directory: "")
+!3 = !DISubroutineType(types: !4)
+!4 = !{}
+!10 = distinct !DISubprogram(name: "loop_contains_store", scope: !2, file: !2, line: 10, type: !3, isLocal: false, isDefinition: true, scopeLine: 10, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!11 = !DILocation(line: 10, column: 3, scope: !10)
+!12 = distinct !DISubprogram(name: "loop_contains_store_multidim_condition_load", scope: !2, file: !2, line: 20, type: !3, isLocal: false, isDefinition: true, scopeLine: 20, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!13 = !DILocation(line: 20, column: 3, scope: !12)
+!16 = distinct !DISubprogram(name: "novec_loop_contains_store_ee_condition_is_invariant", scope: !2, file: !2, line: 40, type: !3, isLocal: false, isDefinition: true, scopeLine: 40, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!17 = !DILocation(line: 40, column: 3, scope: !16)
+!18 = distinct !DISubprogram(name: "loop_contains_store_fcmp_condition", scope: !2, file: !2, line: 50, type: !3, isLocal: false, isDefinition: true, scopeLine: 50, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!19 = !DILocation(line: 50, column: 3, scope: !18)
+!20 = distinct !DISubprogram(name: "loop_contains_store_safe_dependency", scope: !2, file: !2, line: 60, type: !3, isLocal: false, isDefinition: true, scopeLine: 60, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!21 = !DILocation(line: 60, column: 3, scope: !20)
+!22 = distinct !DISubprogram(name: "loop_contains_store_unsafe_dependency", scope: !2, file: !2, line: 70, type: !3, isLocal: false, isDefinition: true, scopeLine: 70, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!23 = !DILocation(line: 70, column: 3, scope: !22)
+!30 = distinct !DISubprogram(name: "novec_loop_contains_store_volatile", scope: !2, file: !2, line: 110, type: !3, isLocal: false, isDefinition: true, scopeLine: 110, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!31 = !DILocation(line: 110, column: 3, scope: !30)
+!32 = distinct !DISubprogram(name: "loop_contains_store_to_invariant_location", scope: !2, file: !2, line: 120, type: !3, isLocal: false, isDefinition: true, scopeLine: 120, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!33 = !DILocation(line: 120, column: 3, scope: !32)
+!36 = distinct !DISubprogram(name: "loop_contains_store_requiring_alias_check", scope: !2, file: !2, line: 140, type: !3, isLocal: false, isDefinition: true, scopeLine: 140, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!37 = !DILocation(line: 140, column: 3, scope: !36)
+!40 = distinct !DISubprogram(name: "loop_contains_store_condition_load_requires_gather", scope: !2, file: !2, line: 160, type: !3, isLocal: false, isDefinition: true, scopeLine: 160, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!41 = !DILocation(line: 160, column: 3, scope: !40)
+!42 = distinct !DISubprogram(name: "loop_contains_store_uncounted_exit_is_a_switch", scope: !2, file: !2, line: 170, type: !3, isLocal: false, isDefinition: true, scopeLine: 170, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!43 = !DILocation(line: 170, column: 3, scope: !42)
+!44 = distinct !DISubprogram(name: "loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute", scope: !2, file: !2, line: 180, type: !3, isLocal: false, isDefinition: true, scopeLine: 180, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!45 = !DILocation(line: 180, column: 3, scope: !44)
+!46 = distinct !DISubprogram(name: "test_nodep", scope: !2, file: !2, line: 190, type: !3, isLocal: false, isDefinition: true, scopeLine: 190, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!47 = !DILocation(line: 190, column: 3, scope: !46)
+!48 = distinct !DISubprogram(name: "histogram_with_uncountable_exit", scope: !2, file: !2, line: 200, type: !3, isLocal: false, isDefinition: true, scopeLine: 200, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!49 = !DILocation(line: 200, column: 3, scope: !48)
+!50 = distinct !DISubprogram(name: "loop_contains_store_between_two_early_exits", scope: !2, file: !2, line: 210, type: !3, isLocal: false, isDefinition: true, scopeLine: 210, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!51 = !DILocation(line: 210, column: 3, scope: !50)
+!52 = distinct !DISubprogram(name: "loop_contains_store_before_two_early_exits", scope: !2, file: !2, line: 220, type: !3, isLocal: false, isDefinition: true, scopeLine: 220, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!53 = !DILocation(line: 220, column: 3, scope: !52)
+!54 = distinct !DISubprogram(name: "one_uncountable_two_countable_exits", scope: !2, file: !2, line: 230, type: !3, isLocal: false, isDefinition: true, scopeLine: 230, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!55 = !DILocation(line: 230, column: 3, scope: !54)
+!56 = distinct !DISubprogram(name: "uncountable_exit_with_reduction", scope: !2, file: !2, line: 240, type: !3, isLocal: false, isDefinition: true, scopeLine: 240, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!57 = !DILocation(line: 240, column: 3, scope: !56)
+!60 = distinct !DISubprogram(name: "uncountable_exit_with_constant_nonunit_stride", scope: !2, file: !2, line: 260, type: !3, isLocal: false, isDefinition: true, scopeLine: 260, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!61 = !DILocation(line: 260, column: 3, scope: !60)
+!62 = distinct !DISubprogram(name: "uncountable_exit_with_invariant_but_unknown_stride", scope: !2, file: !2, line: 270, type: !3, isLocal: false, isDefinition: true, scopeLine: 270, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!63 = !DILocation(line: 270, column: 3, scope: !62)
+!64 = distinct !DISubprogram(name: "uncountable_exit_condition_load_offset_from_iv", scope: !2, file: !2, line: 280, type: !3, isLocal: false, isDefinition: true, scopeLine: 280, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!65 = !DILocation(line: 280, column: 3, scope: !64)
+!66 = distinct !DISubprogram(name: "uncountable_exit_with_masked_ldst_separate_condition", scope: !2, file: !2, line: 290, type: !3, isLocal: false, isDefinition: true, scopeLine: 290, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!67 = !DILocation(line: 290, column: 3, scope: !66)
+!68 = distinct !DISubprogram(name: "novec_uncountable_exit_condition_address_is_invariant", scope: !2, file: !2, line: 300, type: !3, isLocal: false, isDefinition: true, scopeLine: 300, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!69 = !DILocation(line: 300, column: 3, scope: !68)
+!70 = distinct !DISubprogram(name: "novec_uncountable_exit_condition_address_is_addrec_in_outer_loop", scope: !2, file: !2, line: 310, type: !3, isLocal: false, isDefinition: true, scopeLine: 310, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!71 = !DILocation(line: 310, column: 3, scope: !70)
+!72 = distinct !DISubprogram(name: "crash_conditional_load_for_uncountable_exit", scope: !2, file: !2, line: 320, type: !3, isLocal: false, isDefinition: true, scopeLine: 320, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!73 = !DILocation(line: 320, column: 3, scope: !72)
+!74 = distinct !DISubprogram(name: "crash_conditional_load_for_uncountable_exit_argptr", scope: !2, file: !2, line: 330, type: !3, isLocal: false, isDefinition: true, scopeLine: 330, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!75 = !DILocation(line: 330, column: 3, scope: !74)
+!76 = distinct !DISubprogram(name: "combined_exit_conditions", scope: !2, file: !2, line: 340, type: !3, isLocal: false, isDefinition: true, scopeLine: 340, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!77 = !DILocation(line: 340, column: 3, scope: !76)
+!78 = distinct !DISubprogram(name: "uncountable_exit_with_extra_induction", scope: !2, file: !2, line: 350, type: !3, isLocal: false, isDefinition: true, scopeLine: 350, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!79 = !DILocation(line: 350, column: 3, scope: !78)
+!1000 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang", file: !2, isOptimized: true, flags: "-O2", splitDebugFilename: "abc.debug", emissionKind: 2)
>From a57c0d2550dab1d92d21682c7e0260ed7636b444 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Mon, 21 Sep 2026 13:04:31 +0000
Subject: [PATCH 2/3] Fix formatting
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 30 +++++++++----------
1 file changed, 14 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a8a92de5e64cdd..4de9044118c738 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3213,10 +3213,9 @@ static bool handleUncountableExitsWithSideEffects(
VPValue *Cond = getRecipesForUncountableExit(ConditionRecipes, LatchVPBB);
if (!Cond) {
- reportVectorizationFailure(
- "Unable to determine early exit condition for loop with side effects.\n",
- "EarlyExitSideEffectsCond",
- ORE, TheLoop);
+ reportVectorizationFailure("Unable to determine early exit condition for "
+ "loop with side effects.\n",
+ "EarlyExitSideEffectsCond", ORE, TheLoop);
return false;
}
@@ -3255,8 +3254,8 @@ static bool handleUncountableExitsWithSideEffects(
&Predicates)) {
reportVectorizationFailure("Early exit loop with side effects contains "
"critical load that may fault.\n",
- "EarlyExitSideEffectsFaultingLoad",
- ORE, TheLoop);
+ "EarlyExitSideEffectsFaultingLoad", ORE,
+ TheLoop);
return false;
}
}
@@ -3270,10 +3269,10 @@ static bool handleUncountableExitsWithSideEffects(
return false;
if (!match(Ptr, m_VPInstruction<Instruction::GetElementPtr>(
m_LiveIn(), m_Specific(IV)))) {
- reportVectorizationFailure(
- "Early exit loop with side effects contains unsupported critical load.\n",
- "EarlyExitSideEffectsBadCriticalLoad",
- ORE, TheLoop);
+ reportVectorizationFailure("Early exit loop with side effects contains "
+ "unsupported critical load.\n",
+ "EarlyExitSideEffectsBadCriticalLoad", ORE,
+ TheLoop);
return false;
}
@@ -3308,10 +3307,10 @@ static bool handleUncountableExitsWithSideEffects(
if (R.mayReadOrWriteMemory() && &R != Load) {
// TODO: Handle conditional memory operations in the loop.
if (!VPDT.dominates(R.getParent(), LatchVPBB)) {
- reportVectorizationFailure("Early exit loop with side effects "
- "contains unsupported memory operations.\n",
- "EarlyExitSideEffectsUnsupportedMemOps",
- ORE, TheLoop);
+ reportVectorizationFailure(
+ "Early exit loop with side effects "
+ "contains unsupported memory operations.\n",
+ "EarlyExitSideEffectsUnsupportedMemOps", ORE, TheLoop);
return false;
}
cast<VPInstruction>(&R)->addMask(Mask);
@@ -3339,8 +3338,7 @@ static bool handleUncountableExitsWithSideEffects(
if (range_size(Phis) != 1) {
reportVectorizationFailure("Early exit loop with side effects contains "
"unsupported reductions or recurrences.\n",
- "EarlyExitEffectsSideReductions",
- ORE, TheLoop);
+ "EarlyExitEffectsSideReductions", ORE, TheLoop);
return false;
}
VPPhi *ContinueIV = cast<VPPhi>(Phis.begin());
>From ee1c5383245f65aa8fc9501478bf442fe0b764f1 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Tue, 22 Sep 2026 09:33:22 +0000
Subject: [PATCH 3/3] Address review comments
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 15 ++++++++++++---
.../LoopVectorize/early_exit_store_legality.ll | 17 ++++++++++-------
2 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4de9044118c738..d507138334041f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3253,7 +3253,8 @@ static bool handleUncountableExitsWithSideEffects(
PSE.getSE()->getConstant(EltSize), TheLoop, *PSE.getSE(), DT, AC,
&Predicates)) {
reportVectorizationFailure("Early exit loop with side effects contains "
- "critical load that may fault.\n",
+ "load used by the exit condition that may "
+ "fault.\n",
"EarlyExitSideEffectsFaultingLoad", ORE,
TheLoop);
return false;
@@ -3265,12 +3266,20 @@ static bool handleUncountableExitsWithSideEffects(
// accesses for the condition load right now.
auto *IV = cast<VPWidenInductionRecipe>(&HeaderVPBB->front());
if (!match(IV->getStartValue(), m_SpecificInt(0)) ||
- !match(IV->getStepValue(), m_SpecificInt(1)))
+ !match(IV->getStepValue(), m_SpecificInt(1))) {
+ reportVectorizationFailure("Early exit loop with side effects contains "
+ "non-contiguous load used by the exit "
+ "condition.\n",
+ "EarlyExitSideEffectsBadCriticalLoad", ORE,
+ TheLoop);
return false;
+ }
+
if (!match(Ptr, m_VPInstruction<Instruction::GetElementPtr>(
m_LiveIn(), m_Specific(IV)))) {
reportVectorizationFailure("Early exit loop with side effects contains "
- "unsupported critical load.\n",
+ "unsupported load used by the exit "
+ "condition.\n",
"EarlyExitSideEffectsBadCriticalLoad", ORE,
TheLoop);
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index f29f00590a08d6..6fd5237564d9fe 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -428,11 +428,12 @@ exit:
ret void
}
-define void @loop_contains_store_decrementing_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+define void @loop_contains_store_decrementing_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) !dbg !38 {
; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_decrementing_iv'
-; CHECK-DEBUG: LV: We can vectorize this loop!
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with side effects contains non-contiguous load used by the exit condition.
+; CHECK-REMARK: foo.c:150:3: loop not vectorized: Early exit loop with side effects contains non-contiguous load used by the exit condition.
entry:
- br label %for.body
+ br label %for.body, !dbg !39
for.body:
%iv = phi i64 [ 19, %entry ], [ %iv.next, %for.inc ]
@@ -443,12 +444,12 @@ for.body:
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
- br i1 %ee.cond, label %exit, label %for.inc
+ br i1 %ee.cond, label %exit, label %for.inc, !dbg !39
for.inc:
%iv.next = sub nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 0
- br i1 %counted.cond, label %exit, label %for.body
+ br i1 %counted.cond, label %exit, label %for.body, !dbg !39
exit:
ret void
@@ -856,8 +857,8 @@ exit.uncountable:
; getelementptr used for the critical load that feeds the exit condition.
define void @uncountable_exit_condition_load_offset_from_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(42) readonly %pred) !dbg !64 {
; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_condition_load_offset_from_iv'
-; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with side effects contains unsupported critical load.
-; CHECK-REMARK: foo.c:280:3: loop not vectorized: Early exit loop with side effects contains unsupported critical load.
+; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with side effects contains unsupported load used by the exit condition.
+; CHECK-REMARK: foo.c:280:3: loop not vectorized: Early exit loop with side effects contains unsupported load used by the exit condition.
entry:
br label %for.body, !dbg !65
@@ -1132,6 +1133,8 @@ declare i64 @get_an_unknown_offset();
!33 = !DILocation(line: 120, column: 3, scope: !32)
!36 = distinct !DISubprogram(name: "loop_contains_store_requiring_alias_check", scope: !2, file: !2, line: 140, type: !3, isLocal: false, isDefinition: true, scopeLine: 140, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
!37 = !DILocation(line: 140, column: 3, scope: !36)
+!38 = distinct !DISubprogram(name: "loop_contains_store_decrementing_iv", scope: !2, file: !2, line: 150, type: !3, isLocal: false, isDefinition: true, scopeLine: 150, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
+!39 = !DILocation(line: 150, column: 3, scope: !38)
!40 = distinct !DISubprogram(name: "loop_contains_store_condition_load_requires_gather", scope: !2, file: !2, line: 160, type: !3, isLocal: false, isDefinition: true, scopeLine: 160, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
!41 = !DILocation(line: 160, column: 3, scope: !40)
!42 = distinct !DISubprogram(name: "loop_contains_store_uncounted_exit_is_a_switch", scope: !2, file: !2, line: 170, type: !3, isLocal: false, isDefinition: true, scopeLine: 170, flags: DIFlagPrototyped, isOptimized: true, unit: !1000, retainedNodes: !4)
More information about the llvm-commits
mailing list