[llvm] [LV] Add tests for early-exit loops with faulting loads (NFC). (PR #224762)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 20 07:46:16 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/224762
>From 1b6bcc2c6627e19d892f66834e7c1e90006c8f4d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 24 Mar 2026 12:08:15 +0000
Subject: [PATCH] [LV] Add tests for early-exit loops with faulting loads
(NFC).
Pre-committing the tests for using @llvm.speculative.load intrinsic in
LoopVectorize.
---
.../CostModel/AArch64/speculative-load.ll | 77 ++++
.../CostModel/X86/speculative-load.ll | 27 ++
.../early-exit-with-speculative-load-basic.ll | 284 ++++++++++++
...y-exit-with-speculative-load-load-types.ll | 231 ++++++++++
...early-exit-with-speculative-load-safety.ll | 99 +++++
...rly-exit-with-speculative-load-scalable.ll | 54 +++
...early-exit-with-speculative-load-oracle.ll | 419 ++++++++++++++++++
.../LoopVectorize/unsupported_early_exit.ll | 306 +++++++++++++
8 files changed, 1497 insertions(+)
create mode 100644 llvm/test/Analysis/CostModel/AArch64/speculative-load.ll
create mode 100644 llvm/test/Analysis/CostModel/X86/speculative-load.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-load-types.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-safety.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-scalable.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/AArch64/early-exit-with-speculative-load-oracle.ll
diff --git a/llvm/test/Analysis/CostModel/AArch64/speculative-load.ll b/llvm/test/Analysis/CostModel/AArch64/speculative-load.ll
new file mode 100644
index 0000000000000..5268aee2aa200
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/speculative-load.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=COMMON,NEON
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s --check-prefixes=COMMON,SVE
+
+define void @speculative_load_cost_fixed(ptr %p) {
+ ; Scalar types - all valid (<= 16 bytes)
+; COMMON-LABEL: 'speculative_load_cost_fixed'
+; COMMON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = call b8 (ptr, i1, ...) @llvm.speculative.load.b8.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = call b16 (ptr, i1, ...) @llvm.speculative.load.b16.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = call b32 (ptr, i1, ...) @llvm.speculative.load.b32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = call b64 (ptr, i1, ...) @llvm.speculative.load.b64.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = call b128 (ptr, i1, ...) @llvm.speculative.load.b128.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %6 = call <2 x i32> (ptr, i1, ...) @llvm.speculative.load.v2i32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %7 = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %8 = call <2 x i64> (ptr, i1, ...) @llvm.speculative.load.v2i64.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %9 = call <4 x float> (ptr, i1, ...) @llvm.speculative.load.v4f32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = call <2 x double> (ptr, i1, ...) @llvm.speculative.load.v2f64.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %11 = call <8 x i8> (ptr, i1, ...) @llvm.speculative.load.v8i8.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %12 = call <16 x i8> (ptr, i1, ...) @llvm.speculative.load.v16i8.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %13 = call <4 x i16> (ptr, i1, ...) @llvm.speculative.load.v4i16.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %14 = call <8 x i16> (ptr, i1, ...) @llvm.speculative.load.v8i16.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %15 = call <8 x i32> (ptr, i1, ...) @llvm.speculative.load.v8i32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %16 = call <4 x i64> (ptr, i1, ...) @llvm.speculative.load.v4i64.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %17 = call <32 x i8> (ptr, i1, ...) @llvm.speculative.load.v32i8.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %18 = call <16 x i16> (ptr, i1, ...) @llvm.speculative.load.v16i16.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %19 = call <8 x float> (ptr, i1, ...) @llvm.speculative.load.v8f32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %20 = call <4 x double> (ptr, i1, ...) @llvm.speculative.load.v4f64.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ call b8 (ptr, i1, ...) @llvm.speculative.load.b8.p0(ptr %p, i1 false, i64 0)
+ call b16 (ptr, i1, ...) @llvm.speculative.load.b16.p0(ptr %p, i1 false, i64 0)
+ call b32 (ptr, i1, ...) @llvm.speculative.load.b32.p0(ptr %p, i1 false, i64 0)
+ call b64 (ptr, i1, ...) @llvm.speculative.load.b64.p0(ptr %p, i1 false, i64 0)
+ call b128 (ptr, i1, ...) @llvm.speculative.load.b128.p0(ptr %p, i1 false, i64 0)
+
+ ; Vector types <= 16 bytes - valid
+ call <2 x i32> (ptr, i1, ...) @llvm.speculative.load.v2i32.p0(ptr %p, i1 false, i64 0)
+ call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, i64 0)
+ call <2 x i64> (ptr, i1, ...) @llvm.speculative.load.v2i64.p0(ptr %p, i1 false, i64 0)
+ call <4 x float> (ptr, i1, ...) @llvm.speculative.load.v4f32.p0(ptr %p, i1 false, i64 0)
+ call <2 x double> (ptr, i1, ...) @llvm.speculative.load.v2f64.p0(ptr %p, i1 false, i64 0)
+ call <8 x i8> (ptr, i1, ...) @llvm.speculative.load.v8i8.p0(ptr %p, i1 false, i64 0)
+ call <16 x i8> (ptr, i1, ...) @llvm.speculative.load.v16i8.p0(ptr %p, i1 false, i64 0)
+ call <4 x i16> (ptr, i1, ...) @llvm.speculative.load.v4i16.p0(ptr %p, i1 false, i64 0)
+ call <8 x i16> (ptr, i1, ...) @llvm.speculative.load.v8i16.p0(ptr %p, i1 false, i64 0)
+
+ ; Vector types > 16 bytes - invalid
+ call <8 x i32> (ptr, i1, ...) @llvm.speculative.load.v8i32.p0(ptr %p, i1 false, i64 0)
+ call <4 x i64> (ptr, i1, ...) @llvm.speculative.load.v4i64.p0(ptr %p, i1 false, i64 0)
+ call <32 x i8> (ptr, i1, ...) @llvm.speculative.load.v32i8.p0(ptr %p, i1 false, i64 0)
+ call <16 x i16> (ptr, i1, ...) @llvm.speculative.load.v16i16.p0(ptr %p, i1 false, i64 0)
+ call <8 x float> (ptr, i1, ...) @llvm.speculative.load.v8f32.p0(ptr %p, i1 false, i64 0)
+ call <4 x double> (ptr, i1, ...) @llvm.speculative.load.v4f64.p0(ptr %p, i1 false, i64 0)
+ ret void
+}
+
+define void @speculative_load_cost_scalable(ptr %p) {
+; COMMON-LABEL: 'speculative_load_cost_scalable'
+; COMMON-NEXT: Cost Model: Invalid cost for instruction: %1 = call <vscale x 2 x i64> (ptr, i1, ...) @llvm.speculative.load.nxv2i64.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Invalid cost for instruction: %2 = call <vscale x 4 x i32> (ptr, i1, ...) @llvm.speculative.load.nxv4i32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Invalid cost for instruction: %3 = call <vscale x 8 x i16> (ptr, i1, ...) @llvm.speculative.load.nxv8i16.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Invalid cost for instruction: %4 = call <vscale x 16 x i8> (ptr, i1, ...) @llvm.speculative.load.nxv16i8.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Invalid cost for instruction: %5 = call <vscale x 2 x double> (ptr, i1, ...) @llvm.speculative.load.nxv2f64.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Invalid cost for instruction: %6 = call <vscale x 4 x float> (ptr, i1, ...) @llvm.speculative.load.nxv4f32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Invalid cost for instruction: %7 = call <vscale x 8 x float> (ptr, i1, ...) @llvm.speculative.load.nxv8f32.p0(ptr %p, i1 false, i64 0)
+; COMMON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ call <vscale x 2 x i64> (ptr, i1, ...) @llvm.speculative.load.nxv2i64.p0(ptr %p, i1 false, i64 0)
+ call <vscale x 4 x i32> (ptr, i1, ...) @llvm.speculative.load.nxv4i32.p0(ptr %p, i1 false, i64 0)
+ call <vscale x 8 x i16> (ptr, i1, ...) @llvm.speculative.load.nxv8i16.p0(ptr %p, i1 false, i64 0)
+ call <vscale x 16 x i8> (ptr, i1, ...) @llvm.speculative.load.nxv16i8.p0(ptr %p, i1 false, i64 0)
+ call <vscale x 2 x double> (ptr, i1, ...) @llvm.speculative.load.nxv2f64.p0(ptr %p, i1 false, i64 0)
+ call <vscale x 4 x float> (ptr, i1, ...) @llvm.speculative.load.nxv4f32.p0(ptr %p, i1 false, i64 0)
+ call <vscale x 8 x float> (ptr, i1, ...) @llvm.speculative.load.nxv8f32.p0(ptr %p, i1 false, i64 0)
+
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/speculative-load.ll b/llvm/test/Analysis/CostModel/X86/speculative-load.ll
new file mode 100644
index 0000000000000..81b3b56eeb10c
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/speculative-load.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=x86_64 < %s | FileCheck %s
+
+define void @speculative_load_cost(ptr %p) {
+; CHECK-LABEL: 'speculative_load_cost'
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = call b8 (ptr, i1, ...) @llvm.speculative.load.b8.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = call b16 (ptr, i1, ...) @llvm.speculative.load.b16.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = call b32 (ptr, i1, ...) @llvm.speculative.load.b32.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = call b64 (ptr, i1, ...) @llvm.speculative.load.b64.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %5 = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %6 = call <8 x i32> (ptr, i1, ...) @llvm.speculative.load.v8i32.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %7 = call <2 x i64> (ptr, i1, ...) @llvm.speculative.load.v2i64.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %8 = call <4 x float> (ptr, i1, ...) @llvm.speculative.load.v4f32.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %9 = call <2 x double> (ptr, i1, ...) @llvm.speculative.load.v2f64.p0(ptr %p, i1 false, i64 0)
+; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+ call b8 (ptr, i1, ...) @llvm.speculative.load.b8.p0(ptr %p, i1 false, i64 0)
+ call b16 (ptr, i1, ...) @llvm.speculative.load.b16.p0(ptr %p, i1 false, i64 0)
+ call b32 (ptr, i1, ...) @llvm.speculative.load.b32.p0(ptr %p, i1 false, i64 0)
+ call b64 (ptr, i1, ...) @llvm.speculative.load.b64.p0(ptr %p, i1 false, i64 0)
+ call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr %p, i1 false, i64 0)
+ call <8 x i32> (ptr, i1, ...) @llvm.speculative.load.v8i32.p0(ptr %p, i1 false, i64 0)
+ call <2 x i64> (ptr, i1, ...) @llvm.speculative.load.v2i64.p0(ptr %p, i1 false, i64 0)
+ call <4 x float> (ptr, i1, ...) @llvm.speculative.load.v4f32.p0(ptr %p, i1 false, i64 0)
+ call <2 x double> (ptr, i1, ...) @llvm.speculative.load.v2f64.p0(ptr %p, i1 false, i64 0)
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll
new file mode 100644
index 0000000000000..8ca8b9ea7737c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll
@@ -0,0 +1,284 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+target triple = "arm64-apple-macosx"
+
+define i64 @find_first_mismatch(ptr %A, ptr %B, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
+ %ld.B = load i8, ptr %gep.B, align 1
+ %cmp = icmp ne i8 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; Same as @find_first_mismatch, but the induction is not live out.
+define i64 @find_first_mismatch_no_live_out(ptr %A, ptr %B, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
+ %ld.B = load i8, ptr %gep.B, align 1
+ %cmp = icmp ne i8 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 1
+
+exit:
+ ret i64 0
+}
+
+define i64 @early_and_latch_exit_to_same_block(ptr %A, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %cmp = icmp eq i8 %ld.A, 42
+ br i1 %cmp, label %exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+exit:
+ %res = phi i64 [ %iv, %loop.header ], [ -1, %loop.latch ]
+ ret i64 %res
+}
+
+define ptr @address_live_out(ptr %p, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr i8, ptr %p, i64 %iv
+ %l = load i8, ptr %gep, align 1
+ %c = icmp eq i8 %l, 0
+ br i1 %c, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop.header
+
+early.exit:
+ ret ptr %gep
+
+exit:
+ ret ptr null
+}
+
+define i64 @index_cast_live_out(ptr %p, i32 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %idx = zext i32 %iv to i64
+ %gep = getelementptr i8, ptr %p, i64 %idx
+ %l = load i8, ptr %gep, align 1
+ %c = icmp eq i8 %l, 0
+ br i1 %c, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop.header
+
+early.exit:
+ ret i64 %idx
+
+exit:
+ ret i64 -1
+}
+
+define i32 @masked_index_broadcast(ptr %a, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %add = add i64 %iv, %n
+ %cmp = icmp ugt i64 %add, 0
+ br i1 %cmp, label %loop.latch, label %body
+
+body:
+ %idx = and i64 %add, 4294967295
+ %gep = getelementptr [8 x i8], ptr %a, i64 %idx
+ %l = load i64, ptr %gep, align 8
+ %c = icmp eq i64 %l, 0
+ br i1 %c, label %loop.latch, label %early.exit
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv, %n
+ br i1 %ec, label %exit, label %loop.header
+
+early.exit:
+ ret i32 1
+
+exit:
+ ret i32 0
+}
+
+; CHECK-LABEL: define i64 @find_first_mismatch(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+;
+; CHECK-LABEL: define i64 @find_first_mismatch_no_live_out(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: ret i64 1
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 0
+;
+;
+; CHECK-LABEL: define i64 @early_and_latch_exit_to_same_block(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD_A]], 42
+; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ -1, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+;
+; CHECK-LABEL: define ptr @address_live_out(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[GEP_LCSSA:%.*]] = phi ptr [ [[GEP]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret ptr [[GEP_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret ptr null
+;
+;
+; CHECK-LABEL: define i64 @index_cast_live_out(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IDX:%.*]] = zext i32 [[IV]] to i64
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
+; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ [[IDX]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret i64 [[IDX_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+;
+; CHECK-LABEL: define i32 @masked_index_broadcast(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[ADD:%.*]] = add i64 [[IV]], [[N]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i64 [[ADD]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_LATCH]], label %[[BODY:.*]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[IDX:%.*]] = and i64 [[ADD]], 4294967295
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[IDX]]
+; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[C:%.*]] = icmp eq i64 [[L]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[EARLY_EXIT:.*]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: ret i32 1
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 0
+;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-load-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-load-types.ll
new file mode 100644
index 0000000000000..757856f2fbeaf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-load-types.ll
@@ -0,0 +1,231 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+target triple = "arm64-apple-macosx"
+
+define i64 @load_i16(ptr %A, ptr %B, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i16, ptr %A, i64 %iv
+ %ld.A = load i16, ptr %gep.A, align 2
+ %gep.B = getelementptr inbounds i16, ptr %B, i64 %iv
+ %ld.B = load i16, ptr %gep.B, align 2
+ %cmp = icmp ne i16 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @load_i32(ptr %A, ptr %B, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+ %ld.A = load i32, ptr %gep.A, align 4
+ %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+ %ld.B = load i32, ptr %gep.B, align 4
+ %cmp = icmp ne i32 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @load_double(ptr %A, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds double, ptr %A, i64 %iv
+ %ld.A = load double, ptr %gep.A, align 8
+ %bits = bitcast double %ld.A to i64
+ %cmp = icmp eq i64 %bits, 0
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @load_ptr(ptr %A, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds ptr, ptr %A, i64 %iv
+ %ld.A = load ptr, ptr %gep.A, align 8
+ %cmp = icmp eq ptr %ld.A, null
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @load_i1(ptr %A, i64 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i1, ptr %A, i64 %iv
+ %ld.A = load i1, ptr %gep.A, align 1
+ br i1 %ld.A, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+; CHECK-LABEL: define i64 @load_i16(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i16, ptr [[GEP_A]], align 2
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[LD_B:%.*]] = load i16, ptr [[GEP_B]], align 2
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i16 [[LD_A]], [[LD_B]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+;
+; CHECK-LABEL: define i64 @load_i32(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[LD_B:%.*]] = load i32, ptr [[GEP_B]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[LD_A]], [[LD_B]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+;
+; CHECK-LABEL: define i64 @load_double(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load double, ptr [[GEP_A]], align 8
+; CHECK-NEXT: [[BITS:%.*]] = bitcast double [[LD_A]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[BITS]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+;
+; CHECK-LABEL: define i64 @load_ptr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load ptr, ptr [[GEP_A]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq ptr [[LD_A]], null
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+;
+; CHECK-LABEL: define i64 @load_i1(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i1, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i1, ptr [[GEP_A]], align 1
+; CHECK-NEXT: br i1 [[LD_A]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-safety.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-safety.ll
new file mode 100644
index 0000000000000..4666532861e8c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-safety.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+target triple = "arm64-apple-macosx"
+
+ at K = internal global [4 x i8] [i8 9, i8 2, i8 9, i8 9], align 4
+
+; The invariant load is dereferenceable, it must be broadcast rather than
+; widened.
+define i64 @invariant_load(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @invariant_load(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr @K, align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %ld.B = load i8, ptr @K, align 1
+ %cmp = icmp eq i8 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; The load is frozen, oracle materialization may introduce additional users of freeze.
+define i64 @freeze_exit(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @freeze_exit(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[FR:%.*]] = freeze i8 [[LD_A]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[FR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %fr = freeze i8 %ld.A
+ %cmp = icmp eq i8 %fr, 0
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-scalable.ll
new file mode 100644
index 0000000000000..827179661f997
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-scalable.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 6
+; RUN: opt -p loop-vectorize -mattr=+sve -S %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; A scalable VF cannot provide the power-of-2 byte count required by the
+; @llvm.can.load.speculatively check, so a fixed VF is used.
+
+define i64 @find_first_eq_const(ptr %A, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %cmp = icmp eq i8 %ld.A, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; CHECK-LABEL: define i64 @find_first_eq_const(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD_A]], 42
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+;.
+; CHECK: attributes #[[ATTR0]] = { "target-features"="+sve" }
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/early-exit-with-speculative-load-oracle.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/early-exit-with-speculative-load-oracle.ll
new file mode 100644
index 0000000000000..7ae491ba18af8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/early-exit-with-speculative-load-oracle.ll
@@ -0,0 +1,419 @@
+; RUN: opt -p loop-vectorize -force-vector-width=4 -disable-output \
+; RUN: -vplan-print-after=printOptimizedVPlan %s 2>&1 | FileCheck %s --allow-empty
+; CHECK-NOT: VPlan for loop in
+
+target triple = "arm64-apple-macosx"
+
+ at G = external global [1024 x i8]
+
+define i64 @find_first_eq_const(ptr %A, i64 %n) {
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %cmp = icmp eq i8 %ld.A, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @find_first_eq_live_in(ptr %A, i64 %n, i8 %val) {
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %cmp = icmp eq i8 %ld.A, %val
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i32 @i32_induction(ptr %A, ptr %B, i32 %n) {
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i32 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %gep.B = getelementptr inbounds i8, ptr %B, i32 %iv
+ %ld.B = load i8, ptr %gep.B, align 1
+ %cmp = icmp ne i8 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp ne i32 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i32 %iv
+
+exit:
+ ret i32 -1
+}
+
+; Covers replaying scalar binops, div/rem with a constant divisor and freeze.
+define i64 @binop_chain(ptr %A, ptr %B, ptr %C, i64 %n) {
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
+ %ld.B = load i8, ptr %gep.B, align 1
+ %gep.C = getelementptr inbounds i8, ptr %C, i64 %iv
+ %ld.C = load i8, ptr %gep.C, align 1
+ %add = add i8 %ld.A, %ld.B
+ %sub = sub i8 %add, %ld.C
+ %mul = mul i8 %sub, 3
+ %xor = xor i8 %mul, 7
+ %div = udiv i8 %xor, 3
+ %fr = freeze i8 %div
+ %cmp = icmp eq i8 %fr, 0
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @predicated_exit_condition(ptr %mask, ptr %A, ptr %B, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.mask = getelementptr inbounds i8, ptr %mask, i64 %iv
+ %ld.mask = load i8, ptr %gep.mask, align 1
+ %mask.cmp = icmp ne i8 %ld.mask, 0
+ br i1 %mask.cmp, label %check, label %loop.latch
+
+check:
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
+ %ld.B = load i8, ptr %gep.B, align 1
+ %cmp = icmp ne i8 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; As @predicated_exit_condition, but the loads are on the arms of a diamond.
+define i64 @phi_merge_before_exit_condition(ptr %mask, ptr %A, ptr %B, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.mask = getelementptr inbounds i8, ptr %mask, i64 %iv
+ %ld.mask = load i8, ptr %gep.mask, align 1
+ %mask.cmp = icmp ne i8 %ld.mask, 0
+ br i1 %mask.cmp, label %if.true, label %if.false
+
+if.true:
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ br label %merge
+
+if.false:
+ %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
+ %ld.B = load i8, ptr %gep.B, align 1
+ br label %merge
+
+merge:
+ %val = phi i8 [ %ld.B, %if.false ], [ %ld.A, %if.true ]
+ %cmp = icmp eq i8 %val, 0
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @derived_induction_start(ptr %A, ptr %B, i64 %n) {
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %iv.off = phi i64 [ 10, %entry ], [ %iv.off.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv.off
+ %ld.A = load i8, ptr %gep.A, align 1
+ %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv.off
+ %ld.B = load i8, ptr %gep.B, align 1
+ %cmp = icmp ne i8 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %iv.off.next = add nuw nsw i64 %iv.off, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; Test with induction with step 2.
+define i64 @derived_induction_step(ptr %A, i64 %n) {
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %iv.2 = phi i64 [ 0, %entry ], [ %iv.2.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %trunc = trunc i64 %iv.2 to i8
+ %cmp = icmp ne i8 %ld.A, %trunc
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %iv.2.next = add nuw nsw i64 %iv.2, 2
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; Dead recipes are dropped before the oracle plan is built.
+define i64 @dead_recipes_in_loop(ptr %A, i64 %n) {
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %dead.call = call i8 @llvm.abs.i8(i8 0, i1 false)
+ %dead.gep = getelementptr i8, ptr null, i64 8
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %cmp = icmp eq i8 %ld.A, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; The oracle only reads memory through its arguments, so @G is passed in.
+define i64 @global_base(i64 %n) {
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i8, ptr @G, i64 %iv
+ %l = load i8, ptr %gep, align 1
+ %c = icmp eq i8 %l, 42
+ br i1 %c, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; TODO: support calls in the oracle.
+define i64 @call_in_exit_condition(ptr %A, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %abs = call i8 @llvm.abs.i8(i8 %ld.A, i1 false)
+ %cmp = icmp sgt i8 %abs, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; TODO: support fcmp in the oracle.
+define i64 @fcmp_in_exit_condition(ptr %A, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds float, ptr %A, i64 %iv
+ %ld.A = load float, ptr %gep.A, align 4
+ %cmp = fcmp uge float %ld.A, 0.000000e+00
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; TODO: support unary operations in the oracle.
+define i64 @fneg_in_exit_condition(ptr %A, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds float, ptr %A, i64 %iv
+ %ld.A = load float, ptr %gep.A, align 4
+ %fneg = fneg float %ld.A
+ %cmp = fcmp ugt float %fneg, 0.000000e+00
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; TODO: support pointer inductions in the oracle.
+define i64 @pointer_induction(ptr %begin, ptr %end) {
+entry:
+ %is.empty = icmp eq ptr %begin, %end
+ br i1 %is.empty, label %exit, label %loop.header
+
+loop.header:
+ %ptr = phi ptr [ %begin, %entry ], [ %ptr.next, %loop.latch ]
+ %ld = load i8, ptr %ptr, align 1
+ %cmp = icmp eq i8 %ld, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %ptr.next = getelementptr inbounds i8, ptr %ptr, i64 1
+ %ec = icmp eq ptr %ptr.next, %end
+ br i1 %ec, label %exit, label %loop.header
+
+early.exit:
+ %iv = ptrtoint ptr %ptr to i64
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @early_exit_runtime_induction_step(ptr %A, i64 %n, i64 %s) {
+entry:
+ %step = mul i64 %s, 4
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %j = phi i64 [ 0, %entry ], [ %j.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %c = icmp eq i32 %l, 42
+ br i1 %c, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %j.next = add nuw nsw i64 %j, %step
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop.header
+
+early.exit:
+ ret i64 %j
+
+exit:
+ ret i64 -1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
index 5be786a0fc91e..a9393d6b6a570 100644
--- a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
@@ -650,4 +650,310 @@ loop.end:
declare i32 @foo(i32) readonly
declare <vscale x 4 x i32> @foo_vec(<vscale x 4 x i32>)
+; Non-dereferenceable load in the latch (past all early exits) used as live-out.
+; Cannot be safely speculated, so the loop should not be vectorized.
+define i64 @load_in_latch_past_all_exits(ptr %src, ptr %p1) {
+; CHECK-LABEL: define i64 @load_in_latch_past_all_exits(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[P1:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
+; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
+; CHECK: loop.header:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP1]], align 1
+; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], 42
+; CHECK-NEXT: br i1 [[CMP1]], label [[LOOP_END:%.*]], label [[LOOP_LATCH]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD_OUT:%.*]] = load i64, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP_HEADER]], label [[LOOP_END]]
+; CHECK: loop.end:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 0, [[LOOP_HEADER]] ], [ [[LD_OUT]], [[LOOP_LATCH]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ call void @init_mem(ptr %p1, i64 1024)
+ br label %loop.header
+
+loop.header:
+ %index = phi i64 [ %index.next, %loop.latch ], [ 0, %entry ]
+ %gep1 = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %gep1, align 1
+ %cmp1 = icmp eq i8 %ld1, 42
+ br i1 %cmp1, label %loop.end, label %loop.latch
+
+loop.latch:
+ %gep.src = getelementptr inbounds i64, ptr %src, i64 %index
+ %ld.out = load i64, ptr %gep.src, align 4
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop.header, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ 0, %loop.header ], [ %ld.out, %loop.latch ]
+ ret i64 %retval
+}
+
+; Two early exits with a non-dereferenceable load in the latch (past all exits).
+; Cannot be safely speculated, so the loop should not be vectorized.
+define i64 @two_exits_load_in_latch_past_all_exits(ptr %src, ptr %p1, ptr %p2) {
+; CHECK-LABEL: define i64 @two_exits_load_in_latch_past_all_exits(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
+; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
+; CHECK: loop.header:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP1]], align 1
+; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], 42
+; CHECK-NEXT: br i1 [[CMP1]], label [[LOOP_END:%.*]], label [[LOOP_MID:%.*]]
+; CHECK: loop.mid:
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP2]], align 1
+; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT: br i1 [[CMP2]], label [[LOOP_END]], label [[LOOP_LATCH]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD_OUT:%.*]] = load i64, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP_HEADER]], label [[LOOP_END]]
+; CHECK: loop.end:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 0, [[LOOP_HEADER]] ], [ 0, [[LOOP_MID]] ], [ [[LD_OUT]], [[LOOP_LATCH]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop.header
+
+loop.header:
+ %index = phi i64 [ %index.next, %loop.latch ], [ 0, %entry ]
+ %gep1 = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %gep1, align 1
+ %cmp1 = icmp eq i8 %ld1, 42
+ br i1 %cmp1, label %loop.end, label %loop.mid
+
+loop.mid:
+ %gep2 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %gep2, align 1
+ %cmp2 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp2, label %loop.end, label %loop.latch
+
+loop.latch:
+ %gep.src = getelementptr inbounds i64, ptr %src, i64 %index
+ %ld.out = load i64, ptr %gep.src, align 4
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop.header, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ 0, %loop.header ], [ 0, %loop.mid ], [ %ld.out, %loop.latch ]
+ ret i64 %retval
+}
+
+; The load type is not a power-of-2 number of bytes, so the oracle's byte count
+; cannot describe per-lane validity and @llvm.speculative.load cannot be used.
+define i64 @early_exit_non_byte_sized_load(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @early_exit_non_byte_sized_load(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i1, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i1, ptr [[GEP]], align 1
+; CHECK-NEXT: br i1 [[L]], label [[EARLY_EXIT:%.*]], label [[LOOP_LATCH]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK: early.exit:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], [[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: exit:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i1, ptr %A, i64 %iv
+ %l = load i1, ptr %gep, align 1
+ br i1 %l, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; The load type is a pointer, which has no primitive size, so the widened
+; <VF x ptr> speculative load would be rejected by the verifier.
+define i64 @early_exit_pointer_typed_load(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @early_exit_pointer_typed_load(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[GEP]], align 8
+; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[L]], null
+; CHECK-NEXT: br i1 [[C]], label [[EARLY_EXIT:%.*]], label [[LOOP_LATCH]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK: early.exit:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], [[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: exit:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds ptr, ptr %A, i64 %iv
+ %l = load ptr, ptr %gep, align 8
+ %c = icmp eq ptr %l, null
+ br i1 %c, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; A division by a constant is safe in the original loop, but the speculative
+; load's trailing lanes are frozen poison and may be zero, so a widened div
+; could divide by zero in a lane the original loop never executed.
+define i64 @early_exit_udiv_of_speculative_load(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @early_exit_udiv_of_speculative_load(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[D:%.*]] = udiv i32 8, [[L]]
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[D]], 42
+; CHECK-NEXT: br i1 [[C]], label [[EARLY_EXIT:%.*]], label [[LOOP_LATCH]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK: early.exit:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], [[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: exit:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %d = udiv i32 8, %l
+ %c = icmp eq i32 %d, 42
+ br i1 %c, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; Multiple uncountable exits with speculative loads are not supported.
+define i64 @two_early_exits_to_different_blocks(ptr %A, ptr %B, i64 %n) {
+; CHECK-LABEL: define i64 @two_early_exits_to_different_blocks(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[CMP_A:%.*]] = icmp eq i8 [[LD_A]], 42
+; CHECK-NEXT: br i1 [[CMP_A]], label [[EARLY_EXIT_A:%.*]], label [[LOOP_SECOND:%.*]]
+; CHECK: loop.second:
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[CMP_B:%.*]] = icmp eq i8 [[LD_B]], 7
+; CHECK-NEXT: br i1 [[CMP_B]], label [[EARLY_EXIT_B:%.*]], label [[LOOP_LATCH]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: early.exit.A:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], [[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: early.exit.B:
+; CHECK-NEXT: [[IV_LCSSA1:%.*]] = phi i64 [ [[IV]], [[LOOP_SECOND]] ]
+; CHECK-NEXT: [[SUB:%.*]] = sub i64 0, [[IV_LCSSA1]]
+; CHECK-NEXT: ret i64 [[SUB]]
+; CHECK: exit:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld.A = load i8, ptr %gep.A, align 1
+ %cmp.A = icmp eq i8 %ld.A, 42
+ br i1 %cmp.A, label %early.exit.A, label %loop.second
+
+loop.second:
+ %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
+ %ld.B = load i8, ptr %gep.B, align 1
+ %cmp.B = icmp eq i8 %ld.B, 7
+ br i1 %cmp.B, label %early.exit.B, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+early.exit.A:
+ ret i64 %iv
+
+early.exit.B:
+ %sub = sub i64 0, %iv
+ ret i64 %sub
+
+exit:
+ ret i64 -1
+}
+
attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }
More information about the llvm-commits
mailing list