[llvm] [NFC][VPlan] Extract BTC-related tests into `vplan-based-stride-mv-btc.ll` (PR #214047)
Andrei Elovikov via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 08:52:56 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/214047
>From d9d490cd3b464e08fd3fa8a7294b03878d08db75 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 4 Aug 2026 12:25:35 -0700
Subject: [PATCH 1/6] [UTC] Support RUN lines with pre-processing for
`update_analyze_test_checks`
---
.../Inputs/pre-process.ll | 9 +++++++++
.../Inputs/pre-process.ll.expected | 14 ++++++++++++++
.../pre-process.test | 6 ++++++
llvm/utils/UpdateTestChecks/common.py | 13 +++++++++++++
llvm/utils/update_analyze_test_checks.py | 18 ++++++++++++------
llvm/utils/update_llc_test_checks.py | 8 +-------
llvm/utils/update_test_checks.py | 14 +-------------
7 files changed, 56 insertions(+), 26 deletions(-)
create mode 100644 llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/pre-process.ll
create mode 100644 llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/pre-process.ll.expected
create mode 100644 llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/pre-process.test
diff --git a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/pre-process.ll b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/pre-process.ll
new file mode 100644
index 0000000000000..a1d67f49f28c5
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/pre-process.ll
@@ -0,0 +1,9 @@
+; RUN: sed 's/foo/entry/' %s | opt -passes='print<branch-prob>' -disable-output 2>&1 | FileCheck %s --check-prefix=CHECK
+
+define void @test1() {
+foo:
+ br label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/pre-process.ll.expected b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/pre-process.ll.expected
new file mode 100644
index 0000000000000..97a8ade217a2c
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/pre-process.ll.expected
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: sed 's/foo/entry/' %s | opt -passes='print<branch-prob>' -disable-output 2>&1 | FileCheck %s --check-prefix=CHECK
+
+define void @test1() {
+; CHECK-LABEL: 'test1'
+; CHECK-NEXT: ---- Branch Probabilities ----
+; CHECK-NEXT: edge %entry -> %exit probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+;
+foo:
+ br label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/pre-process.test b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/pre-process.test
new file mode 100644
index 0000000000000..81870362c64e5
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/pre-process.test
@@ -0,0 +1,6 @@
+## Test that update_analyze_test_checks.py can run pre-processing commands.
+# RUN: cp -f %S/Inputs/pre-process.ll %t.ll && %update_analyze_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/pre-process.ll.expected
+## Check that running the script again does not change the result:
+# RUN: %update_analyze_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/pre-process.ll.expected
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 8eb3249c3f6c2..fdd2e20d2f386 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -507,6 +507,19 @@ def getSubstitutions(sourcepath):
]
+def split_run_line(run_line):
+ """Split a FileCheck RUN line into its tool, FileCheck, and pre-processing commands."""
+ if "%if" in run_line:
+ match = re.search(r"%{\s*(.*?)\s*%}", run_line)
+ if match:
+ run_line = match.group(1)
+
+ commands = [cmd.strip() for cmd in run_line.split("|")]
+ assert len(commands) >= 2
+ preprocess_cmd = " | ".join(commands[:-2]) or None
+ return commands[-2], commands[-1], preprocess_cmd
+
+
def applySubstitutions(s, substitutions):
for a, b in substitutions:
s = s.replace(a, b)
diff --git a/llvm/utils/update_analyze_test_checks.py b/llvm/utils/update_analyze_test_checks.py
index 5329851e465e3..ed4bb358def5d 100755
--- a/llvm/utils/update_analyze_test_checks.py
+++ b/llvm/utils/update_analyze_test_checks.py
@@ -55,11 +55,11 @@ def update_test(opt_basename: str, ti: common.TestInfo):
common.warn("Skipping unparsable RUN line: " + l)
continue
- (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split("|", 1)])
+ tool_cmd, filecheck_cmd, preprocess_cmd = common.split_run_line(l)
common.verify_filecheck_prefixes(filecheck_cmd)
if not tool_cmd.startswith(opt_basename + " "):
- common.warn("WSkipping non-%s RUN line: %s" % (opt_basename, l))
+ common.warn("Skipping non-%s RUN line: %s" % (opt_basename, l))
continue
if not filecheck_cmd.startswith("FileCheck "):
@@ -72,7 +72,7 @@ def update_test(opt_basename: str, ti: common.TestInfo):
# FIXME: We should use multiple check prefixes to common check lines. For
# now, we just ignore all but the last.
- prefix_list.append((check_prefixes, tool_cmd_args))
+ prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))
ginfo = common.make_analyze_generalizer(version=1)
builder = common.FunctionTestBuilder(
@@ -93,11 +93,17 @@ def update_test(opt_basename: str, ti: common.TestInfo):
ginfo=ginfo,
)
- for prefixes, opt_args in prefix_list:
+ for prefixes, opt_args, preprocess_cmd in prefix_list:
common.debug("Extracted opt cmd:", opt_basename, opt_args, file=sys.stderr)
common.debug("Extracted FileCheck prefixes:", str(prefixes), file=sys.stderr)
- raw_tool_outputs = common.invoke_tool(ti.args.opt_binary, opt_args, ti.path)
+ raw_tool_outputs = common.invoke_tool(
+ ti.args.opt_binary,
+ opt_args,
+ ti.path,
+ preprocess_cmd=preprocess_cmd,
+ verbose=ti.args.verbose,
+ )
regex_map = {
r"Printing analysis ": common.ANALYZE_FUNCTION_RE,
@@ -133,7 +139,7 @@ def update_test(opt_basename: str, ti: common.TestInfo):
func_dict = builder.finish_and_get_func_dict()
is_in_function = False
is_in_function_start = False
- prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
+ prefix_set = set([prefix for prefixes, _, _ in prefix_list for prefix in prefixes])
common.debug("Rewriting FileCheck prefixes:", str(prefix_set), file=sys.stderr)
output_lines = []
diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py
index 98864be62875b..fc356d3e9f77b 100755
--- a/llvm/utils/update_llc_test_checks.py
+++ b/llvm/utils/update_llc_test_checks.py
@@ -39,13 +39,7 @@ def update_test(ti: common.TestInfo):
common.warn("Skipping unparsable RUN line: " + l)
continue
- commands = [cmd.strip() for cmd in l.split("|")]
- assert len(commands) >= 2
- preprocess_cmd = None
- if len(commands) > 2:
- preprocess_cmd = " | ".join(commands[:-2])
- llc_cmd = commands[-2]
- filecheck_cmd = commands[-1]
+ llc_cmd, filecheck_cmd, preprocess_cmd = common.split_run_line(l)
llc_tool = llc_cmd.split(" ")[0]
triple_in_cmd = None
diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py
index 74e87787fd5b8..5d8d7a36f5273 100755
--- a/llvm/utils/update_test_checks.py
+++ b/llvm/utils/update_test_checks.py
@@ -61,19 +61,7 @@ def update_test(ti: common.TestInfo):
common.warn("Skipping unparsable RUN line: " + l)
continue
- cropped_content = l
- if "%if" in l:
- match = re.search(r"%{\s*(.*?)\s*%}", l)
- if match:
- cropped_content = match.group(1)
-
- commands = [cmd.strip() for cmd in cropped_content.split("|")]
- assert len(commands) >= 2
- preprocess_cmd = None
- if len(commands) > 2:
- preprocess_cmd = " | ".join(commands[:-2])
- tool_cmd = commands[-2]
- filecheck_cmd = commands[-1]
+ tool_cmd, filecheck_cmd, preprocess_cmd = common.split_run_line(l)
common.verify_filecheck_prefixes(filecheck_cmd)
if not tool_cmd.startswith(tool_basename + " "):
common.warn("Skipping non-%s RUN line: %s" % (tool_basename, l))
>From c529663e33c37821622997ce2e4f10e3653a1401 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 4 Aug 2026 12:30:27 -0700
Subject: [PATCH 2/6] [NFC][VPlan] Add scalable vectors RUN-line to
vplan-based-stride-mv.ll
---
.../VPlan/vplan-based-stride-mv.ll | 284 ++++++++++++++++++
1 file changed, 284 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index ea44f400bf4a3..45f9704d03821 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -3,6 +3,12 @@
; RUN: -vplan-print-after=scalarizeMemOpsWithIrregularTypes \
; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s
+; Only interested in interactions with trip counts for the scalable
+; vectorization, the rest has shared code path with fixed-width vectors.
+; RUN: llvm-extract -S -rfunc=btc < %s | opt -p loop-vectorize -force-vector-width="vscale x 4" -force-target-supports-scalable-vectors -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops -disable-output \
+; RUN: -vplan-print-after=scalarizeMemOpsWithIrregularTypes \
+; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix SCALABLE
+
define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-LABEL: VPlan for loop in 'basic'
; CHECK: VPlan ' for UF>=1' {
@@ -1520,6 +1526,51 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
+; SCALABLE-LABEL: VPlan for loop in 'non_constant_btc'
+; SCALABLE: VPlan ' for UF>=1' {
+; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<entry>:
+; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %n)
+; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.ph:
+; SCALABLE-NEXT: Successor(s): vector loop
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: <x1> vector loop: {
+; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.body:
+; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; SCALABLE-NEXT: No successors
+; SCALABLE-NEXT: }
+; SCALABLE-NEXT: Successor(s): middle.block
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: middle.block:
+; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<exit>:
+; SCALABLE-NEXT: No successors
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: scalar.ph:
+; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; SCALABLE-NEXT: Successor(s): ir-bb<header>
+;
entry:
br label %header
@@ -1590,6 +1641,51 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
+; SCALABLE-LABEL: VPlan for loop in 'stride_as_btc'
+; SCALABLE: VPlan ' for UF>=1' {
+; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<entry>:
+; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %stride)
+; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.ph:
+; SCALABLE-NEXT: Successor(s): vector loop
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: <x1> vector loop: {
+; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.body:
+; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%stride>
+; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; SCALABLE-NEXT: No successors
+; SCALABLE-NEXT: }
+; SCALABLE-NEXT: Successor(s): middle.block
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: middle.block:
+; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<exit>:
+; SCALABLE-NEXT: No successors
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: scalar.ph:
+; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; SCALABLE-NEXT: Successor(s): ir-bb<header>
+;
entry:
br label %header
@@ -1661,6 +1757,52 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
+; SCALABLE-LABEL: VPlan for loop in 'stride_dependent_btc'
+; SCALABLE: VPlan ' for UF>=1' {
+; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<entry>:
+; SCALABLE-NEXT: IR %n = add i64 %stride, 1
+; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (1 + %stride))
+; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.ph:
+; SCALABLE-NEXT: Successor(s): vector loop
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: <x1> vector loop: {
+; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.body:
+; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; SCALABLE-NEXT: No successors
+; SCALABLE-NEXT: }
+; SCALABLE-NEXT: Successor(s): middle.block
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: middle.block:
+; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<exit>:
+; SCALABLE-NEXT: No successors
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: scalar.ph:
+; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; SCALABLE-NEXT: Successor(s): ir-bb<header>
+;
entry:
%n = add i64 %stride, 1
br label %header
@@ -1734,6 +1876,52 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
+; SCALABLE-LABEL: VPlan for loop in 'stride_btc_checks_order'
+; SCALABLE: VPlan ' for UF>=1' {
+; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<entry>:
+; SCALABLE-NEXT: IR %n = mul i64 %m, %stride
+; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (%stride * %m))
+; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.ph:
+; SCALABLE-NEXT: Successor(s): vector loop
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: <x1> vector loop: {
+; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.body:
+; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; SCALABLE-NEXT: No successors
+; SCALABLE-NEXT: }
+; SCALABLE-NEXT: Successor(s): middle.block
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: middle.block:
+; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<exit>:
+; SCALABLE-NEXT: No successors
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: scalar.ph:
+; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; SCALABLE-NEXT: Successor(s): ir-bb<header>
+;
entry:
%n = mul i64 %m, %stride
br label %header
@@ -1805,6 +1993,52 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
+; SCALABLE-LABEL: VPlan for loop in 'stride_dependent_btc_non_preventive'
+; SCALABLE: VPlan ' for UF>=1' {
+; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<entry>:
+; SCALABLE-NEXT: IR %n = add i64 %stride, 3
+; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
+; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.ph:
+; SCALABLE-NEXT: Successor(s): vector loop
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: <x1> vector loop: {
+; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.body:
+; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; SCALABLE-NEXT: No successors
+; SCALABLE-NEXT: }
+; SCALABLE-NEXT: Successor(s): middle.block
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: middle.block:
+; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<exit>:
+; SCALABLE-NEXT: No successors
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: scalar.ph:
+; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; SCALABLE-NEXT: Successor(s): ir-bb<header>
+;
entry:
%n = add i64 %stride, 3
br label %header
@@ -1912,6 +2146,56 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
+; SCALABLE-LABEL: VPlan for loop in 'stride_btc_independent_memdep_triple_check'
+; SCALABLE: VPlan ' for UF>=1' {
+; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<entry>:
+; SCALABLE-NEXT: IR %p.out = getelementptr i8, ptr %p2, i64 %out.offset
+; SCALABLE-NEXT: IR %n = add i64 %stride, 3
+; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
+; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.ph:
+; SCALABLE-NEXT: Successor(s): vector loop
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: <x1> vector loop: {
+; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: vector.body:
+; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; SCALABLE-NEXT: EMIT ir<%gep.ld2> = getelementptr ir<%p2>, ir<%iv>
+; SCALABLE-NEXT: EMIT-SCALAR ir<%ld2> = load ir<%gep.ld2>
+; SCALABLE-NEXT: EMIT ir<%val> = add ir<%ld>, ir<%ld2>
+; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; SCALABLE-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; SCALABLE-NEXT: No successors
+; SCALABLE-NEXT: }
+; SCALABLE-NEXT: Successor(s): middle.block
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: middle.block:
+; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: ir-bb<exit>:
+; SCALABLE-NEXT: No successors
+; SCALABLE-EMPTY:
+; SCALABLE-NEXT: scalar.ph:
+; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; SCALABLE-NEXT: Successor(s): ir-bb<header>
+;
entry:
%p.out = getelementptr i8, ptr %p2, i64 %out.offset
%n = add i64 %stride, 3
>From 19c5732de4cbda28a9ef95a17b5ad3d1a92df3b9 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 6 Aug 2026 10:12:42 -0700
Subject: [PATCH 3/6] Revert all changes
---
.../VPlan/vplan-based-stride-mv.ll | 284 ------------------
1 file changed, 284 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index 45f9704d03821..ea44f400bf4a3 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -3,12 +3,6 @@
; RUN: -vplan-print-after=scalarizeMemOpsWithIrregularTypes \
; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s
-; Only interested in interactions with trip counts for the scalable
-; vectorization, the rest has shared code path with fixed-width vectors.
-; RUN: llvm-extract -S -rfunc=btc < %s | opt -p loop-vectorize -force-vector-width="vscale x 4" -force-target-supports-scalable-vectors -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops -disable-output \
-; RUN: -vplan-print-after=scalarizeMemOpsWithIrregularTypes \
-; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix SCALABLE
-
define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-LABEL: VPlan for loop in 'basic'
; CHECK: VPlan ' for UF>=1' {
@@ -1526,51 +1520,6 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
-; SCALABLE-LABEL: VPlan for loop in 'non_constant_btc'
-; SCALABLE: VPlan ' for UF>=1' {
-; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<entry>:
-; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %n)
-; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.ph:
-; SCALABLE-NEXT: Successor(s): vector loop
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: <x1> vector loop: {
-; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.body:
-; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
-; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
-; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; SCALABLE-NEXT: No successors
-; SCALABLE-NEXT: }
-; SCALABLE-NEXT: Successor(s): middle.block
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: middle.block:
-; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
-; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
-; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<exit>:
-; SCALABLE-NEXT: No successors
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: scalar.ph:
-; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; SCALABLE-NEXT: Successor(s): ir-bb<header>
-;
entry:
br label %header
@@ -1641,51 +1590,6 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
-; SCALABLE-LABEL: VPlan for loop in 'stride_as_btc'
-; SCALABLE: VPlan ' for UF>=1' {
-; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<entry>:
-; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %stride)
-; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.ph:
-; SCALABLE-NEXT: Successor(s): vector loop
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: <x1> vector loop: {
-; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.body:
-; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
-; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%stride>
-; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
-; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; SCALABLE-NEXT: No successors
-; SCALABLE-NEXT: }
-; SCALABLE-NEXT: Successor(s): middle.block
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: middle.block:
-; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
-; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
-; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<exit>:
-; SCALABLE-NEXT: No successors
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: scalar.ph:
-; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; SCALABLE-NEXT: Successor(s): ir-bb<header>
-;
entry:
br label %header
@@ -1757,52 +1661,6 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
-; SCALABLE-LABEL: VPlan for loop in 'stride_dependent_btc'
-; SCALABLE: VPlan ' for UF>=1' {
-; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<entry>:
-; SCALABLE-NEXT: IR %n = add i64 %stride, 1
-; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (1 + %stride))
-; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.ph:
-; SCALABLE-NEXT: Successor(s): vector loop
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: <x1> vector loop: {
-; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.body:
-; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
-; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
-; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; SCALABLE-NEXT: No successors
-; SCALABLE-NEXT: }
-; SCALABLE-NEXT: Successor(s): middle.block
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: middle.block:
-; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
-; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
-; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<exit>:
-; SCALABLE-NEXT: No successors
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: scalar.ph:
-; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; SCALABLE-NEXT: Successor(s): ir-bb<header>
-;
entry:
%n = add i64 %stride, 1
br label %header
@@ -1876,52 +1734,6 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
-; SCALABLE-LABEL: VPlan for loop in 'stride_btc_checks_order'
-; SCALABLE: VPlan ' for UF>=1' {
-; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<entry>:
-; SCALABLE-NEXT: IR %n = mul i64 %m, %stride
-; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (%stride * %m))
-; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.ph:
-; SCALABLE-NEXT: Successor(s): vector loop
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: <x1> vector loop: {
-; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.body:
-; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
-; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
-; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; SCALABLE-NEXT: No successors
-; SCALABLE-NEXT: }
-; SCALABLE-NEXT: Successor(s): middle.block
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: middle.block:
-; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
-; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
-; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<exit>:
-; SCALABLE-NEXT: No successors
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: scalar.ph:
-; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; SCALABLE-NEXT: Successor(s): ir-bb<header>
-;
entry:
%n = mul i64 %m, %stride
br label %header
@@ -1993,52 +1805,6 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
-; SCALABLE-LABEL: VPlan for loop in 'stride_dependent_btc_non_preventive'
-; SCALABLE: VPlan ' for UF>=1' {
-; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<entry>:
-; SCALABLE-NEXT: IR %n = add i64 %stride, 3
-; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
-; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.ph:
-; SCALABLE-NEXT: Successor(s): vector loop
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: <x1> vector loop: {
-; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.body:
-; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; SCALABLE-NEXT: EMIT store ir<%ld>, ir<%gep.st>
-; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
-; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; SCALABLE-NEXT: No successors
-; SCALABLE-NEXT: }
-; SCALABLE-NEXT: Successor(s): middle.block
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: middle.block:
-; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
-; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
-; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<exit>:
-; SCALABLE-NEXT: No successors
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: scalar.ph:
-; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; SCALABLE-NEXT: Successor(s): ir-bb<header>
-;
entry:
%n = add i64 %stride, 3
br label %header
@@ -2146,56 +1912,6 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
;
-; SCALABLE-LABEL: VPlan for loop in 'stride_btc_independent_memdep_triple_check'
-; SCALABLE: VPlan ' for UF>=1' {
-; SCALABLE-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; SCALABLE-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; SCALABLE-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; SCALABLE-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<entry>:
-; SCALABLE-NEXT: IR %p.out = getelementptr i8, ptr %p2, i64 %out.offset
-; SCALABLE-NEXT: IR %n = add i64 %stride, 3
-; SCALABLE-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
-; SCALABLE-NEXT: Successor(s): scalar.ph, vector.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.ph:
-; SCALABLE-NEXT: Successor(s): vector loop
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: <x1> vector loop: {
-; SCALABLE-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: vector.body:
-; SCALABLE-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; SCALABLE-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; SCALABLE-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; SCALABLE-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; SCALABLE-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; SCALABLE-NEXT: EMIT ir<%gep.ld2> = getelementptr ir<%p2>, ir<%iv>
-; SCALABLE-NEXT: EMIT-SCALAR ir<%ld2> = load ir<%gep.ld2>
-; SCALABLE-NEXT: EMIT ir<%val> = add ir<%ld>, ir<%ld2>
-; SCALABLE-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; SCALABLE-NEXT: EMIT store ir<%val>, ir<%gep.st>
-; SCALABLE-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; SCALABLE-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
-; SCALABLE-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; SCALABLE-NEXT: No successors
-; SCALABLE-NEXT: }
-; SCALABLE-NEXT: Successor(s): middle.block
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: middle.block:
-; SCALABLE-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; SCALABLE-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
-; SCALABLE-NEXT: EMIT branch-on-cond vp<%cmp.n>
-; SCALABLE-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: ir-bb<exit>:
-; SCALABLE-NEXT: No successors
-; SCALABLE-EMPTY:
-; SCALABLE-NEXT: scalar.ph:
-; SCALABLE-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; SCALABLE-NEXT: Successor(s): ir-bb<header>
-;
entry:
%p.out = getelementptr i8, ptr %p2, i64 %out.offset
%n = add i64 %stride, 3
>From d16d1a67444a12fc3df80cc65ffd196c6de20614 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 6 Aug 2026 10:23:01 -0700
Subject: [PATCH 4/6] [NFC][VPlan] Extract BTC-related test into
`vplan-based-stride-mv-btc.ll`
...and add RUN lines with scalable vectors there.
---
.../vplan-based-stride-mv-btc.ll | 1488 +++++++++++++++++
.../LoopVectorize/vplan-based-stride-mv.ll | 802 +--------
2 files changed, 1490 insertions(+), 800 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
new file mode 100644
index 0000000000000..80b3fc3f0ca30
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
@@ -0,0 +1,1488 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+
+; This file complements vplan-based-stride-mv.ll by containing the test
+; functions from VPlan/vplan-based-stride-mv.ll that focus on interactions
+; between stride speculation and backedge taken count as we want to look/verify
+; both fixed and scalable vector width vectorization.
+
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
+; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV
+
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
+; RUN: -enable-mem-access-versioning=true 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV
+
+; RUN: opt < %s -p loop-vectorize -force-vector-width="vscale x 4" -S \
+; RUN: -force-target-supports-scalable-vectors -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops \
+; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV-SCALABLE
+
+; RUN: opt < %s -p loop-vectorize -force-vector-width="vscale x 4" -S \
+; RUN: -force-target-supports-scalable-vectors -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops \
+; RUN: -enable-mem-access-versioning=true 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV-SCALABLE
+
+; Not valuable by itself, but as a basis for the subsequent test to ensure that
+; non-constant trip count doesn't change anything by itself. Also show the order
+; of checks between stride speculation and trip-count check when not
+; tail-folding.
+define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
+; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+; COMPARE-NO-MV-SCALABLE-LABEL: define void @non_constant_btc(
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: ret void
+;
+; COMPARE-LAA-MV-SCALABLE-LABEL: define void @non_constant_btc(
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC == stride, so stride speculation would result in zero vector loop
+; iterations.
+define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_as_btc(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_as_btc(
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: ret void
+;
+; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_as_btc(
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1, [[TMP1]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1, [[TMP2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1, [[N_MOD_VF]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 1, [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %stride
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Similar to above but a slightly more complex dependency between stride and
+; BTC.
+define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_dependent_btc(
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: ret void
+;
+; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_dependent_btc(
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 2, [[TMP1]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 2, [[TMP2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 2, [[N_MOD_VF]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 2, [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
+;
+entry:
+ %n = add i64 %stride, 1
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC dependent on %stride, but stride speculation doesn't necessarily mean no
+; vector loop iterations. The test shows in which order we emit the run time
+; checks for both.
+define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i64 %m) {
+; COMPARE-NO-MV-LABEL: define void @stride_btc_checks_order(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_btc_checks_order(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX1]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX1]], 4
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX1]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_checks_order(
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: ret void
+;
+; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_checks_order(
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
+;
+entry:
+ %n = mul i64 %m, %stride
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC fully defined by stride speculation but still allows vector loop execution.
+define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc_non_preventive(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc_non_preventive(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_dependent_btc_non_preventive(
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: ret void
+;
+; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_dependent_btc_non_preventive(
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4, [[TMP1]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 4, [[TMP2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 4, [[N_MOD_VF]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 4, [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
+;
+entry:
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Doesn't pass legality as run-time memory dependencies check doesn't support
+; strided accesses. If it did, the purpose of this test would be to show how all
+; three run-time checks (mem deps/stride speculation/trip-count) would be
+; ordered in respect to each other. Note that pointer aliasing check could
+; potentially be simplified if done after stride speculation. However, that
+; isn't necessarily the best idea because we could also multi-version for stride
+; and keep aliasing part generic and shared by both vector loops..
+define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset) {
+; COMPARE-NO-MV-LABEL: define void @stride_btc_memdep_triple_check(
+; COMPARE-NO-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_btc_memdep_triple_check(
+; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
+; COMPARE-LAA-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_memdep_triple_check(
+; COMPARE-NO-MV-SCALABLE-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: ret void
+;
+; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_memdep_triple_check(
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4, [[TMP1]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_MEMCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 5
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = add i64 [[TMP3]], -1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP6:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 4, [[TMP7]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 4, [[N_MOD_VF]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 4, [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
+;
+entry:
+ %p.out = getelementptr i8, ptr %p, i64 %out.offset
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Same as above but memdep check doesn't depend on stride
+define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
+; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
+; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[TMP16]], [[WIDE_LOAD]]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
+; COMPARE-LAA-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[P2]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP0]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-NO-MV-SCALABLE-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_MEMCHECK]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 5
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = add i64 [[TMP4]], -1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP6]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP8]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP10:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP10]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP11]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[TMP12]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: ret void
+;
+; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4, [[TMP1]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_MEMCHECK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 5
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = add i64 [[TMP3]], -1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP6:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP0]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 4, [[TMP7]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 4, [[N_MOD_VF]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i64>, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[TMP10]], ptr [[TMP11]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 4, [[N_VEC]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
+;
+entry:
+ %p.out = getelementptr i8, ptr %p2, i64 %out.offset
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.ld2 = getelementptr i64, ptr %p2, i64 %iv
+ %ld2 = load i64, ptr %gep.ld2, align 8
+
+ %val = add i64 %ld, %ld2
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index 4429b0e731616..19dad0269fcf3 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -2357,806 +2357,8 @@ exit:
ret void
}
-; Not valuable by itself, but as a basis for the subsequent test to ensure that
-; non-constant trip count doesn't change anything by itself. Also show the order
-; of checks between stride speculation and trip-count check when not
-; tail-folding.
-define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
-; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP23:![0-9]+]]
-; COMPARE-NO-MV: [[EXIT]]:
-; COMPARE-NO-MV-NEXT: ret void
-;
-; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
-; COMPARE-LAA-MV-NEXT: ret void
-;
-entry:
- br label %header
-
-header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
-
- %idx = mul i64 %iv, %stride
-
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
-
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %ld, ptr %gep.st, align 8
-
- %exitcond = icmp slt i64 %iv.next, %n
- br i1 %exitcond, label %header, label %exit
-
-exit:
- ret void
-}
-
-; BTC == stride, so stride speculation would result in zero vector loop
-; iterations.
-define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
-; COMPARE-NO-MV-LABEL: define void @stride_as_btc(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
-; COMPARE-NO-MV: [[EXIT]]:
-; COMPARE-NO-MV-NEXT: ret void
-;
-; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[EXIT]]:
-; COMPARE-LAA-MV-NEXT: ret void
-;
-entry:
- br label %header
-
-header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
-
- %idx = mul i64 %iv, %stride
-
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
-
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %ld, ptr %gep.st, align 8
-
- %exitcond = icmp slt i64 %iv.next, %stride
- br i1 %exitcond, label %header, label %exit
-
-exit:
- ret void
-}
-
-; Similar to above but a slightly more complex dependency between stride and
-; BTC.
-define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
-; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
-; COMPARE-NO-MV: [[EXIT]]:
-; COMPARE-NO-MV-NEXT: ret void
-;
-; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[EXIT]]:
-; COMPARE-LAA-MV-NEXT: ret void
-;
-entry:
- %n = add i64 %stride, 1
- br label %header
-
-header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
-
- %idx = mul i64 %iv, %stride
-
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
-
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %ld, ptr %gep.st, align 8
-
- %exitcond = icmp slt i64 %iv.next, %n
- br i1 %exitcond, label %header, label %exit
-
-exit:
- ret void
-}
-
-; BTC dependent on %stride, but stride speculation doesn't necessarily mean no
-; vector loop iterations. The test shows in which order we emit the run time
-; checks for both.
-define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i64 %m) {
-; COMPARE-NO-MV-LABEL: define void @stride_btc_checks_order(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
-; COMPARE-NO-MV: [[EXIT]]:
-; COMPARE-NO-MV-NEXT: ret void
-;
-; COMPARE-LAA-MV-LABEL: define void @stride_btc_checks_order(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
-; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX1]], 4
-; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX1]], 4
-; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX1]], [[N_MOD_VF]]
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
-; COMPARE-LAA-MV-NEXT: ret void
-;
-entry:
- %n = mul i64 %m, %stride
- br label %header
-
-header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
-
- %idx = mul i64 %iv, %stride
-
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
-
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %ld, ptr %gep.st, align 8
-
- %exitcond = icmp slt i64 %iv.next, %n
- br i1 %exitcond, label %header, label %exit
-
-exit:
- ret void
-}
-
-; BTC fully defined by stride speculation but still allows vector loop execution.
-define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) {
-; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc_non_preventive(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
-; COMPARE-NO-MV: [[EXIT]]:
-; COMPARE-NO-MV-NEXT: ret void
-;
-; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc_non_preventive(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
-; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
-; COMPARE-LAA-MV-NEXT: ret void
-;
-entry:
- %n = add i64 %stride, 3
- br label %header
-
-header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
-
- %idx = mul i64 %iv, %stride
-
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
-
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %ld, ptr %gep.st, align 8
-
- %exitcond = icmp slt i64 %iv.next, %n
- br i1 %exitcond, label %header, label %exit
-
-exit:
- ret void
-}
-
-; Doesn't pass legality as run-time memory dependencies check doesn't support
-; strided accesses. If it did, the purpose of this test would be to show how all
-; three run-time checks (mem deps/stride speculation/trip-count) would be
-; ordered in respect to each other. Note that pointer aliasing check could
-; potentially be simplified if done after stride speculation. However, that
-; isn't necessarily the best idea because we could also multi-version for stride
-; and keep aliasing part generic and shared by both vector loops..
-define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset) {
-; COMPARE-NO-MV-LABEL: define void @stride_btc_memdep_triple_check(
-; COMPARE-NO-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
-; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
-; COMPARE-NO-MV-NEXT: ret void
-;
-; COMPARE-LAA-MV-LABEL: define void @stride_btc_memdep_triple_check(
-; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
-; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-LAA-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
-; COMPARE-LAA-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
-; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
-; COMPARE-LAA-MV-NEXT: ret void
-;
-entry:
- %p.out = getelementptr i8, ptr %p, i64 %out.offset
- %n = add i64 %stride, 3
- br label %header
-
-header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
-
- %idx = mul i64 %iv, %stride
-
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
-
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %ld, ptr %gep.st, align 8
-
- %exitcond = icmp slt i64 %iv.next, %n
- br i1 %exitcond, label %header, label %exit
-
-exit:
- ret void
-}
-
-; Same as above but memdep check doesn't depend on stride
-define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
-; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
-; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
-; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
-; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[TMP16]], [[WIDE_LOAD]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-NO-MV-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
-; COMPARE-NO-MV-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
-; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
-; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
-; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP33:![0-9]+]]
-; COMPARE-NO-MV: [[EXIT]]:
-; COMPARE-NO-MV-NEXT: ret void
-;
-; COMPARE-LAA-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
-; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
-; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-LAA-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
-; COMPARE-LAA-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[P2]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP0]], ptr [[P_OUT]], align 8
-; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
-; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
-; COMPARE-LAA-MV-NEXT: ret void
-;
-entry:
- %p.out = getelementptr i8, ptr %p2, i64 %out.offset
- %n = add i64 %stride, 3
- br label %header
-
-header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
-
- %idx = mul i64 %iv, %stride
-
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
-
- %gep.ld2 = getelementptr i64, ptr %p2, i64 %iv
- %ld2 = load i64, ptr %gep.ld2, align 8
-
- %val = add i64 %ld, %ld2
-
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %val, ptr %gep.st, align 8
-
- %exitcond = icmp slt i64 %iv.next, %n
- br i1 %exitcond, label %header, label %exit
-
-exit:
- ret void
-}
+;; BTC-related functions from VPlan/vplan-based-stride-mv.ll are in
+;; vplan-based-stride-mv-btc with extra RUN lines for scalable vectors.
define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @actual_stride_not_in_ir(
>From e16ea17bc28a8e32177fe55e26ec4aa5257a4573 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 7 Aug 2026 08:38:16 -0700
Subject: [PATCH 5/6] Add `vscale_range` and force tail-folded RUN lines
---
.../vplan-based-stride-mv-btc.ll | 1314 ++++++++++++++---
1 file changed, 1104 insertions(+), 210 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
index 80b3fc3f0ca30..7328b3c1c9852 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
@@ -11,6 +11,14 @@
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
; RUN: -enable-mem-access-versioning=true 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
+; RUN: -tail-predication=enabled -tail-folding-policy=must-fold-tail \
+; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV-FOLD-TAIL
+
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
+; RUN: -tail-predication=enabled -tail-folding-policy=must-fold-tail \
+; RUN: -enable-mem-access-versioning=true 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV-FOLD_TAIL
+
; RUN: opt < %s -p loop-vectorize -force-vector-width="vscale x 4" -S \
; RUN: -force-target-supports-scalable-vectors -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops \
; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV-SCALABLE
@@ -23,9 +31,9 @@
; non-constant trip count doesn't change anything by itself. Also show the order
; of checks between stride speculation and trip-count check when not
; tail-folding.
-define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
+define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
@@ -82,7 +90,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
@@ -122,12 +130,164 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
+; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @non_constant_btc(
+; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP6]], ptr [[TMP7]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP11]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP15]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP18]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP17]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF7]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP21]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP22]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP25:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP24]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP23]], ptr [[TMP25]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE8]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
+;
+; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @non_constant_btc(
+; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP4]], ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP7]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP9]], ptr [[TMP10]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP12]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP14]], ptr [[TMP15]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP19]], ptr [[TMP20]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
+;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @non_constant_btc(
-; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
@@ -172,11 +332,11 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @non_constant_btc(
-; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
@@ -239,9 +399,9 @@ exit:
; BTC == stride, so stride speculation would result in zero vector loop
; iterations.
-define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
+define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_as_btc(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
@@ -298,7 +458,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
@@ -314,12 +474,143 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
+; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_as_btc(
+; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP6]], ptr [[TMP7]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP11]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP15]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP18]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP17]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF7]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP21]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP22]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP25:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP24]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP23]], ptr [[TMP25]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE8]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
+;
+; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_as_btc(
+; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = load i64, ptr [[P]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP0]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 false, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP2]], ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 false, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP5]], ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 false, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP8]], ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
+;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_as_btc(
-; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
@@ -364,38 +655,11 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_as_btc(
-; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1, [[TMP1]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1, [[TMP2]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1, [[N_MOD_VF]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP3]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP4]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 1, [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[SCALAR_PH:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -403,7 +667,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
@@ -431,9 +695,9 @@ exit:
; Similar to above but a slightly more complex dependency between stride and
; BTC.
-define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
+define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
@@ -491,7 +755,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
@@ -508,13 +772,146 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
+; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_dependent_btc(
+; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP6]], ptr [[TMP7]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP11]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP15]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP18]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP17]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF7]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP21]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP22]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP25:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP24]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP23]], ptr [[TMP25]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE8]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
+;
+; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_dependent_btc(
+; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = load i64, ptr [[P]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP0]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP2]], ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 false, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP5]], ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 false, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP8]], ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
+;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_dependent_btc(
-; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
@@ -559,39 +956,12 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_dependent_btc(
-; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 2, [[TMP1]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 2, [[TMP2]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 2, [[N_MOD_VF]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP3]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP4]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 2, [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -599,7 +969,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
@@ -629,9 +999,9 @@ exit:
; BTC dependent on %stride, but stride speculation doesn't necessarily mean no
; vector loop iterations. The test shows in which order we emit the run time
; checks for both.
-define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i64 %m) {
+define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i64 %m) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_btc_checks_order(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
@@ -689,7 +1059,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_btc_checks_order(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
@@ -731,13 +1101,168 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
+; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_btc_checks_order(
+; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP6]], ptr [[TMP7]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP11]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP15]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP18]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP17]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF7]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP21]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP22]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP25:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP24]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP23]], ptr [[TMP25]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE8]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
+;
+; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_btc_checks_order(
+; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP4]], ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP7]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP9]], ptr [[TMP10]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP12]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP14]], ptr [[TMP15]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP19]], ptr [[TMP20]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
+;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_checks_order(
-; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
@@ -782,13 +1307,13 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_checks_order(
-; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
@@ -807,7 +1332,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -823,7 +1348,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
@@ -851,9 +1376,9 @@ exit:
}
; BTC fully defined by stride speculation but still allows vector loop execution.
-define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) {
+define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc_non_preventive(
-; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
@@ -911,7 +1436,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc_non_preventive(
-; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
@@ -942,13 +1467,146 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
+; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_dependent_btc_non_preventive(
+; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP6]], ptr [[TMP7]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP11]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP15]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP18]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP17]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF7]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP21]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP22]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP25:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP24]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP23]], ptr [[TMP25]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE8]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
+;
+; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_dependent_btc_non_preventive(
+; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = load i64, ptr [[P]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP0]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP2]], ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP5]], ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP8]], ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
+;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_dependent_btc_non_preventive(
-; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
@@ -993,39 +1651,12 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_dependent_btc_non_preventive(
-; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4, [[TMP1]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 4, [[TMP2]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 4, [[N_MOD_VF]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP3]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP4]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 4, [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1033,7 +1664,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
@@ -1067,9 +1698,9 @@ exit:
; potentially be simplified if done after stride speculation. However, that
; isn't necessarily the best idea because we could also multi-version for stride
; and keep aliasing part generic and shared by both vector loops..
-define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset) {
+define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_btc_memdep_triple_check(
-; COMPARE-NO-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
@@ -1088,7 +1719,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_btc_memdep_triple_check(
-; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
@@ -1125,8 +1756,92 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
+; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_btc_memdep_triple_check(
+; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[HEADER]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
+;
+; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_btc_memdep_triple_check(
+; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_MEMCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = load i64, ptr [[P]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP1]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP3]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP6]], ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF5]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP9]], ptr [[TMP10]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
+;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_memdep_triple_check(
-; COMPARE-NO-MV-SCALABLE-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
@@ -1145,47 +1860,13 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_memdep_triple_check(
-; COMPARE-LAA-MV-SCALABLE-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4, [[TMP1]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_MEMCHECK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 5
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = add i64 [[TMP3]], -1
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP6:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 4, [[TMP7]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 4, [[N_MOD_VF]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP9]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 4, [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1193,7 +1874,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
@@ -1222,9 +1903,9 @@ exit:
}
; Same as above but memdep check doesn't depend on stride
-define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
+define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
-; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
@@ -1293,7 +1974,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
-; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
@@ -1335,20 +2016,270 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
+; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_MEMCHECK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE14:.*]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE14]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP7]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_LOAD_IF]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP8]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_LOAD_CONTINUE]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP15:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP11]], %[[PRED_LOAD_IF]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP16:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP14]], %[[PRED_LOAD_IF]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_LOAD_IF3]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP18]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP19]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP21:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP20]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[P2]], i64 [[TMP3]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP22]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP23]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_LOAD_CONTINUE4]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP25:%.*]] = phi <4 x i64> [ [[TMP15]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP21]], %[[PRED_LOAD_IF3]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = phi <4 x i64> [ [[TMP16]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP24]], %[[PRED_LOAD_IF3]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP27]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_LOAD_IF5]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP28:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP29:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP28]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP29]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP30]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP32:%.*]] = getelementptr i64, ptr [[P2]], i64 [[TMP4]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP33:%.*]] = load i64, ptr [[TMP32]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP34:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP33]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_LOAD_CONTINUE6]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP35:%.*]] = phi <4 x i64> [ [[TMP25]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP31]], %[[PRED_LOAD_IF5]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP36:%.*]] = phi <4 x i64> [ [[TMP26]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP34]], %[[PRED_LOAD_IF5]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP37]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_LOAD_IF7]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP38:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP39:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP38]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP40:%.*]] = load i64, ptr [[TMP39]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP41:%.*]] = insertelement <4 x i64> [[TMP35]], i64 [[TMP40]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP42:%.*]] = getelementptr i64, ptr [[P2]], i64 [[TMP5]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP43:%.*]] = load i64, ptr [[TMP42]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP44:%.*]] = insertelement <4 x i64> [[TMP36]], i64 [[TMP43]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_LOAD_CONTINUE8]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP45:%.*]] = phi <4 x i64> [ [[TMP35]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP41]], %[[PRED_LOAD_IF7]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP46:%.*]] = phi <4 x i64> [ [[TMP36]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP44]], %[[PRED_LOAD_IF7]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP47:%.*]] = add <4 x i64> [[TMP45]], [[TMP46]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP48:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP49:%.*]] = extractelement <4 x i64> [[TMP47]], i64 0
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP49]], ptr [[TMP48]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF9]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP50:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP3]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP51:%.*]] = extractelement <4 x i64> [[TMP47]], i64 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP51]], ptr [[TMP50]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE10]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF11]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP52:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP53:%.*]] = extractelement <4 x i64> [[TMP47]], i64 2
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP53]], ptr [[TMP52]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE12]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP37]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_IF13]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP54:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP55:%.*]] = extractelement <4 x i64> [[TMP47]], i64 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[TMP55]], ptr [[TMP54]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[PRED_STORE_CONTINUE14]]
+; COMPARE-NO-MV-FOLD-TAIL: [[PRED_STORE_CONTINUE14]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP56:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP56]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[HEADER]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
+;
+; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_MEMCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_IF]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = load i64, ptr [[P]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = load i64, ptr [[P2]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[PRED_LOAD_IF]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_LOAD_IF]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP9:%.*]] = insertelement <4 x i64> [[TMP5]], i64 [[TMP8]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P2]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> [[TMP6]], i64 [[TMP11]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP13:%.*]] = phi <4 x i64> [ [[TMP5]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP9]], %[[PRED_LOAD_IF1]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP14:%.*]] = phi <4 x i64> [ [[TMP6]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP15]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP16]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P2]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP19]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP21:%.*]] = phi <4 x i64> [ [[TMP13]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP22:%.*]] = phi <4 x i64> [ [[TMP14]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP20]], %[[PRED_LOAD_IF3]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_IF5]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[P]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP23]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP21]], i64 [[TMP24]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP26:%.*]] = getelementptr i64, ptr [[P2]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP26]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP27]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP29:%.*]] = phi <4 x i64> [ [[TMP21]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], %[[PRED_LOAD_IF5]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP30:%.*]] = phi <4 x i64> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP28]], %[[PRED_LOAD_IF5]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP31:%.*]] = add <4 x i64> [[TMP29]], [[TMP30]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP32:%.*]] = extractelement <4 x i64> [[TMP31]], i64 0
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP32]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF7]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP33:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP34:%.*]] = extractelement <4 x i64> [[TMP31]], i64 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP34]], ptr [[TMP33]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE8]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF9]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP35:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP36:%.*]] = extractelement <4 x i64> [[TMP31]], i64 2
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP36]], ptr [[TMP35]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE10]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 true, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF11]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP37:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP38:%.*]] = extractelement <4 x i64> [[TMP31]], i64 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP38]], ptr [[TMP37]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE12]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
+;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_independent_memdep_triple_check(
-; COMPARE-NO-MV-SCALABLE-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-SCALABLE-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_MEMCHECK]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = shl i64 [[TMP3]], 5
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = add i64 [[TMP4]], -1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 5
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = add nsw i64 [[TMP4]], -1
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-NO-MV-SCALABLE-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP6]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
@@ -1400,50 +2331,13 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_independent_memdep_triple_check(
-; COMPARE-LAA-MV-SCALABLE-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-SCALABLE-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4, [[TMP1]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_MEMCHECK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 5
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP5:%.*]] = add i64 [[TMP3]], -1
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP6:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP0]], 2
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 4, [[TMP7]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 4, [[N_MOD_VF]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP8]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i64>, ptr [[TMP9]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[TMP10]], ptr [[TMP11]], align 8
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; COMPARE-LAA-MV-SCALABLE: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 4, [[N_VEC]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; COMPARE-LAA-MV-SCALABLE: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1454,7 +2348,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
>From e2336e639b64b306e087b0ffcfda0aeca721ae40 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 7 Aug 2026 08:52:30 -0700
Subject: [PATCH 6/6] Update CHECKs after merge
---
.../vplan-based-stride-mv-btc.ll | 32 +++++++++----------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
index 7328b3c1c9852..ceafea07d890c 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
@@ -39,7 +39,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -99,7 +99,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
@@ -137,7 +137,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -212,7 +212,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -407,7 +407,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -481,7 +481,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -704,7 +704,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -780,7 +780,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -1008,7 +1008,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -1070,7 +1070,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX1]], 4
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX1]], 3
; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX1]], [[N_MOD_VF]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
@@ -1109,7 +1109,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -1186,7 +1186,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -1385,7 +1385,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -1475,7 +1475,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
@@ -1917,7 +1917,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -2029,7 +2029,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
More information about the llvm-commits
mailing list