[llvm-branch-commits] [llvm] [LoopUtils] Check the first outer iteration for co-moving pointer groups (PR #243836)

Valeriy Savchenko via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Wed Nov 18 04:52:22 PST 2026


https://github.com/SavchenkoValeriy created https://github.com/llvm/llvm-project/pull/243836

When runtime checks are hoisted out of the outer loop, the range of each
pointer group is widened to cover all iterations of the outer loop. If both
groups of a check advance by the same step in the outer loop, e.g. two rows
of the same image:

```cpp
for (y = 0; y < H; y += 2)
  for (x = 0; x < n; x++) {
    img[y*S + 3*x]     = ...;
    img[(y+1)*S + 3*x] = ...;
  }
```

the widened ranges overlap as soon as the outer loop runs twice, so the
check always fails and the vector loop never runs.

If all bounds of both groups are affine AddRecs of the outer loop with the
same step, the relative position of the two ranges is the same in every
outer iteration, so they overlap in some iteration iff they overlap in the
first one.

This change uses the bounds of the first iteration: the check is exact,
invariant in the outer loop, and needs neither the outer trip count nor
a stride check.

---

<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>

>From fe539dcbe13f625201d3d39eb502ed71e231311c Mon Sep 17 00:00:00 2001
From: Valeriy Savchenko <vsavchenko at apple.com>
Date: Fri, 13 Nov 2026 11:21:39 +0000
Subject: [PATCH 1/2] [LV] Add pre-commit tests (NFC)

---
 .../runtime-checks-hoist-same-outer-step.ll   | 633 ++++++++++++++++++
 1 file changed, 633 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll

diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll
new file mode 100644
index 0000000000000..1fffb0cdb89ae
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll
@@ -0,0 +1,633 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+define void @two_rows_same_buffer(ptr %img, ptr %src, i64 %h, i64 %s, i64 %n) {
+; CHECK-LABEL: define void @two_rows_same_buffer(
+; CHECK-SAME: ptr [[IMG:%.*]], ptr [[SRC:%.*]], i64 [[H:%.*]], i64 [[S:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SCEVGEP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[N]]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[S]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[H]], i64 2)
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], -2
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[S]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[N]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], -2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP7]]
+; CHECK-NEXT:    [[SCEVGEP16:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[S]]
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[S]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP11]], -2
+; CHECK-NEXT:    [[SCEVGEP17:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP15]]
+; CHECK-NEXT:    br label %[[OUTER:.*]]
+; CHECK:       [[OUTER]]:
+; CHECK-NEXT:    [[Y:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[Y_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[ROW0:%.*]] = mul i64 [[Y]], [[S]]
+; CHECK-NEXT:    [[Y1:%.*]] = add i64 [[Y]], 1
+; CHECK-NEXT:    [[ROW1:%.*]] = mul i64 [[Y1]], [[S]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[IMG]], [[SCEVGEP]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP16]], [[SCEVGEP17]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
+; CHECK-NEXT:    [[BOUND018:%.*]] = icmp ult ptr [[IMG]], [[SCEVGEP13]]
+; CHECK-NEXT:    [[BOUND119:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP17]]
+; CHECK-NEXT:    [[FOUND_CONFLICT20:%.*]] = and i1 [[BOUND018]], [[BOUND119]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[TMP13]], [[FOUND_CONFLICT20]]
+; CHECK-NEXT:    [[BOUND021:%.*]] = icmp ult ptr [[SCEVGEP16]], [[SCEVGEP13]]
+; CHECK-NEXT:    [[BOUND122:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT23:%.*]] = and i1 [[BOUND021]], [[BOUND122]]
+; CHECK-NEXT:    [[CONFLICT_RDX24:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT23]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX24]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP17]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[ROW0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT25:%.*]] = insertelement <4 x i64> poison, i64 [[ROW1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT26:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT25]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT:    [[TMP20:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT:    store i8 [[TMP29]], ptr [[TMP22]], align 1, !alias.scope [[META3:![0-9]+]], !noalias [[META5:![0-9]+]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT:    store i8 [[TMP30]], ptr [[TMP24]], align 1, !alias.scope [[META3]], !noalias [[META5]]
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 2
+; CHECK-NEXT:    store i8 [[TMP31]], ptr [[TMP26]], align 1, !alias.scope [[META3]], !noalias [[META5]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 3
+; CHECK-NEXT:    store i8 [[TMP32]], ptr [[TMP28]], align 1, !alias.scope [[META3]], !noalias [[META5]]
+; CHECK-NEXT:    [[TMP33:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT:    [[TMP34:%.*]] = add <4 x i64> [[BROADCAST_SPLAT26]], [[TMP18]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x i64> [[TMP34]], i64 0
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP35]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x i64> [[TMP34]], i64 1
+; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <4 x i64> [[TMP34]], i64 2
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i64> [[TMP34]], i64 3
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <4 x i8> [[TMP33]], i64 0
+; CHECK-NEXT:    store i8 [[TMP43]], ptr [[TMP36]], align 1, !alias.scope [[META7:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <4 x i8> [[TMP33]], i64 1
+; CHECK-NEXT:    store i8 [[TMP44]], ptr [[TMP38]], align 1, !alias.scope [[META7]], !noalias [[META0]]
+; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <4 x i8> [[TMP33]], i64 2
+; CHECK-NEXT:    store i8 [[TMP45]], ptr [[TMP40]], align 1, !alias.scope [[META7]], !noalias [[META0]]
+; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <4 x i8> [[TMP33]], i64 3
+; CHECK-NEXT:    store i8 [[TMP46]], ptr [[TMP42]], align 1, !alias.scope [[META7]], !noalias [[META0]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP47]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[X:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[X_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[X3:%.*]] = mul i64 [[X]], 3
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[X]]
+; CHECK-NEXT:    [[V:%.*]] = load i8, ptr [[GEP_SRC]], align 1
+; CHECK-NEXT:    [[I0:%.*]] = add i64 [[ROW0]], [[X3]]
+; CHECK-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[I0]]
+; CHECK-NEXT:    store i8 [[V]], ptr [[GEP_0]], align 1
+; CHECK-NEXT:    [[V1:%.*]] = add i8 [[V]], 1
+; CHECK-NEXT:    [[I1:%.*]] = add i64 [[ROW1]], [[X3]]
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[I1]]
+; CHECK-NEXT:    store i8 [[V1]], ptr [[GEP_1]], align 1
+; CHECK-NEXT:    [[X_NEXT]] = add nuw nsw i64 [[X]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[X_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[Y_NEXT]] = add nuw nsw i64 [[Y]], 2
+; CHECK-NEXT:    [[EC_OUTER:%.*]] = icmp ult i64 [[Y_NEXT]], [[H]]
+; CHECK-NEXT:    br i1 [[EC_OUTER]], label %[[OUTER]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer
+
+outer:
+  %y = phi i64 [ 0, %entry ], [ %y.next, %outer.latch ]
+  %row0 = mul i64 %y, %s
+  %y1 = add i64 %y, 1
+  %row1 = mul i64 %y1, %s
+  br label %inner
+
+inner:
+  %x = phi i64 [ 0, %outer ], [ %x.next, %inner ]
+  %x3 = mul i64 %x, 3
+  %gep.src = getelementptr inbounds i8, ptr %src, i64 %x
+  %v = load i8, ptr %gep.src, align 1
+  %i0 = add i64 %row0, %x3
+  %gep.0 = getelementptr inbounds i8, ptr %img, i64 %i0
+  store i8 %v, ptr %gep.0, align 1
+  %v1 = add i8 %v, 1
+  %i1 = add i64 %row1, %x3
+  %gep.1 = getelementptr inbounds i8, ptr %img, i64 %i1
+  store i8 %v1, ptr %gep.1, align 1
+  %x.next = add nuw nsw i64 %x, 1
+  %ec = icmp eq i64 %x.next, %n
+  br i1 %ec, label %outer.latch, label %inner
+
+outer.latch:
+  %y.next = add nuw nsw i64 %y, 2
+  %ec.outer = icmp ult i64 %y.next, %h
+  br i1 %ec.outer, label %outer, label %exit
+
+exit:
+  ret void
+}
+
+define void @descending_outer(ptr %dst, ptr %src, i64 %m, i64 %n) {
+; CHECK-LABEL: define void @descending_outer(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[M:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[N]], -4
+; CHECK-NEXT:    [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[M]], i64 0)
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[SMIN]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[N]], [[M]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP3]], 2
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
+; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT:    br label %[[OUTER:.*]]
+; CHECK:       [[OUTER]]:
+; CHECK-NEXT:    [[Y:%.*]] = phi i64 [ [[M]], %[[ENTRY]] ], [ [[Y_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[ROW:%.*]] = mul nsw i64 [[Y]], [[N]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP6]], [[SCEVGEP]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP4]], [[SCEVGEP5]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP5]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw i64 [[ROW]], [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4, !alias.scope [[META12:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP6]]
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META12]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[WIDE_LOAD7]], [[WIDE_LOAD]]
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr [[TMP8]], align 4, !alias.scope [[META15]], !noalias [[META12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[X:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[X_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[I:%.*]] = add nsw i64 [[ROW]], [[X]]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[I]]
+; CHECK-NEXT:    [[L_SRC:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I]]
+; CHECK-NEXT:    [[L_DST:%.*]] = load i32, ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[L_DST]], [[L_SRC]]
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[X_NEXT]] = add nuw nsw i64 [[X]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[X_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[Y_NEXT]] = add nsw i64 [[Y]], -1
+; CHECK-NEXT:    [[EC_OUTER:%.*]] = icmp sgt i64 [[Y]], 0
+; CHECK-NEXT:    br i1 [[EC_OUTER]], label %[[OUTER]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer
+
+outer:
+  %y = phi i64 [ %m, %entry ], [ %y.next, %outer.latch ]
+  %row = mul nsw i64 %y, %n
+  br label %inner
+
+inner:
+  %x = phi i64 [ 0, %outer ], [ %x.next, %inner ]
+  %i = add nsw i64 %row, %x
+  %gep.src = getelementptr inbounds i32, ptr %src, i64 %i
+  %l.src = load i32, ptr %gep.src, align 4
+  %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %i
+  %l.dst = load i32, ptr %gep.dst, align 4
+  %add = add i32 %l.dst, %l.src
+  store i32 %add, ptr %gep.dst, align 4
+  %x.next = add nuw nsw i64 %x, 1
+  %ec = icmp eq i64 %x.next, %n
+  br i1 %ec, label %outer.latch, label %inner
+
+outer.latch:
+  %y.next = add nsw i64 %y, -1
+  %ec.outer = icmp sgt i64 %y, 0
+  br i1 %ec.outer, label %outer, label %exit
+
+exit:
+  ret void
+}
+
+define void @uncountable_outer(ptr %dst, ptr %src, ptr %flags, i64 %n) {
+; CHECK-LABEL: define void @uncountable_outer(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[FLAGS:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT:    br label %[[OUTER:.*]]
+; CHECK:       [[OUTER]]:
+; CHECK-NEXT:    [[Y:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[Y_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[ROW:%.*]] = mul nuw nsw i64 [[Y]], [[N]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[Y]], [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP0]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[TMP8]], [[TMP12]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[ROW]], [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META19:![0-9]+]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META19]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr [[TMP4]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[X:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[X_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[I:%.*]] = add nuw nsw i64 [[ROW]], [[X]]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[I]]
+; CHECK-NEXT:    [[L_SRC:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I]]
+; CHECK-NEXT:    [[L_DST:%.*]] = load i32, ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[L_DST]], [[L_SRC]]
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[X_NEXT]] = add nuw nsw i64 [[X]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[X_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[Y_NEXT]] = add nuw nsw i64 [[Y]], 1
+; CHECK-NEXT:    [[GEP_FLAG:%.*]] = getelementptr inbounds i8, ptr [[FLAGS]], i64 [[Y]]
+; CHECK-NEXT:    [[FLAG:%.*]] = load i8, ptr [[GEP_FLAG]], align 1
+; CHECK-NEXT:    [[EC_OUTER:%.*]] = icmp eq i8 [[FLAG]], 0
+; CHECK-NEXT:    br i1 [[EC_OUTER]], label %[[EXIT:.*]], label %[[OUTER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer
+
+outer:
+  %y = phi i64 [ 0, %entry ], [ %y.next, %outer.latch ]
+  %row = mul nuw nsw i64 %y, %n
+  br label %inner
+
+inner:
+  %x = phi i64 [ 0, %outer ], [ %x.next, %inner ]
+  %i = add nuw nsw i64 %row, %x
+  %gep.src = getelementptr inbounds i32, ptr %src, i64 %i
+  %l.src = load i32, ptr %gep.src, align 4
+  %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %i
+  %l.dst = load i32, ptr %gep.dst, align 4
+  %add = add i32 %l.dst, %l.src
+  store i32 %add, ptr %gep.dst, align 4
+  %x.next = add nuw nsw i64 %x, 1
+  %ec = icmp eq i64 %x.next, %n
+  br i1 %ec, label %outer.latch, label %inner
+
+outer.latch:
+  %y.next = add nuw nsw i64 %y, 1
+  %gep.flag = getelementptr inbounds i8, ptr %flags, i64 %y
+  %flag = load i8, ptr %gep.flag, align 1
+  %ec.outer = icmp eq i8 %flag, 0
+  br i1 %ec.outer, label %exit, label %outer
+
+exit:
+  ret void
+}
+
+define void @forked_needs_freeze(ptr %dst, ptr %src.1, ptr %src.2, i64 %m, i64 %n, i64 %t) {
+; CHECK-LABEL: define void @forked_needs_freeze(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC_1:%.*]], ptr [[SRC_2:%.*]], i64 [[M:%.*]], i64 [[N:%.*]], i64 [[T:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[N]], [[M]]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[TMP26]], 2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC_2]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP25:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[SRC_1]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
+; CHECK-NEXT:    br label %[[OUTER:.*]]
+; CHECK:       [[OUTER]]:
+; CHECK-NEXT:    [[Y:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[Y_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[ROW:%.*]] = mul nuw nsw i64 [[Y]], [[N]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze ptr [[SRC_1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = freeze ptr [[SCEVGEP6]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[TMP2]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[TMP1]], [[SCEVGEP7]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP25]], 0
+; CHECK-NEXT:    [[TMP27:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
+; CHECK-NEXT:    [[TMP3:%.*]] = freeze ptr [[SRC_2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze ptr [[SCEVGEP]]
+; CHECK-NEXT:    [[BOUND08:%.*]] = icmp ult ptr [[DST]], [[TMP4]]
+; CHECK-NEXT:    [[BOUND19:%.*]] = icmp ult ptr [[TMP3]], [[SCEVGEP7]]
+; CHECK-NEXT:    [[FOUND_CONFLICT10:%.*]] = and i1 [[BOUND08]], [[BOUND19]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[TMP27]], [[FOUND_CONFLICT10]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP5]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[ROW]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i64> poison, i64 [[T]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT11]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nuw nsw <4 x i64> [[BROADCAST_SPLAT]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT12]]
+; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds [4 x i8], ptr [[SRC_1]], <4 x i64> [[TMP6]]
+; CHECK-NEXT:    [[WIDE_GEP13:%.*]] = getelementptr inbounds [4 x i8], ptr [[SRC_2]], <4 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP8]], <4 x ptr> [[WIDE_GEP]], <4 x ptr> [[WIDE_GEP13]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x ptr> [[TMP9]], i64 0
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !alias.scope [[META26:![0-9]+]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x ptr> [[TMP9]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !alias.scope [[META26]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x ptr> [[TMP9]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !alias.scope [[META26]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x ptr> [[TMP9]], i64 3
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !alias.scope [[META26]]
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <4 x i32> poison, i32 [[TMP11]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP13]], i64 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP15]], i64 2
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP17]], i64 3
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4, !alias.scope [[META29:![0-9]+]], !noalias [[META31:![0-9]+]]
+; CHECK-NEXT:    [[TMP23:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[TMP21]]
+; CHECK-NEXT:    store <4 x i32> [[TMP23]], ptr [[TMP22]], align 4, !alias.scope [[META29]], !noalias [[META31]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[X:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[X_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[I:%.*]] = add nuw nsw i64 [[ROW]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[X]], [[T]]
+; CHECK-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i64 [[I]]
+; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i64 [[I]]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = select i1 [[C]], ptr [[GEP_SRC_1]], ptr [[GEP_SRC_2]]
+; CHECK-NEXT:    [[L_SRC:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I]]
+; CHECK-NEXT:    [[L_DST:%.*]] = load i32, ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[L_DST]], [[L_SRC]]
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[X_NEXT]] = add nuw nsw i64 [[X]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[X_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[Y_NEXT]] = add nuw nsw i64 [[Y]], 1
+; CHECK-NEXT:    [[EC_OUTER:%.*]] = icmp eq i64 [[Y_NEXT]], [[M]]
+; CHECK-NEXT:    br i1 [[EC_OUTER]], label %[[EXIT:.*]], label %[[OUTER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer
+
+outer:
+  %y = phi i64 [ 0, %entry ], [ %y.next, %outer.latch ]
+  %row = mul nuw nsw i64 %y, %n
+  br label %inner
+
+inner:
+  %x = phi i64 [ 0, %outer ], [ %x.next, %inner ]
+  %i = add nuw nsw i64 %row, %x
+  %c = icmp ult i64 %x, %t
+  %gep.src.1 = getelementptr inbounds i32, ptr %src.1, i64 %i
+  %gep.src.2 = getelementptr inbounds i32, ptr %src.2, i64 %i
+  %gep.src = select i1 %c, ptr %gep.src.1, ptr %gep.src.2
+  %l.src = load i32, ptr %gep.src, align 4
+  %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %i
+  %l.dst = load i32, ptr %gep.dst, align 4
+  %add = add i32 %l.dst, %l.src
+  store i32 %add, ptr %gep.dst, align 4
+  %x.next = add nuw nsw i64 %x, 1
+  %ec = icmp eq i64 %x.next, %n
+  br i1 %ec, label %outer.latch, label %inner
+
+outer.latch:
+  %y.next = add nuw nsw i64 %y, 1
+  %ec.outer = icmp eq i64 %y.next, %m
+  br i1 %ec.outer, label %exit, label %outer
+
+exit:
+  ret void
+}
+
+; Negative test: the inner trip count depends on the outer IV, so the upper
+; bounds move by a different step than the lower bounds.
+define void @triangular(ptr %dst, ptr %src, i64 %m, i64 %n) {
+; CHECK-LABEL: define void @triangular(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[M:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 4
+; CHECK-NEXT:    br label %[[OUTER:.*]]
+; CHECK:       [[OUTER]]:
+; CHECK-NEXT:    [[Y:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[Y_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[ROW:%.*]] = mul nuw nsw i64 [[Y]], [[N]]
+; CHECK-NEXT:    [[TC:%.*]] = add nuw nsw i64 [[Y]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[Y]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[Y]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 4, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[TMP3]], [[TMP8]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TC]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[TMP9]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[ROW]], [[INDEX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP10]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4, !alias.scope [[META35:![0-9]+]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP10]]
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4, !alias.scope [[META38:![0-9]+]], !noalias [[META35]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD]]
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP12]], align 4, !alias.scope [[META38]], !noalias [[META35]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[X:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[X_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[I:%.*]] = add nuw nsw i64 [[ROW]], [[X]]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[I]]
+; CHECK-NEXT:    [[L_SRC:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[I]]
+; CHECK-NEXT:    [[L_DST:%.*]] = load i32, ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[L_DST]], [[L_SRC]]
+; CHECK-NEXT:    store i32 [[ADD]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[X_NEXT]] = add nuw nsw i64 [[X]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[X_NEXT]], [[TC]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[Y_NEXT]] = add nuw nsw i64 [[Y]], 1
+; CHECK-NEXT:    [[EC_OUTER:%.*]] = icmp eq i64 [[Y_NEXT]], [[M]]
+; CHECK-NEXT:    br i1 [[EC_OUTER]], label %[[EXIT:.*]], label %[[OUTER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer
+
+outer:
+  %y = phi i64 [ 0, %entry ], [ %y.next, %outer.latch ]
+  %row = mul nuw nsw i64 %y, %n
+  %tc = add nuw nsw i64 %y, 1
+  br label %inner
+
+inner:
+  %x = phi i64 [ 0, %outer ], [ %x.next, %inner ]
+  %i = add nuw nsw i64 %row, %x
+  %gep.src = getelementptr inbounds i32, ptr %src, i64 %i
+  %l.src = load i32, ptr %gep.src, align 4
+  %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %i
+  %l.dst = load i32, ptr %gep.dst, align 4
+  %add = add i32 %l.dst, %l.src
+  store i32 %add, ptr %gep.dst, align 4
+  %x.next = add nuw nsw i64 %x, 1
+  %ec = icmp eq i64 %x.next, %tc
+  br i1 %ec, label %outer.latch, label %inner
+
+outer.latch:
+  %y.next = add nuw nsw i64 %y, 1
+  %ec.outer = icmp eq i64 %y.next, %m
+  br i1 %ec.outer, label %exit, label %outer
+
+exit:
+  ret void
+}
+;.
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]], i1 false, !"LVerDomain"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]], [[META1]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}
+; CHECK: [[META7]] = !{[[META6]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]], [[META10:![0-9]+]]}
+; CHECK: [[META9]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META10]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META9]]}
+; CHECK: [[META12]] = !{[[META13:![0-9]+]]}
+; CHECK: [[META13]] = distinct !{[[META13]], [[META14:![0-9]+]]}
+; CHECK: [[META14]] = distinct !{[[META14]], i1 false, !"LVerDomain"}
+; CHECK: [[META15]] = !{[[META16:![0-9]+]]}
+; CHECK: [[META16]] = distinct !{[[META16]], [[META14]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META9]], [[META10]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META9]]}
+; CHECK: [[META19]] = !{[[META20:![0-9]+]]}
+; CHECK: [[META20]] = distinct !{[[META20]], [[META21:![0-9]+]]}
+; CHECK: [[META21]] = distinct !{[[META21]], i1 false, !"LVerDomain"}
+; CHECK: [[META22]] = !{[[META23:![0-9]+]]}
+; CHECK: [[META23]] = distinct !{[[META23]], [[META21]]}
+; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META9]], [[META10]]}
+; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META9]]}
+; CHECK: [[META26]] = !{[[META27:![0-9]+]]}
+; CHECK: [[META27]] = distinct !{[[META27]], [[META28:![0-9]+]]}
+; CHECK: [[META28]] = distinct !{[[META28]], i1 false, !"LVerDomain"}
+; CHECK: [[META29]] = !{[[META30:![0-9]+]]}
+; CHECK: [[META30]] = distinct !{[[META30]], [[META28]]}
+; CHECK: [[META31]] = !{[[META32:![0-9]+]], [[META27]]}
+; CHECK: [[META32]] = distinct !{[[META32]], [[META28]]}
+; CHECK: [[LOOP33]] = distinct !{[[LOOP33]], [[META9]], [[META10]]}
+; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[META9]]}
+; CHECK: [[META35]] = !{[[META36:![0-9]+]]}
+; CHECK: [[META36]] = distinct !{[[META36]], [[META37:![0-9]+]]}
+; CHECK: [[META37]] = distinct !{[[META37]], i1 false, !"LVerDomain"}
+; CHECK: [[META38]] = !{[[META39:![0-9]+]]}
+; CHECK: [[META39]] = distinct !{[[META39]], [[META37]]}
+; CHECK: [[LOOP40]] = distinct !{[[LOOP40]], [[META9]], [[META10]]}
+; CHECK: [[LOOP41]] = distinct !{[[LOOP41]], [[META9]]}
+;.

>From a444609fed9a53def74c045640984838ffc37bc3 Mon Sep 17 00:00:00 2001
From: Valeriy Savchenko <vsavchenko at apple.com>
Date: Wed, 11 Nov 2026 13:51:56 +0000
Subject: [PATCH 2/2] [LoopUtils] Check the first outer iteration for co-moving
 pointer groups

When runtime checks are hoisted out of the outer loop, the range of each
pointer group is widened to cover all iterations of the outer loop. If both
groups of a check advance by the same step in the outer loop, e.g. two rows
of the same image:

```cpp
for (y = 0; y < H; y += 2)
  for (x = 0; x < n; x++) {
    img[y*S + 3*x]     = ...;
    img[(y+1)*S + 3*x] = ...;
  }
```

the widened ranges overlap as soon as the outer loop runs twice, so the
check always fails and the vector loop never runs.

If all bounds of both groups are affine AddRecs of the outer loop with the
same step, the relative position of the two ranges is the same in every
outer iteration, so they overlap in some iteration iff they overlap in the
first one.

This change uses the bounds of the first iteration: the check is exact,
invariant in the outer loop, and needs neither the outer trip count nor
a stride check.
---
 llvm/lib/Transforms/Utils/LoopUtils.cpp       | 27 +++++--
 .../AArch64/low_trip_memcheck_cost.ll         | 78 +++++++++++++++----
 .../runtime-checks-difference-rmw-hoist.ll    |  5 +-
 .../runtime-checks-hoist-same-outer-step.ll   | 51 ++++++------
 .../LoopVectorize/runtime-checks-hoist.ll     | 32 ++++----
 .../AArch64/matrix-extract-insert.ll          |  5 +-
 6 files changed, 129 insertions(+), 69 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index cefd05ce4413e..de7db30bc8184 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2200,8 +2200,8 @@ getGroupBounds(const RuntimeCheckingPtrGroup *CG, const Loop *TheLoop,
   // which can be significant for low trip counts. The disadvantage is that
   // there is a chance we may now never enter the vectorized inner loop,
   // whereas using a restricted range check could have allowed us to enter at
-  // least once. This is why the behaviour is not currently the default and is
-  // controlled by the parameter 'HoistRuntimeChecks'.
+  // least once. This is why the behaviour is controlled by the parameter
+  // 'HoistRuntimeChecks'.
   const Loop *OuterLoop = TheLoop->getParentLoop();
   if (HoistRuntimeChecks && OuterLoop &&
       haveSameStepIn({Low, High}, OuterLoop, SE)) {
@@ -2235,16 +2235,33 @@ RuntimeCheckBounds
 llvm::getRuntimeCheckBounds(const RuntimePointerCheck &Check,
                             const Loop *TheLoop, ScalarEvolution &SE,
                             bool HoistRuntimeChecks) {
+  // If both groups advance by the same step in the parent loop, their relative
+  // position is the same in every iteration of it, so they overlap in some
+  // iteration iff they overlap in the first one. Checking the first iteration
+  // is exact and invariant in the parent loop, and needs neither widening nor
+  // a stride check.
+  const Loop *ParentLoop = TheLoop->getParentLoop();
+  const auto &[A, B] = Check;
+  bool UseFirstIteration =
+      HoistRuntimeChecks && ParentLoop &&
+      haveSameStepIn({A->Low, A->High, B->Low, B->High}, ParentLoop, SE);
   auto GetBounds = [&](const RuntimeCheckingPtrGroup *CG) {
     LLVM_DEBUG(dbgs() << "LAA: Adding RT check for range:\n");
-    RuntimeCheckGroupBounds Bounds =
-        getGroupBounds(CG, TheLoop, SE, HoistRuntimeChecks);
+    RuntimeCheckGroupBounds Bounds;
+    if (UseFirstIteration) {
+      LLVM_DEBUG(dbgs() << "LAA: Using RT check range of the first outer loop "
+                           "iteration in order to permit hoisting\n");
+      Bounds = {cast<SCEVAddRecExpr>(CG->Low)->getStart(),
+                cast<SCEVAddRecExpr>(CG->High)->getStart(), nullptr};
+    } else {
+      Bounds = getGroupBounds(CG, TheLoop, SE, HoistRuntimeChecks);
+    }
     LLVM_DEBUG(dbgs() << "Start: " << *Bounds.Low << " End: " << *Bounds.High
                       << "\n");
     return Bounds;
   };
   // Braced initialization evaluates the groups in order.
-  return {GetBounds(Check.first), GetBounds(Check.second)};
+  return {GetBounds(A), GetBounds(B)};
 }
 
 /// Expand code for the lower and upper bound \p Bounds of the pointer group
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
index 565e59948e182..488f8fe0c4b90 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
@@ -32,8 +32,46 @@ inner.exit:
 define void @outer_no_tc(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
 ; CHECK-LABEL: LV: Checking a loop in 'outer_no_tc'
 ; CHECK:      Calculating cost of runtime checks:
-; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 2
-; CHECK:      Total cost of runtime checks: 2
+; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 3
+; CHECK:      Total cost of runtime checks: 3
+; CHECK:      LV: Minimum required TC for runtime checks to be profitable:16
+entry:
+  br label %outer.loop
+
+outer.loop:
+  %outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
+  %mul.us = mul nsw i64 %outer.iv, %n
+  %mul.b = shl nsw i64 %mul.us, 1
+  br label %inner.loop
+
+inner.loop:
+  %inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
+  %add.us = add nuw nsw i64 %inner.iv, %mul.us
+  %add.b = add nuw nsw i64 %inner.iv, %mul.b
+  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
+  %0 = load i8, ptr %arrayidx.us, align 1
+  %arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
+  %1 = load i8, ptr %arrayidx7.us, align 1
+  %add9.us = add i8 %1, %0
+  store i8 %add9.us, ptr %arrayidx7.us, align 1
+  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+  %exitcond.not = icmp eq i64 %inner.iv.next, %n
+  br i1 %exitcond.not, label %inner.exit, label %inner.loop
+
+inner.exit:
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %exitcond27.not = icmp eq i64 %outer.iv.next, %m
+  br i1 %exitcond27.not, label %outer.exit, label %outer.loop
+
+outer.exit:
+  ret void
+}
+
+define void @outer_no_tc_same_step(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
+; CHECK-LABEL: LV: Checking a loop in 'outer_no_tc_same_step'
+; CHECK:      Calculating cost of runtime checks:
+; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 3 to 1
+; CHECK:      Total cost of runtime checks: 1
 ; CHECK:      LV: Minimum required TC for runtime checks to be profitable:16
 entry:
   br label %outer.loop
@@ -69,8 +107,8 @@ outer.exit:
 define void @outer_known_tc3(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %n) {
 ; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc3'
 ; CHECK:      Calculating cost of runtime checks:
-; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
-; CHECK:      Total cost of runtime checks: 1
+; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 2
+; CHECK:      Total cost of runtime checks: 2
 ; CHECK:      LV: Minimum required TC for runtime checks to be profitable:16
 entry:
   br label %outer.loop
@@ -78,12 +116,14 @@ entry:
 outer.loop:
   %outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
   %mul.us = mul nsw i64 %outer.iv, %n
+  %mul.b = shl nsw i64 %mul.us, 1
   br label %inner.loop
 
 inner.loop:
   %inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
   %add.us = add nuw nsw i64 %inner.iv, %mul.us
-  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.us
+  %add.b = add nuw nsw i64 %inner.iv, %mul.b
+  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
   %0 = load i8, ptr %arrayidx.us, align 1
   %arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
   %1 = load i8, ptr %arrayidx7.us, align 1
@@ -106,7 +146,7 @@ outer.exit:
 define void @outer_known_tc64(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %n) {
 ; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc64'
 ; CHECK:      Calculating cost of runtime checks:
-; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
+; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 1
 ; CHECK:      Total cost of runtime checks: 1
 ; CHECK:      LV: Minimum required TC for runtime checks to be profitable:16
 entry:
@@ -115,12 +155,14 @@ entry:
 outer.loop:
   %outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
   %mul.us = mul nsw i64 %outer.iv, %n
+  %mul.b = shl nsw i64 %mul.us, 1
   br label %inner.loop
 
 inner.loop:
   %inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
   %add.us = add nuw nsw i64 %inner.iv, %mul.us
-  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.us
+  %add.b = add nuw nsw i64 %inner.iv, %mul.b
+  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
   %0 = load i8, ptr %arrayidx.us, align 1
   %arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
   %1 = load i8, ptr %arrayidx7.us, align 1
@@ -143,8 +185,8 @@ outer.exit:
 define void @outer_pgo_3(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
 ; CHECK-LABEL: LV: Checking a loop in 'outer_pgo_3'
 ; CHECK:      Calculating cost of runtime checks:
-; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
-; CHECK:      Total cost of runtime checks: 1
+; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 2
+; CHECK:      Total cost of runtime checks: 2
 ; CHECK:      LV: Minimum required TC for runtime checks to be profitable:16
 entry:
   br label %outer.loop
@@ -152,12 +194,14 @@ entry:
 outer.loop:
   %outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
   %mul.us = mul nsw i64 %outer.iv, %n
+  %mul.b = shl nsw i64 %mul.us, 1
   br label %inner.loop
 
 inner.loop:
   %inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
   %add.us = add nuw nsw i64 %inner.iv, %mul.us
-  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.us
+  %add.b = add nuw nsw i64 %inner.iv, %mul.b
+  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
   %0 = load i8, ptr %arrayidx.us, align 1
   %arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
   %1 = load i8, ptr %arrayidx7.us, align 1
@@ -180,7 +224,7 @@ outer.exit:
 define void @outer_pgo_minus1(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
 ; CHECK-LABEL: LV: Checking a loop in 'outer_pgo_minus1'
 ; CHECK:      Calculating cost of runtime checks:
-; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
+; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 1
 ; CHECK:      Total cost of runtime checks: 1
 ; CHECK:      LV: Minimum required TC for runtime checks to be profitable:16
 entry:
@@ -189,12 +233,14 @@ entry:
 outer.loop:
   %outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
   %mul.us = mul nsw i64 %outer.iv, %n
+  %mul.b = shl nsw i64 %mul.us, 1
   br label %inner.loop
 
 inner.loop:
   %inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
   %add.us = add nuw nsw i64 %inner.iv, %mul.us
-  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.us
+  %add.b = add nuw nsw i64 %inner.iv, %mul.b
+  %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
   %0 = load i8, ptr %arrayidx.us, align 1
   %arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
   %1 = load i8, ptr %arrayidx7.us, align 1
@@ -217,8 +263,8 @@ outer.exit:
 define void @outer_known_tc3_full_range_checks(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i64 noundef %n) {
 ; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc3_full_range_checks'
 ; CHECK:      Calculating cost of runtime checks:
-; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
-; CHECK:      Total cost of runtime checks: 1
+; CHECK:      We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 2
+; CHECK:      Total cost of runtime checks: 2
 ; CHECK:      LV: Minimum required TC for runtime checks to be profitable:4
 entry:
   br label %outer.loop
@@ -226,12 +272,14 @@ entry:
 outer.loop:
   %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %inner.exit ]
   %0 = mul nsw i64 %outer.iv, %n
+  %mul.src = shl nsw i64 %0, 1
   br label %inner.loop
 
 inner.loop:
   %iv.inner = phi i64 [ 0, %outer.loop ], [ %iv.inner.next, %inner.loop ]
   %1 = add nuw nsw i64 %iv.inner, %0
-  %arrayidx.us = getelementptr inbounds i32, ptr %src, i64 %1
+  %add.src = add nuw nsw i64 %iv.inner, %mul.src
+  %arrayidx.us = getelementptr inbounds i32, ptr %src, i64 %add.src
   %2 = load i32, ptr %arrayidx.us, align 4
   %arrayidx8.us = getelementptr inbounds i32, ptr %dst, i64 %1
   %3 = load i32, ptr %arrayidx8.us, align 4
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll
index 5ca0e8c289af1..dd0c7df895837 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll
@@ -78,9 +78,8 @@ define void @rmw_invariant_distance(ptr %dst, ptr %src, i64 %rows) {
 ; HOIST-LABEL: define void @rmw_invariant_distance(
 ; HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) {
 ; HOIST-NEXT:  [[ENTRY:.*]]:
-; HOIST-NEXT:    [[TMP0:%.*]] = shl i64 [[ROWS]], 6
-; HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
-; HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
+; HOIST-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 64
+; HOIST-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 64
 ; HOIST-NEXT:    br label %[[OUTER_HEADER:.*]]
 ; HOIST:       [[OUTER_HEADER]]:
 ; HOIST-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll
index 1fffb0cdb89ae..686126042e43c 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll
@@ -7,8 +7,6 @@ define void @two_rows_same_buffer(ptr %img, ptr %src, i64 %h, i64 %s, i64 %n) {
 ; CHECK-LABEL: define void @two_rows_same_buffer(
 ; CHECK-SAME: ptr [[IMG:%.*]], ptr [[SRC:%.*]], i64 [[H:%.*]], i64 [[S:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[SCEVGEP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[N]]
-; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[S]], 1
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[H]], i64 2)
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[UMAX]], -1
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], -2
@@ -18,12 +16,19 @@ define void @two_rows_same_buffer(ptr %img, ptr %src, i64 %h, i64 %s, i64 %n) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP4]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], -2
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP7]]
-; CHECK-NEXT:    [[SCEVGEP16:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[S]]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[S]], 1
+; CHECK-NEXT:    [[SCEVGEP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[N]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[S]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[TMP9]], 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], [[TMP5]]
-; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP11]], -2
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], -2
+; CHECK-NEXT:    [[SCEVGEP14:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[S]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP13]], -2
+; CHECK-NEXT:    [[SCEVGEP15:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP14]]
+; CHECK-NEXT:    [[SCEVGEP16:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[S]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP5]], -2
 ; CHECK-NEXT:    [[SCEVGEP17:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP15]]
 ; CHECK-NEXT:    br label %[[OUTER:.*]]
 ; CHECK:       [[OUTER]]:
@@ -34,15 +39,15 @@ define void @two_rows_same_buffer(ptr %img, ptr %src, i64 %h, i64 %s, i64 %n) {
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[IMG]], [[SCEVGEP]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[IMG]], [[SCEVGEP15]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP16]], [[SCEVGEP17]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP13:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
 ; CHECK-NEXT:    [[BOUND018:%.*]] = icmp ult ptr [[IMG]], [[SCEVGEP13]]
-; CHECK-NEXT:    [[BOUND119:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP17]]
+; CHECK-NEXT:    [[BOUND119:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP14]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT20:%.*]] = and i1 [[BOUND018]], [[BOUND119]]
-; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[TMP13]], [[FOUND_CONFLICT20]]
+; CHECK-NEXT:    [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[FOUND_CONFLICT20]], [[STRIDE_CHECK]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[TMP16]]
 ; CHECK-NEXT:    [[BOUND021:%.*]] = icmp ult ptr [[SCEVGEP16]], [[SCEVGEP13]]
 ; CHECK-NEXT:    [[BOUND122:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT23:%.*]] = and i1 [[BOUND021]], [[BOUND122]]
@@ -168,9 +173,7 @@ define void @descending_outer(ptr %dst, ptr %src, i64 %m, i64 %n) {
 ; CHECK-LABEL: define void @descending_outer(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[M:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[N]], -4
-; CHECK-NEXT:    [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[M]], i64 0)
-; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[SMIN]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[M]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
@@ -189,9 +192,7 @@ define void @descending_outer(ptr %dst, ptr %src, i64 %m, i64 %n) {
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP6]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP4]], [[SCEVGEP5]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP11]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[N]], 3
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP5]]
@@ -268,6 +269,8 @@ define void @uncountable_outer(ptr %dst, ptr %src, ptr %flags, i64 %n) {
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[FLAGS:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
 ; CHECK-NEXT:    br label %[[OUTER:.*]]
 ; CHECK:       [[OUTER]]:
 ; CHECK-NEXT:    [[Y:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[Y_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
@@ -275,14 +278,8 @@ define void @uncountable_outer(ptr %dst, ptr %src, ptr %flags, i64 %n) {
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[Y]], [[TMP0]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP0]], [[TMP7]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP9]]
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[TMP8]], [[TMP12]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP2]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -364,10 +361,8 @@ define void @forked_needs_freeze(ptr %dst, ptr %src.1, ptr %src.2, i64 %m, i64 %
 ; CHECK-LABEL: define void @forked_needs_freeze(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC_1:%.*]], ptr [[SRC_2:%.*]], i64 [[M:%.*]], i64 [[N:%.*]], i64 [[T:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[N]], [[M]]
-; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[TMP26]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[N]], 2
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC_2]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP25:%.*]] = shl i64 [[N]], 2
 ; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[SRC_1]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
 ; CHECK-NEXT:    br label %[[OUTER:.*]]
@@ -382,14 +377,12 @@ define void @forked_needs_freeze(ptr %dst, ptr %src.1, ptr %src.2, i64 %m, i64 %
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[TMP2]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[TMP1]], [[SCEVGEP7]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP25]], 0
-; CHECK-NEXT:    [[TMP27:%.*]] = or i1 [[FOUND_CONFLICT]], [[STRIDE_CHECK]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = freeze ptr [[SRC_2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = freeze ptr [[SCEVGEP]]
 ; CHECK-NEXT:    [[BOUND08:%.*]] = icmp ult ptr [[DST]], [[TMP4]]
 ; CHECK-NEXT:    [[BOUND19:%.*]] = icmp ult ptr [[TMP3]], [[SCEVGEP7]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT10:%.*]] = and i1 [[BOUND08]], [[BOUND19]]
-; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[TMP27]], [[FOUND_CONFLICT10]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT10]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[N]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
index aea2fcabf82eb..7996c2ba9e027 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
@@ -151,11 +151,11 @@ outer.exit:
 ; DEBUG: LAA: Found an analyzable loop: inner.loop
 ; DEBUG-NOT: LAA: Creating diff runtime check for:
 ; DEBUG: LAA: Adding RT check for range:
-; DEBUG-NEXT: LAA: Expanded RT check for range to include outer loop in order to permit hoisting
-; DEBUG-NEXT: Start: %dst End: ((4 * (zext i32 %m to i64) * (zext i32 %n to i64)) + %dst)
+; DEBUG-NEXT: LAA: Using RT check range of the first outer loop iteration in order to permit hoisting
+; DEBUG-NEXT: Start: %dst End: ((4 * (zext i32 %n to i64))<nuw><nsw> + %dst)
 ; DEBUG-NEXT: LAA: Adding RT check for range:
-; DEBUG-NEXT: LAA: Expanded RT check for range to include outer loop in order to permit hoisting
-; DEBUG-NEXT: Start: %src End: ((4 * (zext i32 %m to i64) * (zext i32 %n to i64)) + %src)
+; DEBUG-NEXT: LAA: Using RT check range of the first outer loop iteration in order to permit hoisting
+; DEBUG-NEXT: Start: %src End: ((4 * (zext i32 %n to i64))<nuw><nsw> + %src)
 
 define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i32 noundef %m, i32 noundef %n) {
 ; CHECK-LABEL: define void @full_checks(
@@ -164,8 +164,7 @@ define void @full_checks(ptr nocapture noundef %dst, ptr nocapture noundef reado
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[WIDE_M:%.*]] = zext i32 [[M]] to i64
 ; CHECK-NEXT:    [[WIDE_N:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[WIDE_N]], [[WIDE_M]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[WIDE_N]], 2
 ; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
 ; CHECK-NEXT:    br label %[[OUTER_LOOP:.*]]
@@ -2063,9 +2062,10 @@ define void @memcheck_folds_to_false_keeps_expansions(ptr %p, i32 %n, i64 %m) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[J]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], [[TMP4]]
 ; CHECK-NEXT:    [[ROW:%.*]] = mul i64 [[J]], [[CNT]]
+; CHECK-NEXT:    [[ROW_Q:%.*]] = shl i64 [[ROW]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[CNT]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK:       [[VECTOR_PH1]]:
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[CNT]], 3
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[CNT]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[N_VEC]], [[INDUCTION_IV]]
@@ -2078,11 +2078,12 @@ define void @memcheck_folds_to_false_keeps_expansions(ptr %p, i32 %n, i64 %m) {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ [[TMP9]], %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[ROW]], [[INDEX]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[Q]], i64 [[TMP11]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4, !alias.scope [[META100:![0-9]+]]
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[ROW_Q]], [[INDEX]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[Q]], i64 [[TMP12]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP17]], align 4, !alias.scope [[META100:![0-9]+]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4, !alias.scope [[META103:![0-9]+]], !noalias [[META100]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD3]]
@@ -2103,7 +2104,8 @@ define void @memcheck_folds_to_false_keeps_expansions(ptr %p, i32 %n, i64 %m) {
 ; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[INNER]] ]
 ; CHECK-NEXT:    [[K:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
 ; CHECK-NEXT:    [[IDX:%.*]] = add i64 [[ROW]], [[I]]
-; CHECK-NEXT:    [[GEP_Q:%.*]] = getelementptr i32, ptr [[Q]], i64 [[IDX]]
+; CHECK-NEXT:    [[IDX_Q:%.*]] = add i64 [[ROW_Q]], [[I]]
+; CHECK-NEXT:    [[GEP_Q:%.*]] = getelementptr i32, ptr [[Q]], i64 [[IDX_Q]]
 ; CHECK-NEXT:    [[L_Q:%.*]] = load i32, ptr [[GEP_Q]], align 4
 ; CHECK-NEXT:    [[GEP_P:%.*]] = getelementptr i32, ptr [[P]], i64 [[IDX]]
 ; CHECK-NEXT:    [[L_P:%.*]] = load i32, ptr [[GEP_P]], align 4
@@ -2137,13 +2139,15 @@ entry:
 outer:
   %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
   %row = mul i64 %j, %cnt
+  %row.q = shl i64 %row, 1
   br label %inner
 
 inner:
   %i = phi i64 [ 0, %outer ], [ %i.next, %inner ]
   %k = phi i64 [ 0, %outer ], [ %k.next, %inner ]
   %idx = add i64 %row, %i
-  %gep.q = getelementptr i32, ptr %q, i64 %idx
+  %idx.q = add i64 %row.q, %i
+  %gep.q = getelementptr i32, ptr %q, i64 %idx.q
   %l.q = load i32, ptr %gep.q, align 4
   %gep.p = getelementptr i32, ptr %p, i64 %idx
   %l.p = load i32, ptr %gep.p, align 4
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
index 4bd8876512ce4..98e6e09563e87 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
@@ -88,9 +88,8 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
 ; CHECK-NEXT:    br i1 [[CMP210_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       for.cond1.preheader.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[CONV6]], 3
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 360
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP1]]
-; CHECK-NEXT:    [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[I]], 225
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[CONV6]]



More information about the llvm-branch-commits mailing list