[clang] [llvm] [InstCombine] Relax singleUse for GEP(p,add(x,constant)) to 2 GEPs (PR #209551)
Abhay Kanhere via cfe-commits
cfe-commits at lists.llvm.org
Thu Jul 23 16:47:44 PDT 2026
https://github.com/AbhayKanhere updated https://github.com/llvm/llvm-project/pull/209551
>From 95ec5a1dac3d5fbe81a186aeefd9759613622e9f Mon Sep 17 00:00:00 2001
From: Abhay Kanhere <a_kanhere at apple.com>
Date: Tue, 14 Jul 2026 09:33:23 -0700
Subject: [PATCH] [InstCombine] Relax singleUse for GEP(p,add(x,constant)) to 2
GEPs
Bound checks for memory safety involve checking index expression before
access.
e.g.
for ( i =0 ;i <n ; i++ ) {
if( i - k > bound)
llvm.trap()
a[ i - k-1] ...
a[ i - k]
}
so invariably, the index expression is no longer singleUse and this prevents
fold of GEP(p,ADD(x,y))) -> GEP( GEP(p,x),y). This later prevents optimizations
such as wideing the load or store
SingleUse check here is not a correctness check.
Here we relax the requirement of singleUse for constant 'y'. This enables
later optimizations such as aggressive instcombine widening with simplified
GEP with constant offsets.
Note we also fixed a latent bug
instCombine fold is valid only if add is NSW or same width as pointer i.e. no sext.
https://alive2.llvm.org/ce/z/V62MEC -- Negative test -- preexisting latent bug.
https://alive2.llvm.org/ce/z/rmkzRu -- this shows requiring NSW on addition.
https://alive2.llvm.org/ce/z/2BXGrR -- this is valid if operands are same width
Added test:
InstCombine/gep-addlike-multiuse.ll
InstCombine/gep-add-nsw-narrow-index.ll -- reproduces latent bug on main
---
.../CodeGenCXX/microsoft-abi-dynamic-cast.cpp | 8 +-
.../InstCombine/InstructionCombining.cpp | 12 +-
.../InstCombine/gep-add-nsw-narrow-index.ll | 38 +++++
.../InstCombine/gep-addlike-multiuse.ll | 65 +++++++++
.../Transforms/InstCombine/gepofconstgepi8.ll | 4 +-
.../Transforms/InstCombine/getelementptr.ll | 6 +-
llvm/test/Transforms/InstCombine/lshr-phi.ll | 3 +-
.../AArch64/runtime-unroll-generic.ll | 50 ++++---
.../LoopUnroll/WebAssembly/basic-unrolling.ll | 31 ++--
.../LoopUnroll/runtime-unroll-remainder.ll | 35 +++--
.../LoopVectorize/float-induction.ll | 10 +-
.../LoopVectorize/if-conversion-nest.ll | 14 +-
.../Transforms/LoopVectorize/if-conversion.ll | 7 +-
.../Transforms/LoopVectorize/induction.ll | 57 ++++----
.../interleaved-accesses-pred-stores.ll | 30 ++--
.../LoopVectorize/interleaved-accesses.ll | 63 +++++----
.../Transforms/LoopVectorize/loop-scalars.ll | 22 +--
.../LoopVectorize/reduction-inloop-cond.ll | 63 +++++----
.../AArch64/Oz-and-forced-vectorize.ll | 7 +-
.../AArch64/extra-unroll-simplifications.ll | 14 +-
.../AArch64/indvars-vectorization.ll | 7 +-
.../AArch64/instcombine-gep-widen.ll | 59 ++++++++
.../PhaseOrdering/AArch64/interleave_vec.ll | 133 ++++++++++--------
.../AArch64/matrix-extract-insert.ll | 97 +++++++------
.../test/Transforms/PhaseOrdering/X86/vdiv.ll | 49 ++++---
25 files changed, 569 insertions(+), 315 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/gep-add-nsw-narrow-index.ll
create mode 100644 llvm/test/Transforms/InstCombine/gep-addlike-multiuse.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/AArch64/instcombine-gep-widen.ll
diff --git a/clang/test/CodeGenCXX/microsoft-abi-dynamic-cast.cpp b/clang/test/CodeGenCXX/microsoft-abi-dynamic-cast.cpp
index a5d0d5ad6e91f..b61b9b6f26f04 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-dynamic-cast.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-dynamic-cast.cpp
@@ -31,7 +31,8 @@ T* test3(B* x) { return &dynamic_cast<T&>(*x); }
// CHECK-NEXT: [[VBOFFP:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
// CHECK-NEXT: [[VBOFFS:%.*]] = load i32, ptr [[VBOFFP]], align 4
// CHECK-NEXT: [[DELTA:%.*]] = add nsw i32 [[VBOFFS]], 4
-// CHECK-NEXT: [[ADJ:%.*]] = getelementptr inbounds i8, ptr %x, i32 [[DELTA]]
+// CHECK-NEXT: [[BASE:%.*]] = getelementptr i8, ptr %x, i32 [[VBOFFS]]
+// CHECK-NEXT: [[ADJ:%.*]] = getelementptr i8, ptr [[BASE]], i32 4
// CHECK-NEXT: [[CALL:%.*]] = tail call ptr @__RTDynamicCast(ptr [[ADJ]], i32 [[DELTA]], ptr nonnull @"??_R0?AUB@@@8", ptr nonnull @"??_R0?AUT@@@8", i32 1)
// CHECK-NEXT: ret ptr [[CALL]]
@@ -62,8 +63,9 @@ T* test6(B* x) { return dynamic_cast<T*>(x); }
// CHECK-NEXT: [[VBOFFP:%.*]] = getelementptr inbounds nuw i8, ptr [[VBTBL]], i32 4
// CHECK-NEXT: [[VBOFFS:%.*]] = load i32, ptr [[VBOFFP]], align 4
// CHECK-NEXT: [[DELTA:%.*]] = add nsw i32 [[VBOFFS]], 4
-// CHECK-NEXT: [[ADJ:%.*]] = getelementptr inbounds i8, ptr %x, i32 [[DELTA]]
-// CHECK-NEXT: [[CALL:%.*]] = tail call ptr @__RTDynamicCast(ptr nonnull [[ADJ]], i32 [[DELTA]], ptr {{.*}}@"??_R0?AUB@@@8", ptr {{.*}}@"??_R0?AUT@@@8", i32 0)
+// CHECK-NEXT: [[BASE:%.*]] = getelementptr i8, ptr %x, i32 [[VBOFFS]]
+// CHECK-NEXT: [[ADJ:%.*]] = getelementptr i8, ptr [[BASE]], i32 4
+// CHECK-NEXT: [[CALL:%.*]] = tail call ptr @__RTDynamicCast(ptr [[ADJ]], i32 [[DELTA]], ptr {{.*}}@"??_R0?AUB@@@8", ptr {{.*}}@"??_R0?AUT@@@8", i32 0)
// CHECK-NEXT: br label
// CHECK: [[RET:%.*]] = phi ptr
// CHECK-NEXT: ret ptr [[RET]]
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index acb144e1e807d..3d41f430f912c 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3656,10 +3656,16 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
return GEPNoWrapFlags::none();
};
- // Try to replace ADD + GEP with GEP + GEP.
+ // Try to replace ADD + GEP with GEP + GEP. Relax singleUse for a constant
+ // addend. The GEP sign-extends the index to the pointer width, so the split
+ // is sound only when sext(idx1 + idx2) = sext(idx1) + sext(idx2): require
+ // nsw unless the index is already pointer-width (no sign-extension).
Value *Idx1, *Idx2;
- if (match(GEP.getOperand(1),
- m_OneUse(m_AddLike(m_Value(Idx1), m_Value(Idx2))))) {
+ if (match(GEP.getOperand(1), m_AddLike(m_Value(Idx1), m_Value(Idx2))) &&
+ (GEP.getOperand(1)->hasOneUse() || match(Idx2, m_ConstantInt())) &&
+ (match(GEP.getOperand(1), m_NSWAddLike(m_Value(), m_Value())) ||
+ GEP.getOperand(1)->getType()->getScalarSizeInBits() >=
+ DL.getPointerSizeInBits(GEP.getPointerAddressSpace()))) {
// %idx = add i64 %idx1, %idx2
// %gep = getelementptr i32, ptr %ptr, i64 %idx
// as:
diff --git a/llvm/test/Transforms/InstCombine/gep-add-nsw-narrow-index.ll b/llvm/test/Transforms/InstCombine/gep-add-nsw-narrow-index.ll
new file mode 100644
index 0000000000000..d758e33cbc4a5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-add-nsw-narrow-index.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+; A 32-bit pointer index type on a 64-bit pointer: the GEP sign-extends the
+; index to the pointer width. Splitting GEP(p, add(x, y)) into
+; GEP(GEP(p, x), y) is only sound when the add is nsw, because otherwise
+; sext(x + y) != sext(x) + sext(y) once x + y signed-overflows. These are
+; single-use adds (no constant addend), so they exercise the original fold,
+; which previously split unconditionally and miscompiled.
+
+target datalayout = "p:64:64:64:32"
+
+; Original single-use case, no nsw: previously split and miscompiled
+; (e.g. x = 0x7fffffff, y = 1). Must NOT split now.
+define ptr @single_use_no_nsw(ptr %p, i32 %x, i32 %y) {
+; CHECK-LABEL: define ptr @single_use_no_nsw(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT: [[IDX:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x i8], ptr [[P]], i32 [[IDX]]
+; CHECK-NEXT: ret ptr [[GEP]]
+;
+ %idx = add i32 %x, %y
+ %gep = getelementptr i32, ptr %p, i32 %idx
+ ret ptr %gep
+}
+
+; Single-use with nsw: the split is sound, so it still fires.
+define ptr @single_use_nsw(ptr %p, i32 %x, i32 %y) {
+; CHECK-LABEL: define ptr @single_use_nsw(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [4 x i8], ptr [[P]], i32 [[X]]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x i8], ptr [[TMP1]], i32 [[Y]]
+; CHECK-NEXT: ret ptr [[GEP]]
+;
+ %idx = add nsw i32 %x, %y
+ %gep = getelementptr i32, ptr %p, i32 %idx
+ ret ptr %gep
+}
diff --git a/llvm/test/Transforms/InstCombine/gep-addlike-multiuse.ll b/llvm/test/Transforms/InstCombine/gep-addlike-multiuse.ll
new file mode 100644
index 0000000000000..2863932a81d80
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-addlike-multiuse.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+; Bounds-checked array access `a[i + k]`: the index `add i, k` feeds BOTH the
+; bounds-check compare (which dominates the access) AND the element GEP, so it
+; is inherently multi-use. When the addend `k` is a constant we still split it
+; out of the GEP: the inner `gep base, %idx` is shared with sibling accesses
+; (CSE) and the exposed constant offset enables later optimizations
+; (e.g. AggressiveInstCombine load-widening)
+
+define ptr @boundscheck_const_addend(ptr %base, i64 %idx, i64 %n) {
+; CHECK-LABEL: define ptr @boundscheck_const_addend(
+; CHECK-SAME: ptr [[BASE:%.*]], i64 [[IDX:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[OFF:%.*]] = add nsw i64 [[IDX]], 3
+; CHECK-NEXT: [[OOB:%.*]] = icmp ult i64 [[OFF]], [[N]]
+; CHECK-NEXT: br i1 [[OOB]], label %[[OK:.*]], label %[[TRAP:.*]]
+; CHECK: [[OK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[IDX]]
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr [[TMP1]], i64 3
+; CHECK-NEXT: ret ptr [[P]]
+; CHECK: [[TRAP]]:
+; CHECK-NEXT: call void @llvm.trap()
+; CHECK-NEXT: unreachable
+;
+ %off = add nsw i64 %idx, 3
+ %oob = icmp uge i64 %off, %n
+ br i1 %oob, label %trap, label %ok
+
+ok:
+ %p = getelementptr inbounds i8, ptr %base, i64 %off
+ ret ptr %p
+
+trap:
+ call void @llvm.trap()
+ unreachable
+}
+
+; Negative: a multi-use add with a *variable* addend must NOT split unless profitable
+define ptr @boundscheck_var_addend(ptr %base, i64 %idx, i64 %v, i64 %n) {
+; CHECK-LABEL: define ptr @boundscheck_var_addend(
+; CHECK-SAME: ptr [[BASE:%.*]], i64 [[IDX:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[OFF:%.*]] = add nsw i64 [[IDX]], [[V]]
+; CHECK-NEXT: [[OOB:%.*]] = icmp ult i64 [[OFF]], [[N]]
+; CHECK-NEXT: br i1 [[OOB]], label %[[OK:.*]], label %[[TRAP:.*]]
+; CHECK: [[OK]]:
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i8, ptr [[BASE]], i64 [[OFF]]
+; CHECK-NEXT: ret ptr [[P]]
+; CHECK: [[TRAP]]:
+; CHECK-NEXT: call void @llvm.trap()
+; CHECK-NEXT: unreachable
+;
+ %off = add nsw i64 %idx, %v
+ %oob = icmp uge i64 %off, %n
+ br i1 %oob, label %trap, label %ok
+
+ok:
+ %p = getelementptr inbounds i8, ptr %base, i64 %off
+ ret ptr %p
+
+trap:
+ call void @llvm.trap()
+ unreachable
+}
+
+declare void @llvm.trap()
diff --git a/llvm/test/Transforms/InstCombine/gepofconstgepi8.ll b/llvm/test/Transforms/InstCombine/gepofconstgepi8.ll
index 64484dd72ad3f..9e652b568d5bf 100644
--- a/llvm/test/Transforms/InstCombine/gepofconstgepi8.ll
+++ b/llvm/test/Transforms/InstCombine/gepofconstgepi8.ll
@@ -241,10 +241,10 @@ define ptr @test_nonzero_multiuse_index(ptr %base, i64 %a) {
; CHECK-LABEL: define ptr @test_nonzero_multiuse_index(
; CHECK-SAME: ptr [[BASE:%.*]], i64 [[A:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 -4
; CHECK-NEXT: [[INDEX:%.*]] = add i64 [[A]], 2
; CHECK-NEXT: call void @use64(i64 [[INDEX]])
-; CHECK-NEXT: [[P2:%.*]] = getelementptr [4 x i8], ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [4 x i8], ptr [[BASE]], i64 [[A]]
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP0]], i64 4
; CHECK-NEXT: ret ptr [[P2]]
;
entry:
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index da3ae69d41a8e..18e9f62fe51af 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -696,10 +696,10 @@ define i32 @test28() nounwind {
; CHECK-NEXT: [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[BB10]] ]
; CHECK-NEXT: [[T12_REC:%.*]] = xor i32 [[INDVAR]], -1
; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[T12_REC]] to i64
-; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
-; CHECK-NEXT: [[T12:%.*]] = getelementptr inbounds i8, ptr [[ORIENTATIONS]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[ORIENTATIONS]], i64 [[TMP0]]
+; CHECK-NEXT: [[T12:%.*]] = getelementptr i8, ptr [[TMP1]], i64 1
; CHECK-NEXT: [[T16:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str1, ptr nonnull [[T12]]) #[[ATTR0]]
-; CHECK-NEXT: [[T84:%.*]] = icmp eq i64 [[TMP1]], 0
+; CHECK-NEXT: [[T84:%.*]] = icmp eq i32 [[INDVAR]], 0
; CHECK-NEXT: [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
; CHECK-NEXT: br i1 [[T84]], label [[BB17:%.*]], label [[BB10]]
; CHECK: bb17:
diff --git a/llvm/test/Transforms/InstCombine/lshr-phi.ll b/llvm/test/Transforms/InstCombine/lshr-phi.ll
index d1c2b1e98083f..31a4036072450 100644
--- a/llvm/test/Transforms/InstCombine/lshr-phi.ll
+++ b/llvm/test/Transforms/InstCombine/lshr-phi.ll
@@ -21,7 +21,8 @@ define i32 @hash_string(ptr nocapture %key) nounwind readonly {
; CHECK-NEXT: [[T7:%.*]] = xor i32 [[T2]], [[T6]]
; CHECK-NEXT: [[T8]] = and i32 [[T7]], 16383
; CHECK-NEXT: [[T]] = add i64 [[INDVAR]], 1
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[KEY]], i64 [[T]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[KEY]], i64 [[INDVAR]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP0]], i64 1
; CHECK-NEXT: [[T9:%.*]] = load i8, ptr [[SCEVGEP]], align 1
; CHECK-NEXT: [[T10:%.*]] = icmp eq i8 [[T9]], 0
; CHECK-NEXT: br i1 [[T10]], label [[BB2]], label [[BB]]
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll b/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll
index 3411c997747d9..fe8af305a3760 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll
@@ -30,39 +30,45 @@ define void @runtime_unroll_generic(i32 %arg_0, ptr %arg_1, ptr %arg_2, ptr %arg
; CHECK-A55-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX20]], align 4
; CHECK-A55-NEXT: [[ADD21:%.*]] = add nsw i32 [[MUL16]], [[TMP3]]
; CHECK-A55-NEXT: store i32 [[ADD21]], ptr [[ARRAYIDX20]], align 4
-; CHECK-A55-NEXT: [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
-; CHECK-A55-NEXT: [[ARRAYIDX10_1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-A55-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX10_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP22]], i64 2
; CHECK-A55-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX10_1]], align 2
; CHECK-A55-NEXT: [[CONV_1:%.*]] = sext i16 [[TMP4]] to i32
-; CHECK-A55-NEXT: [[ARRAYIDX14_1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-A55-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX14_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP23]], i64 2
; CHECK-A55-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX14_1]], align 2
; CHECK-A55-NEXT: [[CONV15_1:%.*]] = sext i16 [[TMP5]] to i32
; CHECK-A55-NEXT: [[MUL16_1:%.*]] = mul nsw i32 [[CONV15_1]], [[CONV_1]]
-; CHECK-A55-NEXT: [[ARRAYIDX20_1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-A55-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX20_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP24]], i64 4
; CHECK-A55-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX20_1]], align 4
; CHECK-A55-NEXT: [[ADD21_1:%.*]] = add nsw i32 [[MUL16_1]], [[TMP6]]
; CHECK-A55-NEXT: store i32 [[ADD21_1]], ptr [[ARRAYIDX20_1]], align 4
-; CHECK-A55-NEXT: [[INDVARS_IV_NEXT_1:%.*]] = or disjoint i64 [[INDVARS_IV]], 2
-; CHECK-A55-NEXT: [[ARRAYIDX10_2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-A55-NEXT: [[TMP26:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX10_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP26]], i64 4
; CHECK-A55-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX10_2]], align 2
; CHECK-A55-NEXT: [[CONV_2:%.*]] = sext i16 [[TMP7]] to i32
-; CHECK-A55-NEXT: [[ARRAYIDX14_2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-A55-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX14_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP28]], i64 4
; CHECK-A55-NEXT: [[TMP8:%.*]] = load i16, ptr [[ARRAYIDX14_2]], align 2
; CHECK-A55-NEXT: [[CONV15_2:%.*]] = sext i16 [[TMP8]] to i32
; CHECK-A55-NEXT: [[MUL16_2:%.*]] = mul nsw i32 [[CONV15_2]], [[CONV_2]]
-; CHECK-A55-NEXT: [[ARRAYIDX20_2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-A55-NEXT: [[TMP30:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX20_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP30]], i64 8
; CHECK-A55-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX20_2]], align 4
; CHECK-A55-NEXT: [[ADD21_2:%.*]] = add nsw i32 [[MUL16_2]], [[TMP9]]
; CHECK-A55-NEXT: store i32 [[ADD21_2]], ptr [[ARRAYIDX20_2]], align 4
-; CHECK-A55-NEXT: [[INDVARS_IV_NEXT_2:%.*]] = or disjoint i64 [[INDVARS_IV]], 3
-; CHECK-A55-NEXT: [[ARRAYIDX10_3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-A55-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX10_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP32]], i64 6
; CHECK-A55-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10_3]], align 2
; CHECK-A55-NEXT: [[CONV_3:%.*]] = sext i16 [[TMP10]] to i32
-; CHECK-A55-NEXT: [[ARRAYIDX14_3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-A55-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX14_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP34]], i64 6
; CHECK-A55-NEXT: [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX14_3]], align 2
; CHECK-A55-NEXT: [[CONV15_3:%.*]] = sext i16 [[TMP11]] to i32
; CHECK-A55-NEXT: [[MUL16_3:%.*]] = mul nsw i32 [[CONV15_3]], [[CONV_3]]
-; CHECK-A55-NEXT: [[ARRAYIDX20_3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-A55-NEXT: [[TMP36:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV]]
+; CHECK-A55-NEXT: [[ARRAYIDX20_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP36]], i64 12
; CHECK-A55-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX20_3]], align 4
; CHECK-A55-NEXT: [[ADD21_3:%.*]] = add nsw i32 [[MUL16_3]], [[TMP12]]
; CHECK-A55-NEXT: store i32 [[ADD21_3]], ptr [[ARRAYIDX20_3]], align 4
@@ -91,30 +97,34 @@ define void @runtime_unroll_generic(i32 %arg_0, ptr %arg_1, ptr %arg_2, ptr %arg
; CHECK-A55-NEXT: [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 1
; CHECK-A55-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY6_EPIL_1:%.*]]
; CHECK-A55: for.body6.epil.1:
-; CHECK-A55-NEXT: [[INDVARS_IV_NEXT_EPIL:%.*]] = add nuw nsw i64 [[INDVARS_IV_UNR]], 1
-; CHECK-A55-NEXT: [[ARRAYIDX10_EPIL_1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_EPIL]]
+; CHECK-A55-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT: [[ARRAYIDX10_EPIL_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP25]], i64 2
; CHECK-A55-NEXT: [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX10_EPIL_1]], align 2
; CHECK-A55-NEXT: [[CONV_EPIL_1:%.*]] = sext i16 [[TMP16]] to i32
-; CHECK-A55-NEXT: [[ARRAYIDX14_EPIL_1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_EPIL]]
+; CHECK-A55-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT: [[ARRAYIDX14_EPIL_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP27]], i64 2
; CHECK-A55-NEXT: [[TMP17:%.*]] = load i16, ptr [[ARRAYIDX14_EPIL_1]], align 2
; CHECK-A55-NEXT: [[CONV15_EPIL_1:%.*]] = sext i16 [[TMP17]] to i32
; CHECK-A55-NEXT: [[MUL16_EPIL_1:%.*]] = mul nsw i32 [[CONV15_EPIL_1]], [[CONV_EPIL_1]]
-; CHECK-A55-NEXT: [[ARRAYIDX20_EPIL_1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_EPIL]]
+; CHECK-A55-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT: [[ARRAYIDX20_EPIL_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP29]], i64 4
; CHECK-A55-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX20_EPIL_1]], align 4
; CHECK-A55-NEXT: [[ADD21_EPIL_1:%.*]] = add nsw i32 [[MUL16_EPIL_1]], [[TMP18]]
; CHECK-A55-NEXT: store i32 [[ADD21_EPIL_1]], ptr [[ARRAYIDX20_EPIL_1]], align 4
; CHECK-A55-NEXT: [[EPIL_ITER_CMP_1_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 2
; CHECK-A55-NEXT: br i1 [[EPIL_ITER_CMP_1_NOT]], label [[FOR_END]], label [[FOR_BODY6_EPIL_2:%.*]]
; CHECK-A55: for.body6.epil.2:
-; CHECK-A55-NEXT: [[INDVARS_IV_NEXT_EPIL_1:%.*]] = add nuw nsw i64 [[INDVARS_IV_UNR]], 2
-; CHECK-A55-NEXT: [[ARRAYIDX10_EPIL_2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
+; CHECK-A55-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_2]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT: [[ARRAYIDX10_EPIL_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP31]], i64 4
; CHECK-A55-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX10_EPIL_2]], align 2
; CHECK-A55-NEXT: [[CONV_EPIL_2:%.*]] = sext i16 [[TMP19]] to i32
-; CHECK-A55-NEXT: [[ARRAYIDX14_EPIL_2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
+; CHECK-A55-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[ARG_3]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT: [[ARRAYIDX14_EPIL_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP33]], i64 4
; CHECK-A55-NEXT: [[TMP20:%.*]] = load i16, ptr [[ARRAYIDX14_EPIL_2]], align 2
; CHECK-A55-NEXT: [[CONV15_EPIL_2:%.*]] = sext i16 [[TMP20]] to i32
; CHECK-A55-NEXT: [[MUL16_EPIL_2:%.*]] = mul nsw i32 [[CONV15_EPIL_2]], [[CONV_EPIL_2]]
-; CHECK-A55-NEXT: [[ARRAYIDX20_EPIL_2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
+; CHECK-A55-NEXT: [[TMP35:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARG_1]], i64 [[INDVARS_IV_UNR]]
+; CHECK-A55-NEXT: [[ARRAYIDX20_EPIL_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP35]], i64 8
; CHECK-A55-NEXT: [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX20_EPIL_2]], align 4
; CHECK-A55-NEXT: [[ADD21_EPIL_2:%.*]] = add nsw i32 [[MUL16_EPIL_2]], [[TMP21]]
; CHECK-A55-NEXT: store i32 [[ADD21_EPIL_2]], ptr [[ARRAYIDX20_EPIL_2]], align 4
diff --git a/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll b/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll
index d842c4adcb74d..f188f4060856e 100644
--- a/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll
+++ b/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll
@@ -76,23 +76,26 @@ define hidden void @compile_time_partial(ptr nocapture %a, ptr nocapture readonl
; CHECK-NEXT: [[ADD:%.*]] = add i16 [[I]], 1
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A:%.*]], i32 [[I_07]]
; CHECK-NEXT: store i16 [[ADD]], ptr [[ARRAYIDX2]], align 2
-; CHECK-NEXT: [[INC:%.*]] = or disjoint i32 [[I_07]], 1
-; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i32 [[INC]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i32 [[I_07]]
+; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i32 2
; CHECK-NEXT: [[I_1:%.*]] = load i16, ptr [[ARRAYIDX_1]], align 2
; CHECK-NEXT: [[ADD_1:%.*]] = add i16 [[I_1]], 1
-; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i32 [[INC]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i32 [[I_07]]
+; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 2
; CHECK-NEXT: store i16 [[ADD_1]], ptr [[ARRAYIDX2_1]], align 2
-; CHECK-NEXT: [[INC_1:%.*]] = or disjoint i32 [[I_07]], 2
-; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i32 [[INC_1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i32 [[I_07]]
+; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 4
; CHECK-NEXT: [[I_2:%.*]] = load i16, ptr [[ARRAYIDX_2]], align 2
; CHECK-NEXT: [[ADD_2:%.*]] = add i16 [[I_2]], 1
-; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i32 [[INC_1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i32 [[I_07]]
+; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 4
; CHECK-NEXT: store i16 [[ADD_2]], ptr [[ARRAYIDX2_2]], align 2
-; CHECK-NEXT: [[INC_2:%.*]] = or disjoint i32 [[I_07]], 3
-; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i32 [[INC_2]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i32 [[I_07]]
+; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i32 6
; CHECK-NEXT: [[I_3:%.*]] = load i16, ptr [[ARRAYIDX_3]], align 2
; CHECK-NEXT: [[ADD_3:%.*]] = add i16 [[I_3]], 1
-; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i32 [[INC_2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i32 [[I_07]]
+; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i32 6
; CHECK-NEXT: store i16 [[ADD_3]], ptr [[ARRAYIDX2_3]], align 2
; CHECK-NEXT: [[INC_3]] = add nuw nsw i32 [[I_07]], 4
; CHECK-NEXT: [[EXITCOND_NOT_3:%.*]] = icmp eq i32 [[INC_3]], 1000
@@ -155,13 +158,15 @@ define hidden void @runtime(ptr nocapture %a, ptr nocapture readonly %b, ptr noc
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[I1]], [[I]]
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i32 [[I_09]]
; CHECK-NEXT: store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[INC:%.*]] = or disjoint i32 [[I_09]], 1
-; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i32 [[INC]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [4 x i8], ptr [[B]], i32 [[I_09]]
+; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr i8, ptr [[TMP1]], i32 4
; CHECK-NEXT: [[I_1:%.*]] = load i32, ptr [[ARRAYIDX_1]], align 4
-; CHECK-NEXT: [[ARRAYIDX1_1:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i32 [[INC]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [4 x i8], ptr [[C]], i32 [[I_09]]
+; CHECK-NEXT: [[ARRAYIDX1_1:%.*]] = getelementptr i8, ptr [[TMP2]], i32 4
; CHECK-NEXT: [[I1_1:%.*]] = load i32, ptr [[ARRAYIDX1_1]], align 4
; CHECK-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[I1_1]], [[I_1]]
-; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i32 [[INC]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [4 x i8], ptr [[A]], i32 [[I_09]]
+; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr i8, ptr [[TMP3]], i32 4
; CHECK-NEXT: store i32 [[MUL_1]], ptr [[ARRAYIDX2_1]], align 4
; CHECK-NEXT: [[INC_1]] = add nuw i32 [[I_09]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add nuw i32 [[NITER]], 2
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
index 3c9c3bd10a93c..0a9958cc59891 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
@@ -34,20 +34,22 @@ define i32 @unroll(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %N)
; CHECK-NEXT: [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 1
; CHECK-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA:%.*]], label [[FOR_BODY_EPIL_1:%.*]]
; CHECK: for.body.epil.1:
-; CHECK-NEXT: [[INDVARS_IV_NEXT_EPIL:%.*]] = add nuw nsw i64 [[INDVARS_IV_EPIL_INIT]], 1
-; CHECK-NEXT: [[ARRAYIDX_EPIL_1:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDVARS_IV_NEXT_EPIL]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDVARS_IV_EPIL_INIT]]
+; CHECK-NEXT: [[ARRAYIDX_EPIL_1:%.*]] = getelementptr i8, ptr [[TMP16]], i64 4
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX_EPIL_1]], align 4
-; CHECK-NEXT: [[ARRAYIDX2_EPIL_1:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDVARS_IV_NEXT_EPIL]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[INDVARS_IV_EPIL_INIT]]
+; CHECK-NEXT: [[ARRAYIDX2_EPIL_1:%.*]] = getelementptr i8, ptr [[TMP18]], i64 4
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX2_EPIL_1]], align 4
; CHECK-NEXT: [[MUL_EPIL_1:%.*]] = mul nsw i32 [[TMP4]], [[TMP3]]
; CHECK-NEXT: [[ADD_EPIL_1:%.*]] = add nsw i32 [[MUL_EPIL_1]], [[ADD_EPIL]]
; CHECK-NEXT: [[EPIL_ITER_CMP_1_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 2
; CHECK-NEXT: br i1 [[EPIL_ITER_CMP_1_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA]], label [[FOR_BODY_EPIL_2:%.*]]
; CHECK: for.body.epil.2:
-; CHECK-NEXT: [[INDVARS_IV_NEXT_EPIL_1:%.*]] = add nuw nsw i64 [[INDVARS_IV_EPIL_INIT]], 2
-; CHECK-NEXT: [[ARRAYIDX_EPIL_2:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDVARS_IV_EPIL_INIT]]
+; CHECK-NEXT: [[ARRAYIDX_EPIL_2:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8
; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX_EPIL_2]], align 4
-; CHECK-NEXT: [[ARRAYIDX2_EPIL_2:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDVARS_IV_NEXT_EPIL_1]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[INDVARS_IV_EPIL_INIT]]
+; CHECK-NEXT: [[ARRAYIDX2_EPIL_2:%.*]] = getelementptr i8, ptr [[TMP22]], i64 8
; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX2_EPIL_2]], align 4
; CHECK-NEXT: [[MUL_EPIL_2:%.*]] = mul nsw i32 [[TMP6]], [[TMP5]]
; CHECK-NEXT: [[ADD_EPIL_2:%.*]] = add nsw i32 [[MUL_EPIL_2]], [[ADD_EPIL_1]]
@@ -71,24 +73,27 @@ define i32 @unroll(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %N)
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL]], [[C_010]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP24]], i64 4
; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX_1]], align 4
-; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 4
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX2_1]], align 4
; CHECK-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[TMP10]], [[TMP9]]
; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i32 [[MUL_1]], [[ADD]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_1:%.*]] = or disjoint i64 [[INDVARS_IV]], 2
-; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 8
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX_2]], align 4
-; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP19]], i64 8
; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX2_2]], align 4
; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[TMP12]], [[TMP11]]
; CHECK-NEXT: [[ADD_2:%.*]] = add nsw i32 [[MUL_2]], [[ADD_1]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_2:%.*]] = or disjoint i64 [[INDVARS_IV]], 3
-; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP21]], i64 12
; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX_3]], align 4
-; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP23]], i64 12
; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX2_3]], align 4
; CHECK-NEXT: [[MUL_3:%.*]] = mul nsw i32 [[TMP14]], [[TMP13]]
; CHECK-NEXT: [[ADD_3]] = add nsw i32 [[MUL_3]], [[ADD_2]]
diff --git a/llvm/test/Transforms/LoopVectorize/float-induction.ll b/llvm/test/Transforms/LoopVectorize/float-induction.ll
index 813542d71904e..49f30b7edddbf 100644
--- a/llvm/test/Transforms/LoopVectorize/float-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/float-induction.ll
@@ -949,14 +949,14 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
; VEC1_INTERL2-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC1_INTERL2: [[VECTOR_BODY]]:
; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = or disjoint i64 [[INDEX]], 1
; VEC1_INTERL2-NEXT: [[DOTCAST2:%.*]] = sitofp i64 [[INDEX]] to float
; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = fmul fast float [[DOTCAST2]], -5.000000e-01
; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP0]], [[DOTCAST2]]
; VEC1_INTERL2-NEXT: [[OFFSET_IDX4:%.*]] = fadd fast float [[INIT]], [[TMP8]]
; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = fadd fast float [[OFFSET_IDX4]], [[TMP0]]
; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP6]]
+; VEC1_INTERL2-NEXT: [[TMP23:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP23]], i64 4
; VEC1_INTERL2-NEXT: store float [[OFFSET_IDX4]], ptr [[TMP10]], align 4
; VEC1_INTERL2-NEXT: store float [[TMP9]], ptr [[TMP11]], align 4
; VEC1_INTERL2-NEXT: [[TMP12:%.*]] = fadd fast float [[OFFSET_IDX4]], [[TMP0]]
@@ -966,11 +966,13 @@ define void @fp_iv_loop3(float %init, ptr noalias nocapture %A, ptr noalias noca
; VEC1_INTERL2-NEXT: [[TMP16:%.*]] = fadd fast float [[TMP14]], [[TMP12]]
; VEC1_INTERL2-NEXT: [[TMP17:%.*]] = fadd fast float [[TMP15]], [[TMP13]]
; VEC1_INTERL2-NEXT: [[TMP18:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP19:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP6]]
+; VEC1_INTERL2-NEXT: [[TMP24:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP24]], i64 4
; VEC1_INTERL2-NEXT: store float [[TMP16]], ptr [[TMP18]], align 4
; VEC1_INTERL2-NEXT: store float [[TMP17]], ptr [[TMP19]], align 4
; VEC1_INTERL2-NEXT: [[TMP20:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i64 [[INDEX]]
-; VEC1_INTERL2-NEXT: [[TMP21:%.*]] = getelementptr inbounds [4 x i8], ptr [[C]], i64 [[TMP6]]
+; VEC1_INTERL2-NEXT: [[TMP25:%.*]] = getelementptr [4 x i8], ptr [[C]], i64 [[INDEX]]
+; VEC1_INTERL2-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP25]], i64 4
; VEC1_INTERL2-NEXT: store float [[TMP14]], ptr [[TMP20]], align 4
; VEC1_INTERL2-NEXT: store float [[TMP15]], ptr [[TMP21]], align 4
; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
index 39c3d0a56a28e..df8c52f8dcb45 100644
--- a/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
@@ -16,9 +16,10 @@ define void @foo(ptr nocapture %A, ptr nocapture %B, i32 %n) {
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[N]], -1
; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 4
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP4]]
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP3]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP3]]
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP14]], i64 4
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -123,9 +124,10 @@ define void @multi_variable_if_nest(ptr nocapture %A, ptr nocapture %B, i32 %n)
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[N]], -1
; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 4
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP4]]
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP3]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP3]]
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP9]], i64 4
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion.ll b/llvm/test/Transforms/LoopVectorize/if-conversion.ll
index f3b8305e71630..f1565ecf88dd2 100644
--- a/llvm/test/Transforms/LoopVectorize/if-conversion.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion.ll
@@ -39,10 +39,11 @@ define void @function0(ptr nocapture %a, ptr nocapture %b, i32 %start, i32 %end)
; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP7]] to i64
; CHECK-NEXT: [[TMP9:%.*]] = add nsw i64 [[TMP0]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = shl nsw i64 [[TMP9]], 2
-; CHECK-NEXT: [[TMP11:%.*]] = add nsw i64 [[TMP10]], 4
-; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP11]], i64 4
; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[TMP21]], i64 4
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index f2ede41effb54..b8af55cda41c6 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -923,15 +923,16 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) {
; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; IND-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
; IND-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
-; IND-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
; IND-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[OFFSET_IDX]]
-; IND-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP3]]
+; IND-NEXT: [[TMP28:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; IND-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP28]], i64 32
; IND-NEXT: [[TMP6:%.*]] = load float, ptr [[TMP4]], align 4
; IND-NEXT: [[TMP7:%.*]] = load float, ptr [[TMP5]], align 4
; IND-NEXT: [[TMP8:%.*]] = insertelement <2 x float> poison, float [[TMP6]], i64 0
; IND-NEXT: [[TMP9:%.*]] = insertelement <2 x float> [[TMP8]], float [[TMP7]], i64 1
; IND-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[OFFSET_IDX]]
-; IND-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP3]]
+; IND-NEXT: [[TMP29:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; IND-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP29]], i64 32
; IND-NEXT: [[TMP12:%.*]] = load float, ptr [[TMP10]], align 4
; IND-NEXT: [[TMP13:%.*]] = load float, ptr [[TMP11]], align 4
; IND-NEXT: [[TMP14:%.*]] = insertelement <2 x float> poison, float [[TMP12]], i64 0
@@ -984,13 +985,13 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) {
; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP35:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
-; UNROLL-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
-; UNROLL-NEXT: [[TMP4:%.*]] = or disjoint i64 [[OFFSET_IDX]], 16
-; UNROLL-NEXT: [[TMP5:%.*]] = or disjoint i64 [[OFFSET_IDX]], 24
; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[OFFSET_IDX]]
-; UNROLL-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP3]]
-; UNROLL-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP4]]
-; UNROLL-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP5]]
+; UNROLL-NEXT: [[TMP45:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; UNROLL-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP45]], i64 32
+; UNROLL-NEXT: [[TMP46:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; UNROLL-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP46]], i64 64
+; UNROLL-NEXT: [[TMP47:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; UNROLL-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP47]], i64 96
; UNROLL-NEXT: [[TMP10:%.*]] = load float, ptr [[TMP6]], align 4
; UNROLL-NEXT: [[TMP11:%.*]] = load float, ptr [[TMP7]], align 4
; UNROLL-NEXT: [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP10]], i64 0
@@ -1000,9 +1001,12 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) {
; UNROLL-NEXT: [[TMP16:%.*]] = insertelement <2 x float> poison, float [[TMP14]], i64 0
; UNROLL-NEXT: [[TMP17:%.*]] = insertelement <2 x float> [[TMP16]], float [[TMP15]], i64 1
; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[OFFSET_IDX]]
-; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP3]]
-; UNROLL-NEXT: [[TMP20:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP4]]
-; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP5]]
+; UNROLL-NEXT: [[TMP48:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP48]], i64 32
+; UNROLL-NEXT: [[TMP49:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; UNROLL-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[TMP49]], i64 64
+; UNROLL-NEXT: [[TMP50:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP50]], i64 96
; UNROLL-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP18]], align 4
; UNROLL-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP19]], align 4
; UNROLL-NEXT: [[TMP24:%.*]] = insertelement <2 x float> poison, float [[TMP22]], i64 0
@@ -1145,15 +1149,16 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) {
; INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = or disjoint i64 [[OFFSET_IDX]], 32
; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[OFFSET_IDX]]
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP5]]
+; INTERLEAVE-NEXT: [[TMP25:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP25]], i64 128
; INTERLEAVE-NEXT: [[WIDE_VEC:%.*]] = load <32 x float>, ptr [[TMP6]], align 4
; INTERLEAVE-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x float> [[WIDE_VEC]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
; INTERLEAVE-NEXT: [[WIDE_VEC2:%.*]] = load <32 x float>, ptr [[TMP7]], align 4
; INTERLEAVE-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <32 x float> [[WIDE_VEC2]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[OFFSET_IDX]]
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP5]]
+; INTERLEAVE-NEXT: [[TMP26:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP26]], i64 128
; INTERLEAVE-NEXT: [[WIDE_VEC4:%.*]] = load <32 x float>, ptr [[TMP8]], align 4
; INTERLEAVE-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <32 x float> [[WIDE_VEC4]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
; INTERLEAVE-NEXT: [[WIDE_VEC6:%.*]] = load <32 x float>, ptr [[TMP9]], align 4
@@ -1847,13 +1852,13 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]
; INTERLEAVE: vector.body:
; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 4
-; INTERLEAVE-NEXT: [[DOTIDX:%.*]] = shl nsw i64 [[INDEX]], 4
-; INTERLEAVE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[DOTIDX]]
-; INTERLEAVE-NEXT: [[DOTIDX5:%.*]] = shl nsw i64 [[TMP14]], 4
+; INTERLEAVE-NEXT: [[DOTIDX5:%.*]] = shl nsw i64 [[INDEX]], 4
; INTERLEAVE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[DOTIDX5]]
-; INTERLEAVE-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP18]], align 1, !alias.scope [[META17:![0-9]+]]
-; INTERLEAVE-NEXT: [[WIDE_VEC3:%.*]] = load <16 x i32>, ptr [[TMP19]], align 1, !alias.scope [[META17]]
+; INTERLEAVE-NEXT: [[TMP12:%.*]] = shl i64 [[INDEX]], 4
+; INTERLEAVE-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]]
+; INTERLEAVE-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i64 64
+; INTERLEAVE-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP19]], align 1, !alias.scope [[META17:![0-9]+]]
+; INTERLEAVE-NEXT: [[WIDE_VEC3:%.*]] = load <16 x i32>, ptr [[TMP14]], align 1, !alias.scope [[META17]]
; INTERLEAVE-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[INDEX]]
; INTERLEAVE-NEXT: [[TMP41:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 4
; INTERLEAVE-NEXT: [[TMP15:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
@@ -1862,10 +1867,10 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
; INTERLEAVE-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[TMP17]], i64 20
; INTERLEAVE-NEXT: [[TMP43:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
; INTERLEAVE-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[TMP43]], i64 28
-; INTERLEAVE-NEXT: [[DOTSPLIT9:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP14]]
-; INTERLEAVE-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT9]], i64 4
; INTERLEAVE-NEXT: [[TMP22:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP22]], i64 44
+; INTERLEAVE-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP22]], i64 36
+; INTERLEAVE-NEXT: [[TMP44:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; INTERLEAVE-NEXT: [[TMP45:%.*]] = getelementptr i8, ptr [[TMP44]], i64 44
; INTERLEAVE-NEXT: [[TMP24:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
; INTERLEAVE-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[TMP24]], i64 52
; INTERLEAVE-NEXT: [[TMP26:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
@@ -1879,9 +1884,9 @@ define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) {
; INTERLEAVE-NEXT: [[TMP31:%.*]] = extractelement <16 x i32> [[WIDE_VEC]], i64 12
; INTERLEAVE-NEXT: store i32 [[TMP31]], ptr [[TMP20]], align 1, !alias.scope [[META20]], !noalias [[META17]]
; INTERLEAVE-NEXT: [[TMP32:%.*]] = extractelement <16 x i32> [[WIDE_VEC3]], i64 0
-; INTERLEAVE-NEXT: store i32 [[TMP32]], ptr [[TMP21]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; INTERLEAVE-NEXT: store i32 [[TMP32]], ptr [[TMP23]], align 1, !alias.scope [[META20]], !noalias [[META17]]
; INTERLEAVE-NEXT: [[TMP33:%.*]] = extractelement <16 x i32> [[WIDE_VEC3]], i64 4
-; INTERLEAVE-NEXT: store i32 [[TMP33]], ptr [[TMP23]], align 1, !alias.scope [[META20]], !noalias [[META17]]
+; INTERLEAVE-NEXT: store i32 [[TMP33]], ptr [[TMP45]], align 1, !alias.scope [[META20]], !noalias [[META17]]
; INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <16 x i32> [[WIDE_VEC3]], i64 8
; INTERLEAVE-NEXT: store i32 [[TMP34]], ptr [[TMP25]], align 1, !alias.scope [[META20]], !noalias [[META17]]
; INTERLEAVE-NEXT: [[TMP35:%.*]] = extractelement <16 x i32> [[WIDE_VEC3]], i64 12
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
index 34dc6c4f7a7d8..8838e0f8ee509 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
@@ -120,35 +120,35 @@ define void @interleaved_with_cond_store_1(ptr %p, i64 %x, i64 %n) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [16 x i8], ptr [[P:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [16 x i8], ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 8
-; CHECK-NEXT: [[DOTSPLIT5:%.*]] = getelementptr inbounds [16 x i8], ptr [[P]], i64 [[TMP0]]
+; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [16 x i8], ptr [[P:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[DOTSPLIT5:%.*]] = getelementptr inbounds [16 x i8], ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT5]], i64 8
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [16 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP2]], i64 24
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0
-; CHECK-NEXT: store i64 [[TMP7]], ptr [[TMP1]], align 8
+; CHECK-NEXT: store i64 [[TMP7]], ptr [[DOTSPLIT]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
; CHECK: pred.store.if1:
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [16 x i8], ptr [[P]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [16 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP17]], i64 16
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 2
; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP9]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]
; CHECK: pred.store.continue2:
-; CHECK-NEXT: [[WIDE_VEC3:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_VEC3:%.*]] = load <4 x i64>, ptr [[DOTSPLIT]], align 8
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[WIDE_VEC3]], i64 0
-; CHECK-NEXT: store i64 [[TMP11]], ptr [[TMP2]], align 8
+; CHECK-NEXT: store i64 [[TMP11]], ptr [[TMP3]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[WIDE_VEC3]], i64 2
-; CHECK-NEXT: store i64 [[TMP12]], ptr [[TMP3]], align 8
+; CHECK-NEXT: store i64 [[TMP12]], ptr [[TMP6]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -226,9 +226,9 @@ define void @interleaved_with_cond_store_2(ptr %p, i64 %x, i64 %n) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [16 x i8], ptr [[P:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [16 x i8], ptr [[P]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [16 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP6]], i64 16
; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [16 x i8], ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 8
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
@@ -246,8 +246,8 @@ define void @interleaved_with_cond_store_2(ptr %p, i64 %x, i64 %n) {
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]
; CHECK: pred.store.if1:
-; CHECK-NEXT: [[DOTSPLIT4:%.*]] = getelementptr inbounds [16 x i8], ptr [[P]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT4]], i64 8
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr [16 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP14]], i64 24
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 2
; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP9]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index 1a97a87d2edfa..3e4eaa400c86e 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -867,34 +867,34 @@ define void @PR27626_0(ptr %p, i32 %z, i64 %n) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = or disjoint i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [8 x i8], ptr [[P:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP4]]
-; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 4
-; CHECK-NEXT: [[DOTSPLIT1:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT1]], i64 4
-; CHECK-NEXT: [[DOTSPLIT2:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT2]], i64 4
-; CHECK-NEXT: [[DOTSPLIT3:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP9]], i64 16
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP18]], i64 24
+; CHECK-NEXT: [[DOTSPLIT3:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT3]], i64 4
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP19]], i64 12
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP20]], i64 20
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP21]], i64 28
; CHECK-NEXT: store i32 [[Z:%.*]], ptr [[TMP5]], align 4
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP6]], align 4
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP7]], align 4
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 0
-; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP9]], align 4
+; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 2
; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP10]], align 4
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 4
; CHECK-NEXT: store i32 [[TMP15]], ptr [[TMP11]], align 4
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 6
-; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP12]], align 4
+; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP22]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
@@ -1051,35 +1051,35 @@ define void @PR27626_2(ptr %p, i64 %n, i32 %z) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = or disjoint i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [8 x i8], ptr [[P:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP10]], i64 16
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP20]], i64 24
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 -8
-; CHECK-NEXT: [[DOTSPLIT:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT]], i64 4
-; CHECK-NEXT: [[DOTSPLIT1:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT1]], i64 4
-; CHECK-NEXT: [[DOTSPLIT2:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT2]], i64 4
-; CHECK-NEXT: [[DOTSPLIT3:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[TMP4]]
+; CHECK-NEXT: [[DOTSPLIT3:%.*]] = getelementptr inbounds [8 x i8], ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[DOTSPLIT3]], i64 4
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP21]], i64 12
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP22]], i64 20
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP23]], i64 28
; CHECK-NEXT: store i32 [[Z:%.*]], ptr [[TMP5]], align 4
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP6]], align 4
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP7]], align 4
; CHECK-NEXT: store i32 [[Z]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 0
-; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP10]], align 4
+; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP13]], align 4
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 2
; CHECK-NEXT: store i32 [[TMP15]], ptr [[TMP11]], align 4
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 4
; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[WIDE_VEC]], i64 6
-; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP13]], align 4
+; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP24]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
@@ -1506,7 +1506,8 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[TMP18]], i64 2
; CHECK-NEXT: [[LOAD1:%.*]] = load i16, ptr [[GEP1]], align 4
; CHECK-NEXT: [[CONV1:%.*]] = sext i16 [[LOAD1]] to i32
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds [2 x i8], ptr [[A]], i64 [[IV2]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [2 x i8], ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[TMP20]], i64 4
; CHECK-NEXT: [[LOAD2]] = load i16, ptr [[GEP2]], align 4
; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[LOAD2]] to i32
; CHECK-NEXT: [[MUL01:%.*]] = mul nsw i32 [[CONV]], [[CONV1]]
diff --git a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
index 7843b91d8b40c..96096ae016300 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
@@ -73,11 +73,12 @@ define void @scalar_store(ptr %a, ptr %b, i64 %n) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 2
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP10]], i64 8
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [8 x i8], ptr [[A:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [8 x i8], ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP9]], i64 16
; CHECK-NEXT: store ptr [[TMP4]], ptr [[TMP6]], align 8
; CHECK-NEXT: store ptr [[TMP5]], ptr [[TMP7]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -133,12 +134,13 @@ define void @expansion(ptr %a, ptr %b, i64 %n) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [8 x i8], ptr [[B:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [8 x i8], ptr [[B]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [8 x i8], ptr [[A:%.*]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [8 x i8], ptr [[A]], i64 [[TMP3]]
-; CHECK-NEXT: store ptr [[TMP4]], ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP10]], i64 16
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [8 x i8], ptr [[A1:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr [8 x i8], ptr [[A1]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP9]], i64 16
+; CHECK-NEXT: store ptr [[TMP6]], ptr [[TMP11]], align 8
; CHECK-NEXT: store ptr [[TMP5]], ptr [[TMP7]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -151,8 +153,8 @@ define void @expansion(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[VAR0:%.*]] = getelementptr inbounds [8 x i8], ptr [[B]], i64 [[I]]
-; CHECK-NEXT: [[VAR3:%.*]] = getelementptr inbounds [8 x i8], ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[VAR0:%.*]] = getelementptr inbounds [8 x i8], ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[VAR3:%.*]] = getelementptr inbounds [8 x i8], ptr [[A1]], i64 [[I]]
; CHECK-NEXT: store ptr [[VAR0]], ptr [[VAR3]], align 8
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 2
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll
index c8c30c727b667..6b64de6914ddb 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll
@@ -379,11 +379,12 @@ define i32 @simple_chained_rdx(ptr noalias %a, ptr noalias %b, ptr noalias %cond
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
; CHECK-NEXT: br i1 [[TMP11]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
; CHECK: pred.load.if1:
-; CHECK-NEXT: [[TMP12:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 4
; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP14]], i64 1
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP22]], i64 4
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP17]], i64 1
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
@@ -393,11 +394,12 @@ define i32 @simple_chained_rdx(ptr noalias %a, ptr noalias %b, ptr noalias %cond
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
; CHECK-NEXT: br i1 [[TMP21]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
; CHECK: pred.load.if3:
-; CHECK-NEXT: [[TMP22:%.*]] = or disjoint i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP32]], i64 8
; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP24]], i64 2
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP51:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[TMP51]], i64 8
; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4
; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP27]], i64 2
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
@@ -407,11 +409,12 @@ define i32 @simple_chained_rdx(ptr noalias %a, ptr noalias %b, ptr noalias %cond
; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
; CHECK-NEXT: br i1 [[TMP31]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
; CHECK: pred.load.if5:
-; CHECK-NEXT: [[TMP32:%.*]] = or disjoint i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [4 x i8], ptr [[A]], i64 [[TMP32]]
+; CHECK-NEXT: [[TMP52:%.*]] = getelementptr [4 x i8], ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[TMP52]], i64 12
; CHECK-NEXT: [[TMP34:%.*]] = load i32, ptr [[TMP33]], align 4
; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP29]], i32 [[TMP34]], i64 3
-; CHECK-NEXT: [[TMP36:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[TMP32]]
+; CHECK-NEXT: [[TMP53:%.*]] = getelementptr [4 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[TMP53]], i64 12
; CHECK-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP36]], align 4
; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP37]], i64 3
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
@@ -507,9 +510,6 @@ define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocap
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ <i64 5, i64 -1, i64 -1, i64 -1>, [[VECTOR_PH]] ], [ [[PREDPHI15:%.*]], [[PRED_LOAD_CONTINUE14]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [8 x i8], ptr [[COND]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer
@@ -525,7 +525,8 @@ define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocap
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1
; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
; CHECK: pred.load.if1:
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [8 x i8], ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP47:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP47]], i64 8
; CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP11]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP9]], i64 [[TMP12]], i64 1
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
@@ -534,7 +535,8 @@ define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocap
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2
; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
; CHECK: pred.load.if3:
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [8 x i8], ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP48:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP48]], i64 16
; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 4
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP17]], i64 2
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
@@ -543,7 +545,8 @@ define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocap
; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3
; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
; CHECK: pred.load.if5:
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [8 x i8], ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP54:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP54]], i64 24
; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP21]], align 4
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP22]], i64 3
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
@@ -563,7 +566,8 @@ define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocap
; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP26]], i64 1
; CHECK-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
; CHECK: pred.load.if9:
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [8 x i8], ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP55:%.*]] = getelementptr [8 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[TMP55]], i64 8
; CHECK-NEXT: [[TMP34:%.*]] = load i64, ptr [[TMP33]], align 4
; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP34]], i64 1
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE10]]
@@ -572,7 +576,8 @@ define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocap
; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP26]], i64 2
; CHECK-NEXT: br i1 [[TMP37]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
; CHECK: pred.load.if11:
-; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds [8 x i8], ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP56:%.*]] = getelementptr [8 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[TMP56]], i64 16
; CHECK-NEXT: [[TMP39:%.*]] = load i64, ptr [[TMP38]], align 4
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x i64> [[TMP36]], i64 [[TMP39]], i64 2
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE12]]
@@ -581,7 +586,8 @@ define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocap
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[TMP26]], i64 3
; CHECK-NEXT: br i1 [[TMP42]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]]
; CHECK: pred.load.if13:
-; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds [8 x i8], ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP57:%.*]] = getelementptr [8 x i8], ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[TMP57]], i64 24
; CHECK-NEXT: [[TMP44:%.*]] = load i64, ptr [[TMP43]], align 4
; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x i64> [[TMP41]], i64 [[TMP44]], i64 3
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE14]]
@@ -806,9 +812,6 @@ define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float 2.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, [[VECTOR_PH]] ], [ [[PREDPHI15:%.*]], [[PRED_LOAD_CONTINUE14]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [4 x i8], ptr [[COND]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
@@ -824,7 +827,8 @@ define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1
; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
; CHECK: pred.load.if1:
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i8], ptr [[SRC1]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP53:%.*]] = getelementptr [4 x i8], ptr [[SRC1]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP53]], i64 4
; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP12]], i64 1
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
@@ -833,7 +837,8 @@ define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2
; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
; CHECK: pred.load.if3:
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [4 x i8], ptr [[SRC1]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP54:%.*]] = getelementptr [4 x i8], ptr [[SRC1]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP54]], i64 8
; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[TMP16]], align 4
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP17]], i64 2
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
@@ -842,7 +847,8 @@ define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3
; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]]
; CHECK: pred.load.if5:
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [4 x i8], ptr [[SRC1]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP55:%.*]] = getelementptr [4 x i8], ptr [[SRC1]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP55]], i64 12
; CHECK-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP22]], i64 3
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
@@ -863,7 +869,8 @@ define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP26]], i64 1
; CHECK-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]]
; CHECK: pred.load.if9:
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [4 x i8], ptr [[SRC2]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP56:%.*]] = getelementptr [4 x i8], ptr [[SRC2]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[TMP56]], i64 4
; CHECK-NEXT: [[TMP34:%.*]] = load float, ptr [[TMP33]], align 4
; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP31]], float [[TMP34]], i64 1
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE10]]
@@ -872,7 +879,8 @@ define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP26]], i64 2
; CHECK-NEXT: br i1 [[TMP37]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
; CHECK: pred.load.if11:
-; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds [4 x i8], ptr [[SRC2]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP57:%.*]] = getelementptr [4 x i8], ptr [[SRC2]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[TMP57]], i64 8
; CHECK-NEXT: [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x float> [[TMP36]], float [[TMP39]], i64 2
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE12]]
@@ -881,7 +889,8 @@ define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond,
; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[TMP26]], i64 3
; CHECK-NEXT: br i1 [[TMP42]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]]
; CHECK: pred.load.if13:
-; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds [4 x i8], ptr [[SRC2]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP58:%.*]] = getelementptr [4 x i8], ptr [[SRC2]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[TMP58]], i64 12
; CHECK-NEXT: [[TMP44:%.*]] = load float, ptr [[TMP43]], align 4
; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP44]], i64 3
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE14]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/Oz-and-forced-vectorize.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/Oz-and-forced-vectorize.ll
index 049d1baf5c615..e6cb6b39a3904 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/Oz-and-forced-vectorize.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/Oz-and-forced-vectorize.ll
@@ -39,10 +39,11 @@ define void @foo(ptr noalias nocapture %ptrA, ptr noalias nocapture readonly %pt
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP0]], i64 1
; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
; CHECK: [[PRED_STORE_IF3]]:
-; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[PTRB]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[PTRB]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 4
; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[PTRA]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[PTRA]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 4
; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = fmul float [[TMP10]], [[TMP12]]
; CHECK-NEXT: store float [[TMP13]], ptr [[TMP11]], align 4
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll
index bfed4b193562f..15b1e9253f225 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll
@@ -26,10 +26,11 @@ define void @partial_unroll_forced(i32 %N, ptr %src, ptr noalias %dst) {
; CHECK-NEXT: [[DST_IDX:%.*]] = getelementptr [16 x i8], ptr [[DST]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[ADD:%.*]] = fadd <8 x half> [[L]], [[L]]
; CHECK-NEXT: store <8 x half> [[ADD]], ptr [[DST_IDX]], align 16
-; CHECK-NEXT: [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[SRC_IDX_1:%.*]] = getelementptr [16 x i8], ptr [[SRC]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [16 x i8], ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[SRC_IDX_1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16
; CHECK-NEXT: [[L_1:%.*]] = load <8 x half>, ptr [[SRC_IDX_1]], align 16
-; CHECK-NEXT: [[DST_IDX_1:%.*]] = getelementptr [16 x i8], ptr [[DST]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [16 x i8], ptr [[DST]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[DST_IDX_1:%.*]] = getelementptr i8, ptr [[TMP2]], i64 16
; CHECK-NEXT: [[ADD_1:%.*]] = fadd <8 x half> [[L_1]], [[L_1]]
; CHECK-NEXT: store <8 x half> [[ADD_1]], ptr [[DST_IDX_1]], align 16
; CHECK-NEXT: [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
@@ -100,11 +101,12 @@ define void @cse_matching_load_from_previous_unrolled_iteration(i32 %N, ptr %src
; CHECK-NEXT: [[MUL:%.*]] = mul <2 x i32> [[L_4]], [[L_12]]
; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr [8 x i8], ptr [[DST]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store <2 x i32> [[MUL]], ptr [[GEP_DST]], align 8
-; CHECK-NEXT: [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[GEP_SRC_12_1:%.*]] = getelementptr [8 x i8], ptr [[SRC_12]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [8 x i8], ptr [[SRC_12]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[GEP_SRC_12_1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 8
; CHECK-NEXT: [[L_12_1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_12_1]], align 8
; CHECK-NEXT: [[MUL_1:%.*]] = mul <2 x i32> [[L_12]], [[L_12_1]]
-; CHECK-NEXT: [[GEP_DST_1:%.*]] = getelementptr [8 x i8], ptr [[DST]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [8 x i8], ptr [[DST]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[GEP_DST_1:%.*]] = getelementptr i8, ptr [[TMP2]], i64 8
; CHECK-NEXT: store <2 x i32> [[MUL_1]], ptr [[GEP_DST_1]], align 8
; CHECK-NEXT: [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
index fac74551a8add..0f01a2e43155d 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/indvars-vectorization.ll
@@ -33,10 +33,11 @@ define void @s172(i32 noundef %xa, i32 noundef %xb, ptr noundef %a, ptr noundef
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP13]], [[UMIN]]
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP14]], [[TMP0]]
; CHECK-NEXT: [[TMP16:%.*]] = shl i64 [[TMP15]], 2
-; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[TMP16]], 4
-; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP16]]
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[TMP17]], i64 4
; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
-; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP16]]
+; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[TMP18]], i64 4
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP6]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP5]], [[SCEVGEP4]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/instcombine-gep-widen.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/instcombine-gep-widen.ll
new file mode 100644
index 0000000000000..3fab64e2b9c07
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/instcombine-gep-widen.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -O2 -S | FileCheck %s
+; RUN: opt < %s -passes="default<O2>" -S | FileCheck %s
+
+; A bounds-checked little-endian i32 store lowered to four byte stores. The
+; index i+3 feeds both the bounds check and the last store, so it is multi-use.
+; InstCombine relaxing the single-use requirement on GEP(p, add(x, C)) for a
+; constant C lets the four stores share one base GEP with constant offsets, so
+; they widen to a single i32 store.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @store_le32(ptr %a, i64 %i, i64 %n, i32 %v) {
+; CHECK-LABEL: define void @store_le32(
+; CHECK-SAME: ptr nofree writeonly captures(none) [[A:%.*]], i64 [[I:%.*]], i64 [[N:%.*]], i32 [[V:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[I3:%.*]] = add nsw i64 [[I]], 3
+; CHECK-NEXT: [[OOB_NOT:%.*]] = icmp ult i64 [[I3]], [[N]]
+; CHECK-NEXT: br i1 [[OOB_NOT]], label %[[OK:.*]], label %[[TRAP:.*]]
+; CHECK: [[TRAP]]:
+; CHECK-NEXT: tail call void @llvm.trap()
+; CHECK-NEXT: unreachable
+; CHECK: [[OK]]:
+; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: store i32 [[V]], ptr [[P0]], align 1
+; CHECK-NEXT: ret void
+;
+entry:
+ %i3 = add nsw i64 %i, 3 ; <-- MULTI-USE temp: bounds check + %p3 GEP
+ %oob = icmp uge i64 %i3, %n ; use #1 of %i3 (the bounds check)
+ br i1 %oob, label %trap, label %ok
+
+trap:
+ tail call void @llvm.trap()
+ unreachable
+
+ok:
+ %p0 = getelementptr inbounds i8, ptr %a, i64 %i
+ %b0 = trunc i32 %v to i8
+ store i8 %b0, ptr %p0, align 1
+ %i1 = add nsw i64 %i, 1 ; single-use: GEP fold works even pre-patch
+ %p1 = getelementptr inbounds i8, ptr %a, i64 %i1
+ %v1 = lshr i32 %v, 8
+ %b1 = trunc i32 %v1 to i8
+ store i8 %b1, ptr %p1, align 1
+ %i2 = add nsw i64 %i, 2
+ %p2 = getelementptr inbounds i8, ptr %a, i64 %i2
+ %v2 = lshr i32 %v, 16
+ %b2 = trunc i32 %v2 to i8
+ store i8 %b2, ptr %p2, align 1
+ %p3 = getelementptr inbounds i8, ptr %a, i64 %i3 ; use #2 of %i3
+ %v3 = lshr i32 %v, 24
+ %b3 = trunc i32 %v3 to i8
+ store i8 %b3, ptr %p3, align 1
+ ret void
+}
+
+declare void @llvm.trap()
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
index 79531da1cb16e..4b5a9dd18e9f1 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
@@ -12,17 +12,19 @@ define void @same_op2(ptr noalias noundef %a, ptr noundef %b, ptr noundef %c) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP10]], i64 32
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_VEC15:%.*]] = load <8 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 32
; CHECK-NEXT: [[WIDE_VEC18:%.*]] = load <8 x float>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[WIDE_VEC21:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP12]], i64 32
; CHECK-NEXT: [[WIDE_VEC24:%.*]] = load <8 x float>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[WIDE_VEC27:%.*]] = load <8 x float>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP7:%.*]] = fmul fast <8 x float> [[WIDE_VEC18]], [[WIDE_VEC]]
@@ -124,13 +126,14 @@ define void @same_op2_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 32
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x float>, ptr [[TMP4]], align 4
; CHECK-NEXT: [[WIDE_VEC13:%.*]] = load <8 x float>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP12]], i64 32
; CHECK-NEXT: [[WIDE_VEC16:%.*]] = load <8 x float>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[WIDE_VEC19:%.*]] = load <8 x float>, ptr [[TMP7]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <8 x float> [[WIDE_VEC]], [[TMP1]]
@@ -417,21 +420,24 @@ define void @same_op4(ptr noalias noundef %a, ptr noundef %b, ptr noundef %c) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
-; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
-; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 16
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 32
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP23]], align 4
; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x float>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <4 x float>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP25]], i64 16
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP26]], i64 32
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP27]], i64 48
; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x float>, ptr [[TMP8]], align 4
; CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <4 x float>, ptr [[TMP9]], align 4
@@ -441,9 +447,12 @@ define void @same_op4(ptr noalias noundef %a, ptr noundef %b, ptr noundef %c) {
; CHECK-NEXT: [[TMP13:%.*]] = fmul fast <4 x float> [[WIDE_LOAD19]], [[WIDE_LOAD15]]
; CHECK-NEXT: [[TMP14:%.*]] = fmul fast <4 x float> [[WIDE_LOAD20]], [[WIDE_LOAD16]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP28]], i64 16
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP29]], i64 32
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP24]], i64 48
; CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <4 x float>, ptr [[TMP16]], align 4
; CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <4 x float>, ptr [[TMP17]], align 4
@@ -548,13 +557,13 @@ define void @same_op4_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
-; CHECK-NEXT: [[TMP12:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP12]], i64 32
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP21]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP20]], align 4
; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x float>, ptr [[TMP6]], align 4
@@ -564,9 +573,12 @@ define void @same_op4_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK-NEXT: [[TMP10:%.*]] = fmul fast <4 x float> [[WIDE_LOAD13]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP11:%.*]] = fmul fast <4 x float> [[WIDE_LOAD14]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP22]], i64 16
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP23]], i64 32
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP24]], i64 48
; CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <4 x float>, ptr [[TMP13]], align 4
; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <4 x float>, ptr [[TMP14]], align 4
@@ -672,10 +684,12 @@ define void @same_op6(ptr noalias noundef %a, ptr noundef %b, ptr noundef %c) {
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x float>, ptr [[ARRAYIDX9]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <4 x float> [[TMP3]], [[TMP2]]
; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[ARRAYIDX9]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
-; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[TMP5]]
-; CHECK-NEXT: [[ARRAYIDX6_4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP5]]
-; CHECK-NEXT: [[ARRAYIDX9_4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 16
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX6_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 16
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX9_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP12]], i64 16
; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[ARRAYIDX_4]], align 4
; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[ARRAYIDX6_4]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x float> [[TMP7]], [[TMP6]]
@@ -782,9 +796,10 @@ define void @same_op6_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK-NEXT: [[TMP7:%.*]] = load <4 x float>, ptr [[ARRAYIDX7]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = fadd fast <4 x float> [[TMP7]], [[TMP6]]
; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[ARRAYIDX7]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
-; CHECK-NEXT: [[ARRAYIDX4_4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP9]]
-; CHECK-NEXT: [[ARRAYIDX7_4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX4_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 16
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX7_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 16
; CHECK-NEXT: [[TMP10:%.*]] = load <2 x float>, ptr [[ARRAYIDX4_4]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = fmul fast <2 x float> [[TMP10]], [[TMP4]]
; CHECK-NEXT: [[TMP12:%.*]] = load <2 x float>, ptr [[ARRAYIDX7_4]], align 4
@@ -883,10 +898,12 @@ define void @same_op8(ptr noalias noundef %a, ptr noundef %b, ptr noundef %c) {
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x float>, ptr [[ARRAYIDX9]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <4 x float> [[TMP3]], [[TMP2]]
; CHECK-NEXT: store <4 x float> [[TMP4]], ptr [[ARRAYIDX9]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = or disjoint i64 [[INDVARS_IV]], 4
-; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[TMP5]]
-; CHECK-NEXT: [[ARRAYIDX6_4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[TMP5]]
-; CHECK-NEXT: [[ARRAYIDX9_4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 16
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX6_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 16
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX9_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP12]], i64 16
; CHECK-NEXT: [[TMP6:%.*]] = load <4 x float>, ptr [[ARRAYIDX_4]], align 4
; CHECK-NEXT: [[TMP7:%.*]] = load <4 x float>, ptr [[ARRAYIDX6_4]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP7]], [[TMP6]]
@@ -979,22 +996,30 @@ define void @same_op8_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
; CHECK-SAME: ptr noalias nofree noundef captures(none) [[A:%.*]], ptr nofree noundef readonly captures(none) [[B:%.*]], ptr nofree noundef readonly captures(none) [[C:%.*]]) local_unnamed_addr #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[C]], align 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <16 x float>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_VEC19:%.*]] = load <16 x float>, ptr [[TMP6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <16 x float> [[WIDE_VEC]], [[TMP1]]
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = fadd fast <16 x float> [[WIDE_VEC19]], [[TMP4]]
-; CHECK-NEXT: store <16 x float> [[INTERLEAVED_VEC]], ptr [[TMP6]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 144
-; CHECK-NEXT: br i1 [[TMP25]], label %[[FOR_END11:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x float>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <4 x float> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[TMP13:%.*]] = load <4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <4 x float> [[TMP13]], [[TMP4]]
+; CHECK-NEXT: store <4 x float> [[TMP14]], ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[ARRAYIDX4_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 16
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[ARRAYIDX7_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16
+; CHECK-NEXT: [[TMP9:%.*]] = load <4 x float>, ptr [[ARRAYIDX4_4]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = fmul fast <4 x float> [[TMP9]], [[TMP2]]
+; CHECK-NEXT: [[TMP11:%.*]] = load <4 x float>, ptr [[ARRAYIDX7_4]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = fadd fast <4 x float> [[TMP11]], [[TMP10]]
+; CHECK-NEXT: store <4 x float> [[TMP12]], ptr [[ARRAYIDX7_4]], align 4
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[OFFSET_IDX]], 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp samesign ult i64 [[OFFSET_IDX]], 1144
+; CHECK-NEXT: br i1 [[CMP]], label %[[VECTOR_BODY]], label %[[FOR_END11:.*]]
; CHECK: [[FOR_END11]]:
; CHECK-NEXT: ret void
;
@@ -1099,11 +1124,10 @@ define void @saxpy_5(i64 %n, float %a, ptr readonly %x, ptr noalias %y) {
; CHECK-NEXT: [[TMP14:%.*]] = load <4 x float>, ptr [[YGEP1]], align 4
; CHECK-NEXT: [[TMP15:%.*]] = fadd fast <4 x float> [[TMP14]], [[TMP13]]
; CHECK-NEXT: store <4 x float> [[TMP15]], ptr [[YGEP1]], align 4
-; CHECK-NEXT: [[I5:%.*]] = add nuw nsw i64 [[I1]], 4
-; CHECK-NEXT: [[XGEP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[X]], i64 [[I5]]
+; CHECK-NEXT: [[XGEP5:%.*]] = getelementptr inbounds nuw i8, ptr [[XGEP1]], i64 16
; CHECK-NEXT: [[X5:%.*]] = load float, ptr [[XGEP5]], align 4
; CHECK-NEXT: [[AX5:%.*]] = fmul fast float [[X5]], [[A]]
-; CHECK-NEXT: [[YGEP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[Y]], i64 [[I5]]
+; CHECK-NEXT: [[YGEP5:%.*]] = getelementptr inbounds nuw i8, ptr [[YGEP1]], i64 16
; CHECK-NEXT: [[Y5:%.*]] = load float, ptr [[YGEP5]], align 4
; CHECK-NEXT: [[AXPY5:%.*]] = fadd fast float [[Y5]], [[AX5]]
; CHECK-NEXT: store float [[AXPY5]], ptr [[YGEP5]], align 4
@@ -1174,5 +1198,4 @@ exit:
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
;.
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
index b43d12ecf8b3b..472bffb2ee4dd 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
@@ -88,9 +88,10 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
; CHECK-NEXT: br i1 [[CMP210_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: for.cond1.preheader.preheader:
; CHECK-NEXT: [[TMP0:%.*]] = shl nuw nsw i64 [[CONV6]], 3
-; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 360
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP1]]
-; CHECK-NEXT: [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP1]], i64 360
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[SCEVGEP20:%.*]] = getelementptr i8, ptr [[TMP4]], i64 360
; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[I]], 225
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP2]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[CONV6]]
@@ -151,24 +152,25 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
; CHECK-NEXT: br label [[VECTOR_BODY_1:%.*]]
; CHECK: vector.body.1:
; CHECK-NEXT: [[INDEX_1:%.*]] = phi i64 [ 0, [[VECTOR_PH_1]] ], [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY_1]] ]
-; CHECK-NEXT: [[TMP33:%.*]] = add nuw nsw i64 [[INDEX_1]], 15
-; CHECK-NEXT: [[TMP47:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP33]]
-; CHECK-NEXT: [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP47]], i64 16
-; CHECK-NEXT: [[WIDE_LOAD_1:%.*]] = load <2 x double>, ptr [[TMP47]], align 8, !alias.scope [[META0]]
-; CHECK-NEXT: [[WIDE_LOAD21_1:%.*]] = load <2 x double>, ptr [[TMP48]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDEX_1]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP27]], i64 120
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP27]], i64 136
+; CHECK-NEXT: [[WIDE_LOAD_1:%.*]] = load <2 x double>, ptr [[TMP28]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[WIDE_LOAD21_1:%.*]] = load <2 x double>, ptr [[TMP29]], align 8, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP49:%.*]] = load double, ptr [[TMP32]], align 8, !alias.scope [[META3]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT22_1:%.*]] = insertelement <2 x double> poison, double [[TMP49]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT23_1:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT22_1]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP50:%.*]] = fmul <2 x double> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT23_1]]
; CHECK-NEXT: [[TMP51:%.*]] = fmul <2 x double> [[WIDE_LOAD21_1]], [[BROADCAST_SPLAT23_1]]
-; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP33]]
-; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP52]], i64 16
-; CHECK-NEXT: [[WIDE_LOAD24_1:%.*]] = load <2 x double>, ptr [[TMP52]], align 8, !alias.scope [[META5]], !noalias [[META0]]
-; CHECK-NEXT: [[WIDE_LOAD25_1:%.*]] = load <2 x double>, ptr [[TMP53]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDEX_1]]
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP30]], i64 120
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP30]], i64 136
+; CHECK-NEXT: [[WIDE_LOAD24_1:%.*]] = load <2 x double>, ptr [[TMP34]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: [[WIDE_LOAD25_1:%.*]] = load <2 x double>, ptr [[TMP35]], align 8, !alias.scope [[META5]], !noalias [[META0]]
; CHECK-NEXT: [[TMP54:%.*]] = fsub <2 x double> [[WIDE_LOAD24_1]], [[TMP50]]
; CHECK-NEXT: [[TMP55:%.*]] = fsub <2 x double> [[WIDE_LOAD25_1]], [[TMP51]]
-; CHECK-NEXT: store <2 x double> [[TMP54]], ptr [[TMP52]], align 8, !alias.scope [[META5]], !noalias [[META0]]
-; CHECK-NEXT: store <2 x double> [[TMP55]], ptr [[TMP53]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: store <2 x double> [[TMP54]], ptr [[TMP34]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: store <2 x double> [[TMP55]], ptr [[TMP35]], align 8, !alias.scope [[META5]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT_1]] = add nuw i64 [[INDEX_1]], 4
; CHECK-NEXT: [[TMP56:%.*]] = icmp eq i64 [[INDEX_NEXT_1]], [[N_VEC_1]]
; CHECK-NEXT: br i1 [[TMP56]], label [[MIDDLE_BLOCK_1:%.*]], label [[VECTOR_BODY_1]], !llvm.loop [[LOOP7]]
@@ -180,14 +182,15 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
; CHECK-NEXT: br label [[FOR_BODY4_US_1:%.*]]
; CHECK: for.body4.1:
; CHECK-NEXT: [[INDVARS_IV_1:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_BODY4_US_1]] ], [ [[INDVARS_IV_PH_1]], [[FOR_BODY4_US_PREHEADER_1]] ]
-; CHECK-NEXT: [[TMP57:%.*]] = add nuw nsw i64 [[INDVARS_IV_1]], 15
; CHECK-NEXT: [[TMP58:%.*]] = icmp samesign ult i64 [[INDVARS_IV_1]], 210
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP58]])
-; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP57]]
+; CHECK-NEXT: [[TMP36:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDVARS_IV_1]]
+; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP36]], i64 120
; CHECK-NEXT: [[MATRIXEXT_US_1:%.*]] = load double, ptr [[TMP59]], align 8
; CHECK-NEXT: [[MATRIXEXT8_US_1:%.*]] = load double, ptr [[TMP32]], align 8
; CHECK-NEXT: [[MUL_US_1:%.*]] = fmul double [[MATRIXEXT_US_1]], [[MATRIXEXT8_US_1]]
-; CHECK-NEXT: [[TMP60:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP57]]
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDVARS_IV_1]]
+; CHECK-NEXT: [[TMP60:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP33]], i64 120
; CHECK-NEXT: [[MATRIXEXT11_US_1:%.*]] = load double, ptr [[TMP60]], align 8
; CHECK-NEXT: [[SUB_US_1:%.*]] = fsub double [[MATRIXEXT11_US_1]], [[MUL_US_1]]
; CHECK-NEXT: store double [[SUB_US_1]], ptr [[TMP60]], align 8
@@ -211,24 +214,25 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
; CHECK-NEXT: br label [[VECTOR_BODY_2:%.*]]
; CHECK: vector.body.2:
; CHECK-NEXT: [[INDEX_2:%.*]] = phi i64 [ 0, [[VECTOR_PH_2]] ], [ [[INDEX_NEXT_2:%.*]], [[VECTOR_BODY_2]] ]
-; CHECK-NEXT: [[TMP64:%.*]] = add nuw nsw i64 [[INDEX_2]], 30
-; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP64]]
-; CHECK-NEXT: [[TMP79:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP78]], i64 16
-; CHECK-NEXT: [[WIDE_LOAD_2:%.*]] = load <2 x double>, ptr [[TMP78]], align 8, !alias.scope [[META0]]
-; CHECK-NEXT: [[WIDE_LOAD21_2:%.*]] = load <2 x double>, ptr [[TMP79]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDEX_2]]
+; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP38]], i64 240
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP38]], i64 256
+; CHECK-NEXT: [[WIDE_LOAD_2:%.*]] = load <2 x double>, ptr [[TMP39]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[WIDE_LOAD21_2:%.*]] = load <2 x double>, ptr [[TMP40]], align 8, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP80:%.*]] = load double, ptr [[TMP63]], align 8, !alias.scope [[META3]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT22_2:%.*]] = insertelement <2 x double> poison, double [[TMP80]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT23_2:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT22_2]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP81:%.*]] = fmul <2 x double> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT23_2]]
; CHECK-NEXT: [[TMP82:%.*]] = fmul <2 x double> [[WIDE_LOAD21_2]], [[BROADCAST_SPLAT23_2]]
-; CHECK-NEXT: [[TMP83:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP64]]
-; CHECK-NEXT: [[TMP84:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP83]], i64 16
-; CHECK-NEXT: [[WIDE_LOAD24_2:%.*]] = load <2 x double>, ptr [[TMP83]], align 8, !alias.scope [[META5]], !noalias [[META0]]
-; CHECK-NEXT: [[WIDE_LOAD25_2:%.*]] = load <2 x double>, ptr [[TMP84]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDEX_2]]
+; CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP44]], i64 240
+; CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP44]], i64 256
+; CHECK-NEXT: [[WIDE_LOAD24_2:%.*]] = load <2 x double>, ptr [[TMP45]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: [[WIDE_LOAD25_2:%.*]] = load <2 x double>, ptr [[TMP46]], align 8, !alias.scope [[META5]], !noalias [[META0]]
; CHECK-NEXT: [[TMP85:%.*]] = fsub <2 x double> [[WIDE_LOAD24_2]], [[TMP81]]
; CHECK-NEXT: [[TMP86:%.*]] = fsub <2 x double> [[WIDE_LOAD25_2]], [[TMP82]]
-; CHECK-NEXT: store <2 x double> [[TMP85]], ptr [[TMP83]], align 8, !alias.scope [[META5]], !noalias [[META0]]
-; CHECK-NEXT: store <2 x double> [[TMP86]], ptr [[TMP84]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: store <2 x double> [[TMP85]], ptr [[TMP45]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: store <2 x double> [[TMP86]], ptr [[TMP46]], align 8, !alias.scope [[META5]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT_2]] = add nuw i64 [[INDEX_2]], 4
; CHECK-NEXT: [[TMP87:%.*]] = icmp eq i64 [[INDEX_NEXT_2]], [[N_VEC_2]]
; CHECK-NEXT: br i1 [[TMP87]], label [[MIDDLE_BLOCK_2:%.*]], label [[VECTOR_BODY_2]], !llvm.loop [[LOOP7]]
@@ -240,14 +244,15 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
; CHECK-NEXT: br label [[FOR_BODY4_US_2:%.*]]
; CHECK: for.body4.2:
; CHECK-NEXT: [[INDVARS_IV_2:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_2:%.*]], [[FOR_BODY4_US_2]] ], [ [[INDVARS_IV_PH_2]], [[FOR_BODY4_US_PREHEADER_2]] ]
-; CHECK-NEXT: [[TMP88:%.*]] = add nuw nsw i64 [[INDVARS_IV_2]], 30
; CHECK-NEXT: [[TMP89:%.*]] = icmp samesign ult i64 [[INDVARS_IV_2]], 195
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP89]])
-; CHECK-NEXT: [[TMP90:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP88]]
+; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDVARS_IV_2]]
+; CHECK-NEXT: [[TMP90:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP52]], i64 240
; CHECK-NEXT: [[MATRIXEXT_US_2:%.*]] = load double, ptr [[TMP90]], align 8
; CHECK-NEXT: [[MATRIXEXT8_US_2:%.*]] = load double, ptr [[TMP63]], align 8
; CHECK-NEXT: [[MUL_US_2:%.*]] = fmul double [[MATRIXEXT_US_2]], [[MATRIXEXT8_US_2]]
-; CHECK-NEXT: [[TMP91:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP88]]
+; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDVARS_IV_2]]
+; CHECK-NEXT: [[TMP91:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP53]], i64 240
; CHECK-NEXT: [[MATRIXEXT11_US_2:%.*]] = load double, ptr [[TMP91]], align 8
; CHECK-NEXT: [[SUB_US_2:%.*]] = fsub double [[MATRIXEXT11_US_2]], [[MUL_US_2]]
; CHECK-NEXT: store double [[SUB_US_2]], ptr [[TMP91]], align 8
@@ -271,24 +276,25 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
; CHECK-NEXT: br label [[VECTOR_BODY_3:%.*]]
; CHECK: vector.body.3:
; CHECK-NEXT: [[INDEX_3:%.*]] = phi i64 [ 0, [[VECTOR_PH_3]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY_3]] ]
-; CHECK-NEXT: [[TMP95:%.*]] = add nuw nsw i64 [[INDEX_3]], 45
-; CHECK-NEXT: [[TMP109:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP95]]
-; CHECK-NEXT: [[TMP110:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP109]], i64 16
-; CHECK-NEXT: [[WIDE_LOAD_3:%.*]] = load <2 x double>, ptr [[TMP109]], align 8, !alias.scope [[META0]]
-; CHECK-NEXT: [[WIDE_LOAD21_3:%.*]] = load <2 x double>, ptr [[TMP110]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDEX_3]]
+; CHECK-NEXT: [[TMP72:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP67]], i64 360
+; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP67]], i64 376
+; CHECK-NEXT: [[WIDE_LOAD_3:%.*]] = load <2 x double>, ptr [[TMP72]], align 8, !alias.scope [[META0]]
+; CHECK-NEXT: [[WIDE_LOAD21_3:%.*]] = load <2 x double>, ptr [[TMP74]], align 8, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP111:%.*]] = load double, ptr [[TMP94]], align 8, !alias.scope [[META3]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT22_3:%.*]] = insertelement <2 x double> poison, double [[TMP111]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT23_3:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT22_3]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP112:%.*]] = fmul <2 x double> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT23_3]]
; CHECK-NEXT: [[TMP113:%.*]] = fmul <2 x double> [[WIDE_LOAD21_3]], [[BROADCAST_SPLAT23_3]]
-; CHECK-NEXT: [[TMP114:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP95]]
-; CHECK-NEXT: [[TMP115:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP114]], i64 16
-; CHECK-NEXT: [[WIDE_LOAD24_3:%.*]] = load <2 x double>, ptr [[TMP114]], align 8, !alias.scope [[META5]], !noalias [[META0]]
-; CHECK-NEXT: [[WIDE_LOAD25_3:%.*]] = load <2 x double>, ptr [[TMP115]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: [[TMP64:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDEX_3]]
+; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP64]], i64 360
+; CHECK-NEXT: [[TMP66:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP64]], i64 376
+; CHECK-NEXT: [[WIDE_LOAD24_3:%.*]] = load <2 x double>, ptr [[TMP65]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: [[WIDE_LOAD25_3:%.*]] = load <2 x double>, ptr [[TMP66]], align 8, !alias.scope [[META5]], !noalias [[META0]]
; CHECK-NEXT: [[TMP116:%.*]] = fsub <2 x double> [[WIDE_LOAD24_3]], [[TMP112]]
; CHECK-NEXT: [[TMP117:%.*]] = fsub <2 x double> [[WIDE_LOAD25_3]], [[TMP113]]
-; CHECK-NEXT: store <2 x double> [[TMP116]], ptr [[TMP114]], align 8, !alias.scope [[META5]], !noalias [[META0]]
-; CHECK-NEXT: store <2 x double> [[TMP117]], ptr [[TMP115]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: store <2 x double> [[TMP116]], ptr [[TMP65]], align 8, !alias.scope [[META5]], !noalias [[META0]]
+; CHECK-NEXT: store <2 x double> [[TMP117]], ptr [[TMP66]], align 8, !alias.scope [[META5]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT_3]] = add nuw i64 [[INDEX_3]], 4
; CHECK-NEXT: [[TMP118:%.*]] = icmp eq i64 [[INDEX_NEXT_3]], [[N_VEC_3]]
; CHECK-NEXT: br i1 [[TMP118]], label [[MIDDLE_BLOCK_3:%.*]], label [[VECTOR_BODY_3]], !llvm.loop [[LOOP7]]
@@ -300,14 +306,15 @@ define void @matrix_extract_insert_loop(i32 %i, ptr nonnull align 8 dereferencea
; CHECK-NEXT: br label [[FOR_BODY4_US_3:%.*]]
; CHECK: for.body4.3:
; CHECK-NEXT: [[INDVARS_IV_3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_BODY4_US_3]] ], [ [[INDVARS_IV_PH_3]], [[FOR_BODY4_US_PREHEADER_3]] ]
-; CHECK-NEXT: [[TMP119:%.*]] = add nuw nsw i64 [[INDVARS_IV_3]], 45
; CHECK-NEXT: [[TMP120:%.*]] = icmp samesign ult i64 [[INDVARS_IV_3]], 180
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP120]])
-; CHECK-NEXT: [[TMP121:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[TMP119]]
+; CHECK-NEXT: [[TMP71:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[A]], i64 [[INDVARS_IV_3]]
+; CHECK-NEXT: [[TMP121:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP71]], i64 360
; CHECK-NEXT: [[MATRIXEXT_US_3:%.*]] = load double, ptr [[TMP121]], align 8
; CHECK-NEXT: [[MATRIXEXT8_US_3:%.*]] = load double, ptr [[TMP94]], align 8
; CHECK-NEXT: [[MUL_US_3:%.*]] = fmul double [[MATRIXEXT_US_3]], [[MATRIXEXT8_US_3]]
-; CHECK-NEXT: [[TMP122:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[TMP119]]
+; CHECK-NEXT: [[TMP73:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[B]], i64 [[INDVARS_IV_3]]
+; CHECK-NEXT: [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP73]], i64 360
; CHECK-NEXT: [[MATRIXEXT11_US_3:%.*]] = load double, ptr [[TMP122]], align 8
; CHECK-NEXT: [[SUB_US_3:%.*]] = fsub double [[MATRIXEXT11_US_3]], [[MUL_US_3]]
; CHECK-NEXT: store double [[SUB_US_3]], ptr [[TMP122]], align 8
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
index c520bb78c20c2..10071dfd2e210 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll
@@ -133,47 +133,54 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 {
; CHECK-NEXT: [[TMP30:%.*]] = fmul fast double [[T0]], [[TMP22]]
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store double [[TMP30]], ptr [[ARRAYIDX2]], align 8, !tbaa [[DOUBLE_TBAA3]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP44]], i64 8
; CHECK-NEXT: [[T0_1:%.*]] = load double, ptr [[ARRAYIDX_1]], align 8, !tbaa [[DOUBLE_TBAA3]]
; CHECK-NEXT: [[TMP31:%.*]] = fmul fast double [[T0_1]], [[TMP23]]
-; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: [[TMP47:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP47]], i64 8
; CHECK-NEXT: store double [[TMP31]], ptr [[ARRAYIDX2_1]], align 8, !tbaa [[DOUBLE_TBAA3]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
-; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT: [[TMP50:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP50]], i64 16
; CHECK-NEXT: [[T0_2:%.*]] = load double, ptr [[ARRAYIDX_2]], align 8, !tbaa [[DOUBLE_TBAA3]]
; CHECK-NEXT: [[TMP32:%.*]] = fmul fast double [[T0_2]], [[TMP24]]
-; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP53]], i64 16
; CHECK-NEXT: store double [[TMP32]], ptr [[ARRAYIDX2_2]], align 8, !tbaa [[DOUBLE_TBAA3]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
-; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP56]], i64 24
; CHECK-NEXT: [[T0_3:%.*]] = load double, ptr [[ARRAYIDX_3]], align 8, !tbaa [[DOUBLE_TBAA3]]
; CHECK-NEXT: [[TMP33:%.*]] = fmul fast double [[T0_3]], [[TMP25]]
-; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP45]], i64 24
; CHECK-NEXT: store double [[TMP33]], ptr [[ARRAYIDX2_3]], align 8, !tbaa [[DOUBLE_TBAA3]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
-; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV_NEXT_3]]
+; CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP46]], i64 32
; CHECK-NEXT: [[T0_4:%.*]] = load double, ptr [[ARRAYIDX_4]], align 8, !tbaa [[DOUBLE_TBAA3]]
; CHECK-NEXT: [[TMP34:%.*]] = fmul fast double [[T0_4]], [[TMP26]]
-; CHECK-NEXT: [[ARRAYIDX2_4:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV_NEXT_3]]
+; CHECK-NEXT: [[TMP48:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP48]], i64 32
; CHECK-NEXT: store double [[TMP34]], ptr [[ARRAYIDX2_4]], align 8, !tbaa [[DOUBLE_TBAA3]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_4:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV_NEXT_4]]
+; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP49]], i64 40
; CHECK-NEXT: [[T0_5:%.*]] = load double, ptr [[ARRAYIDX_5]], align 8, !tbaa [[DOUBLE_TBAA3]]
; CHECK-NEXT: [[TMP35:%.*]] = fmul fast double [[T0_5]], [[TMP27]]
-; CHECK-NEXT: [[ARRAYIDX2_5:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV_NEXT_4]]
+; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP51]], i64 40
; CHECK-NEXT: store double [[TMP35]], ptr [[ARRAYIDX2_5]], align 8, !tbaa [[DOUBLE_TBAA3]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 6
-; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV_NEXT_5]]
+; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP52]], i64 48
; CHECK-NEXT: [[T0_6:%.*]] = load double, ptr [[ARRAYIDX_6]], align 8, !tbaa [[DOUBLE_TBAA3]]
; CHECK-NEXT: [[TMP36:%.*]] = fmul fast double [[T0_6]], [[TMP28]]
-; CHECK-NEXT: [[ARRAYIDX2_6:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV_NEXT_5]]
+; CHECK-NEXT: [[TMP54:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP54]], i64 48
; CHECK-NEXT: store double [[TMP36]], ptr [[ARRAYIDX2_6]], align 8, !tbaa [[DOUBLE_TBAA3]]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_6:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 7
-; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV_NEXT_6]]
+; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Y]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP55]], i64 56
; CHECK-NEXT: [[T0_7:%.*]] = load double, ptr [[ARRAYIDX_7]], align 8, !tbaa [[DOUBLE_TBAA3]]
; CHECK-NEXT: [[TMP37:%.*]] = fmul fast double [[T0_7]], [[TMP29]]
-; CHECK-NEXT: [[ARRAYIDX2_7:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV_NEXT_6]]
+; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[ARRAYIDX2_7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP57]], i64 56
; CHECK-NEXT: store double [[TMP37]], ptr [[ARRAYIDX2_7]], align 8, !tbaa [[DOUBLE_TBAA3]]
; CHECK-NEXT: [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8
; CHECK-NEXT: [[EXITCOND_NOT_7:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_7]], [[WIDE_TRIP_COUNT]]
More information about the cfe-commits
mailing list