[llvm] [LV] Add vplan folds for urem(X, PowerOf2) -> and(X, PowerOf2 - 1) (PR #212198)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 05:21:56 PDT 2026
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/212198
>From c75b9f4ae6b411f6bc8014c7ed62f0d59864bd3b Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Mon, 27 Jul 2026 08:35:52 +0000
Subject: [PATCH] [LV][NFC] Add vplan folds for urem(X, PowerOf2) -> and(X,
PowerOf2 - 1)
In this PR I've added support for the vplan fold:
urem(X, Y) -> and(X, Y - 1)
when Y is a power of 2. This should reduce the cost of the urem and
ensure the vplan is accurately costed. Such a change would normally
affect over 300 test files due to this being a common pattern in
the vector preheader. For now, I've limited the scope to only
simplifying occurences that are not in the vector preheader. In a
follow-on PR I will extend this to add support for
sub(X, urem(X, Y)) -> and(X, -Y)
as well permitting folds in the preheader.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 +++
.../LoopVectorize/AArch64/bounded-load.ll | 30 ++++----
.../AArch64/discarded-interleave-group.ll | 4 +-
.../LoopVectorize/RISCV/bounded-load.ll | 2 +-
.../LoopVectorize/VPlan/vplan-fold-dbg.ll | 67 +++++++++++++++++
.../LoopVectorize/X86/uniform_mem_op.ll | 8 +-
.../LoopVectorize/bounded-load-predicated.ll | 74 +++++++++----------
.../LoopVectorize/bounded-load-user-ic.ll | 4 +-
.../LoopVectorize/bounded-load-vf-ranges.ll | 6 +-
.../Transforms/LoopVectorize/bounded-store.ll | 2 +-
.../test/Transforms/LoopVectorize/debugloc.ll | 46 ------------
.../runtime-check-small-bounded-ranges.ll | 22 +++---
12 files changed, 152 insertions(+), 122 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/vplan-fold-dbg.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 83e23449df5f5..541a3da855992 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1366,6 +1366,15 @@ static void simplifyRecipe(VPSingleDefRecipe *Def) {
}
const APInt *APC;
+ // TODO: Enable optimizations in the vector preheader in a follow-up PR.
+ // This check currently means we only simplify before region dissolution.
+ VPBasicBlock *Preheader = Plan->getVectorPreheader();
+ if (CanCreateNewRecipe && Preheader && Def->getParent() != Preheader &&
+ match(Def, m_URem(m_VPValue(X), m_APInt(APC))) && APC->isPowerOf2()) {
+ return Def->replaceAllUsesWith(Builder.createAnd(
+ X, Plan->getConstantInt(*APC - 1), Def->getDebugLoc()));
+ }
+
if (CanCreateNewRecipe && match(Def, m_c_Mul(m_VPValue(A), m_APInt(APC))) &&
APC->isPowerOf2()) {
auto *MulR = cast<VPRecipeWithIRFlags>(Def);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll b/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
index 2e0f55236f65b..bb1bcf18b0a41 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/bounded-load.ll
@@ -28,7 +28,7 @@ define i32 @bounded_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 -3
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 -7
@@ -69,7 +69,7 @@ define i32 @bounded_load_reduction_bound2(ptr %A, i32 %N) {
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX15:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI16:%.*]] = phi <4 x i32> [ [[TMP16]], %[[VEC_EPILOG_PH]] ], [ [[TMP20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = urem i32 [[INDEX15]], 2
+; CHECK-NEXT: [[TMP17:%.*]] = and i32 [[INDEX15]], 1
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP17]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i64 -3
; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <4 x i32>, ptr [[TMP19]], align 4
@@ -142,7 +142,7 @@ define i32 @bounded_load_reduction_bound4(ptr %A, i32 %N) {
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 4
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 8
@@ -178,7 +178,7 @@ define i32 @bounded_load_reduction_bound4(ptr %A, i32 %N) {
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX12:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI13:%.*]] = phi <4 x i32> [ [[TMP13]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = urem i32 [[INDEX12]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = and i32 [[INDEX12]], 3
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP14]]
; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, ptr [[TMP15]], align 4
; CHECK-NEXT: [[TMP16]] = add <4 x i32> [[VEC_PHI13]], [[WIDE_LOAD14]]
@@ -247,13 +247,13 @@ define i16 @bounded_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <8 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[TMP3]], i64 8
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i16, ptr [[TMP4]], i64 8
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2
-; CHECK-NEXT: [[TMP5]] = add <8 x i16> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP6]] = add <8 x i16> [[VEC_PHI2]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2
+; CHECK-NEXT: [[TMP5]] = add <8 x i16> [[VEC_PHI]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP6]] = add <8 x i16> [[VEC_PHI2]], [[WIDE_LOAD4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
@@ -275,7 +275,7 @@ define i16 @bounded_load_reduction_bound4_i16(ptr %A, i32 %N) {
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX6:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i16> [ [[TMP9]], %[[VEC_EPILOG_PH]] ], [ [[TMP12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = urem i32 [[INDEX6]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = and i32 [[INDEX6]], 3
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[TMP10]]
; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i16>, ptr [[TMP11]], align 2
; CHECK-NEXT: [[TMP12]] = add <4 x i16> [[VEC_PHI7]], [[WIDE_LOAD8]]
@@ -348,7 +348,7 @@ define i32 @bounded_user_ic_exceeds_window(ptr %A, i32 %N) {
; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 4
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 8
@@ -447,7 +447,7 @@ define i32 @scalable_vf_rejected(ptr %A, i64 %N) #0 {
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = urem i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP4]], 1
; CHECK-NEXT: [[TMP9:%.*]] = mul nuw nsw i64 [[TMP4]], 3
@@ -544,7 +544,7 @@ define i32 @reverse_load_with_bounded(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 4
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 8
@@ -598,7 +598,7 @@ define i32 @reverse_load_with_bounded(ptr %A, ptr %B, i32 %N) {
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX20:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT25:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI21:%.*]] = phi <4 x i32> [ [[TMP25]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP26:%.*]] = urem i32 [[INDEX20]], 4
+; CHECK-NEXT: [[TMP26:%.*]] = and i32 [[INDEX20]], 3
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP26]]
; CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <4 x i32>, ptr [[TMP27]], align 4
; CHECK-NEXT: [[TMP28:%.*]] = sub i32 [[N]], [[INDEX20]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
index 01a12b67dcf4b..69686e3f74cf4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll
@@ -22,7 +22,7 @@ define void @urem_lookup(ptr noalias %src, ptr noalias %dst, ptr noalias %tbl, i
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP2]], %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[GEP_SRC]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
-; CHECK-NEXT: [[TMP4:%.*]] = urem <vscale x 4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = and <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TBL]], <vscale x 4 x i64> [[TMP4]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP5]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
; CHECK-NEXT: [[TMP6:%.*]] = fmul <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_GATHER]]
@@ -82,7 +82,7 @@ define void @over_conservative_merge(ptr noalias %A, ptr noalias %B, i64 %N) #0
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = urem <vscale x 4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = and <vscale x 4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr float, ptr [[A]], <vscale x 4 x i64> [[TMP3]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
; CHECK-NEXT: [[TMP4:%.*]] = fmul <vscale x 4 x float> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/bounded-load.ll b/llvm/test/Transforms/LoopVectorize/RISCV/bounded-load.ll
index 814bbddb42c1b..3c5d29371d8cf 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/bounded-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/bounded-load.ll
@@ -19,7 +19,7 @@ define i32 @evl_tail_folding_rejected(ptr %A, i32 %N) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[TMP3:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP3]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-fold-dbg.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-fold-dbg.ll
new file mode 100644
index 0000000000000..390f829a13606
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-fold-dbg.ll
@@ -0,0 +1,67 @@
+; RUN: opt -p debugify,loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S \
+; RUN: -vplan-print-after=printOptimizedVPlan < %s 2> %t | FileCheck %s
+; RUN: cat %t | FileCheck %s --check-prefix=VPLAN
+
+; The debug attached to the urem instruction should remain after
+; being folded to `and i64 %iv, 15`
+define void @urem_fold(ptr %dst, ptr %src, i64 %n) {
+; CHECK-LABEL: define void @urem_fold(
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[FOLD:%.*]] = and i64 {{.*}}, 15, !dbg [[DBG:![0-9]+]]
+; CHECK: [[LOOP:.*:]]
+; CHECK: [[ORIG:%.*]] = urem i64 [[IV:%.*]], 16, !dbg [[DBG]]
+;
+; VPLAN-LABEL: VPlan for loop in 'urem_fold' after printOptimizedVPlan
+; VPLAN: EMIT vp{{.*}} = and vp{{.*}}, ir<15>, !dbg
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %urem = urem i64 %iv, 16
+ %arrayidx = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %urem
+ %load1 = load i32, ptr %arrayidx, align 4
+ %arrayidx2 = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %iv
+ %load2 = load i32, ptr %arrayidx2, align 4
+ %add = add nsw i32 %load2, %load1
+ store i32 %add, ptr %arrayidx2, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret void
+
+}
+
+define void @test_select_chain_debugloc(ptr noalias %dst, ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @test_select_chain_debugloc(
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[VEC_SEL1:%.*]] = select <4 x i1> {{.*}}, <4 x i1> {{.*}}, <4 x i1> zeroinitializer
+; CHECK: [[VEC_SEL2:%.*]] = select <4 x i1> [[VEC_SEL1]], <4 x i32> {{.*}}, <4 x i32> {{.*}}, !dbg [[DBG_OUTER:![0-9]+]]
+; CHECK: [[LOOP:.*:]]
+; CHECK: [[ORIG_SEL_INNER:%.*]] = select i1 {{.*}}, i32 {{.*}}, i32 {{.*}}, !dbg [[DBG_INNER:![0-9]+]]
+; CHECK: [[ORIG_SEL_OUTER:%.*]] = select i1 {{.*}}, i32 [[ORIG_SEL_INNER]], i32 %lb, !dbg [[DBG_OUTER]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %la = load i32, ptr %gep.a, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %lb = load i32, ptr %gep.b, align 4
+ %m0 = icmp sgt i32 %la, 0
+ %m1 = icmp slt i32 %lb, 100
+ %inner = select i1 %m1, i32 %la, i32 %lb
+ %outer = select i1 %m0, i32 %inner, i32 %lb
+ %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i32 %outer, ptr %gep.dst, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
index 6b3195f559849..3b384674daa8d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
@@ -362,10 +362,10 @@ define i32 @test_count_bits(ptr %test_base) {
; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP11]], align 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i8> poison, i8 [[TMP15]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT11]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = urem <4 x i64> [[VEC_IND]], splat (i64 8)
-; CHECK-NEXT: [[TMP17:%.*]] = urem <4 x i64> [[STEP_ADD]], splat (i64 8)
-; CHECK-NEXT: [[TMP18:%.*]] = urem <4 x i64> [[STEP_ADD_2]], splat (i64 8)
-; CHECK-NEXT: [[TMP19:%.*]] = urem <4 x i64> [[STEP_ADD_3]], splat (i64 8)
+; CHECK-NEXT: [[TMP16:%.*]] = and <4 x i64> [[VEC_IND]], splat (i64 7)
+; CHECK-NEXT: [[TMP17:%.*]] = and <4 x i64> [[STEP_ADD]], splat (i64 7)
+; CHECK-NEXT: [[TMP18:%.*]] = and <4 x i64> [[STEP_ADD_2]], splat (i64 7)
+; CHECK-NEXT: [[TMP19:%.*]] = and <4 x i64> [[STEP_ADD_3]], splat (i64 7)
; CHECK-NEXT: [[TMP20:%.*]] = trunc <4 x i64> [[TMP16]] to <4 x i8>
; CHECK-NEXT: [[TMP21:%.*]] = trunc <4 x i64> [[TMP17]] to <4 x i8>
; CHECK-NEXT: [[TMP22:%.*]] = trunc <4 x i64> [[TMP18]] to <4 x i8>
diff --git a/llvm/test/Transforms/LoopVectorize/bounded-load-predicated.ll b/llvm/test/Transforms/LoopVectorize/bounded-load-predicated.ll
index 0c1f159ac6129..b5ecc62e44106 100644
--- a/llvm/test/Transforms/LoopVectorize/bounded-load-predicated.ll
+++ b/llvm/test/Transforms/LoopVectorize/bounded-load-predicated.ll
@@ -25,14 +25,14 @@ define i32 @clamped_load_predicated(ptr %A, i32 %n) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 1)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = urem <4 x i32> [[VEC_IND]], splat (i32 128)
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
-; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK-NEXT: [[TMP28:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 127)
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
+; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP28]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
@@ -42,7 +42,7 @@ define i32 @clamped_load_predicated(ptr %A, i32 %n) {
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
; CHECK: [[PRED_LOAD_IF1]]:
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP28]], i64 1
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP14]], i64 1
@@ -52,7 +52,7 @@ define i32 @clamped_load_predicated(ptr %A, i32 %n) {
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[TMP4]], i64 2
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[TMP28]], i64 2
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP18]]
; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP20]], i64 2
@@ -62,21 +62,21 @@ define i32 @clamped_load_predicated(ptr %A, i32 %n) {
; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP4]], i64 3
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP28]], i64 3
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP24]]
; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4
; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP26]], i64 3
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP27]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP28]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP29]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP33:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP27]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP33]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP29]])
+; CHECK-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP32]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -135,10 +135,10 @@ define i32 @clamped_load_predicated(ptr %A, i32 %n) {
; IC4-NEXT: [[TMP7:%.*]] = icmp eq <4 x i32> [[TMP3]], zeroinitializer
; IC4-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[TMP4]], zeroinitializer
; IC4-NEXT: [[TMP9:%.*]] = icmp eq <4 x i32> [[TMP5]], zeroinitializer
-; IC4-NEXT: [[TMP10:%.*]] = urem <4 x i32> [[VEC_IND]], splat (i32 128)
-; IC4-NEXT: [[TMP11:%.*]] = urem <4 x i32> [[STEP_ADD]], splat (i32 128)
-; IC4-NEXT: [[TMP12:%.*]] = urem <4 x i32> [[STEP_ADD_2]], splat (i32 128)
-; IC4-NEXT: [[TMP13:%.*]] = urem <4 x i32> [[STEP_ADD_3]], splat (i32 128)
+; IC4-NEXT: [[TMP10:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 127)
+; IC4-NEXT: [[TMP11:%.*]] = and <4 x i32> [[STEP_ADD]], splat (i32 127)
+; IC4-NEXT: [[TMP12:%.*]] = and <4 x i32> [[STEP_ADD_2]], splat (i32 127)
+; IC4-NEXT: [[TMP13:%.*]] = and <4 x i32> [[STEP_ADD_3]], splat (i32 127)
; IC4-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; IC4-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; IC4: [[PRED_LOAD_IF]]:
@@ -390,14 +390,14 @@ define i32 @clamped_load_predicated(ptr %A, i32 %n) {
; IC8-NEXT: [[TMP15:%.*]] = icmp eq <4 x i32> [[TMP7]], zeroinitializer
; IC8-NEXT: [[TMP16:%.*]] = icmp eq <4 x i32> [[TMP8]], zeroinitializer
; IC8-NEXT: [[TMP17:%.*]] = icmp eq <4 x i32> [[TMP9]], zeroinitializer
-; IC8-NEXT: [[TMP18:%.*]] = urem <4 x i32> [[VEC_IND]], splat (i32 128)
-; IC8-NEXT: [[TMP19:%.*]] = urem <4 x i32> [[STEP_ADD]], splat (i32 128)
-; IC8-NEXT: [[TMP20:%.*]] = urem <4 x i32> [[STEP_ADD_2]], splat (i32 128)
-; IC8-NEXT: [[TMP21:%.*]] = urem <4 x i32> [[STEP_ADD_3]], splat (i32 128)
-; IC8-NEXT: [[TMP22:%.*]] = urem <4 x i32> [[STEP_ADD_4]], splat (i32 128)
-; IC8-NEXT: [[TMP23:%.*]] = urem <4 x i32> [[STEP_ADD_5]], splat (i32 128)
-; IC8-NEXT: [[TMP24:%.*]] = urem <4 x i32> [[STEP_ADD_6]], splat (i32 128)
-; IC8-NEXT: [[TMP25:%.*]] = urem <4 x i32> [[STEP_ADD_7]], splat (i32 128)
+; IC8-NEXT: [[TMP18:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 127)
+; IC8-NEXT: [[TMP19:%.*]] = and <4 x i32> [[STEP_ADD]], splat (i32 127)
+; IC8-NEXT: [[TMP20:%.*]] = and <4 x i32> [[STEP_ADD_2]], splat (i32 127)
+; IC8-NEXT: [[TMP21:%.*]] = and <4 x i32> [[STEP_ADD_3]], splat (i32 127)
+; IC8-NEXT: [[TMP22:%.*]] = and <4 x i32> [[STEP_ADD_4]], splat (i32 127)
+; IC8-NEXT: [[TMP23:%.*]] = and <4 x i32> [[STEP_ADD_5]], splat (i32 127)
+; IC8-NEXT: [[TMP24:%.*]] = and <4 x i32> [[STEP_ADD_6]], splat (i32 127)
+; IC8-NEXT: [[TMP25:%.*]] = and <4 x i32> [[STEP_ADD_7]], splat (i32 127)
; IC8-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP10]], i64 0
; IC8-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; IC8: [[PRED_LOAD_IF]]:
@@ -829,7 +829,7 @@ define i32 @clamped_load_predicated_ic_capped(ptr %A, i32 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 1)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = urem <4 x i32> [[VEC_IND]], splat (i32 16)
+; CHECK-NEXT: [[TMP4:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 15)
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
@@ -936,10 +936,10 @@ define i32 @clamped_load_predicated_ic_capped(ptr %A, i32 %n) {
; IC4-NEXT: [[TMP7:%.*]] = icmp eq <4 x i32> [[TMP3]], zeroinitializer
; IC4-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[TMP4]], zeroinitializer
; IC4-NEXT: [[TMP9:%.*]] = icmp eq <4 x i32> [[TMP5]], zeroinitializer
-; IC4-NEXT: [[TMP10:%.*]] = urem <4 x i32> [[VEC_IND]], splat (i32 16)
-; IC4-NEXT: [[TMP11:%.*]] = urem <4 x i32> [[STEP_ADD]], splat (i32 16)
-; IC4-NEXT: [[TMP12:%.*]] = urem <4 x i32> [[STEP_ADD_2]], splat (i32 16)
-; IC4-NEXT: [[TMP13:%.*]] = urem <4 x i32> [[STEP_ADD_3]], splat (i32 16)
+; IC4-NEXT: [[TMP10:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 15)
+; IC4-NEXT: [[TMP11:%.*]] = and <4 x i32> [[STEP_ADD]], splat (i32 15)
+; IC4-NEXT: [[TMP12:%.*]] = and <4 x i32> [[STEP_ADD_2]], splat (i32 15)
+; IC4-NEXT: [[TMP13:%.*]] = and <4 x i32> [[STEP_ADD_3]], splat (i32 15)
; IC4-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; IC4-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; IC4: [[PRED_LOAD_IF]]:
@@ -1191,14 +1191,14 @@ define i32 @clamped_load_predicated_ic_capped(ptr %A, i32 %n) {
; IC8-NEXT: [[TMP15:%.*]] = icmp eq <4 x i32> [[TMP7]], zeroinitializer
; IC8-NEXT: [[TMP16:%.*]] = icmp eq <4 x i32> [[TMP8]], zeroinitializer
; IC8-NEXT: [[TMP17:%.*]] = icmp eq <4 x i32> [[TMP9]], zeroinitializer
-; IC8-NEXT: [[TMP18:%.*]] = urem <4 x i32> [[VEC_IND]], splat (i32 16)
-; IC8-NEXT: [[TMP19:%.*]] = urem <4 x i32> [[STEP_ADD]], splat (i32 16)
-; IC8-NEXT: [[TMP20:%.*]] = urem <4 x i32> [[STEP_ADD_2]], splat (i32 16)
-; IC8-NEXT: [[TMP21:%.*]] = urem <4 x i32> [[STEP_ADD_3]], splat (i32 16)
-; IC8-NEXT: [[TMP22:%.*]] = urem <4 x i32> [[STEP_ADD_4]], splat (i32 16)
-; IC8-NEXT: [[TMP23:%.*]] = urem <4 x i32> [[STEP_ADD_5]], splat (i32 16)
-; IC8-NEXT: [[TMP24:%.*]] = urem <4 x i32> [[STEP_ADD_6]], splat (i32 16)
-; IC8-NEXT: [[TMP25:%.*]] = urem <4 x i32> [[STEP_ADD_7]], splat (i32 16)
+; IC8-NEXT: [[TMP18:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 15)
+; IC8-NEXT: [[TMP19:%.*]] = and <4 x i32> [[STEP_ADD]], splat (i32 15)
+; IC8-NEXT: [[TMP20:%.*]] = and <4 x i32> [[STEP_ADD_2]], splat (i32 15)
+; IC8-NEXT: [[TMP21:%.*]] = and <4 x i32> [[STEP_ADD_3]], splat (i32 15)
+; IC8-NEXT: [[TMP22:%.*]] = and <4 x i32> [[STEP_ADD_4]], splat (i32 15)
+; IC8-NEXT: [[TMP23:%.*]] = and <4 x i32> [[STEP_ADD_5]], splat (i32 15)
+; IC8-NEXT: [[TMP24:%.*]] = and <4 x i32> [[STEP_ADD_6]], splat (i32 15)
+; IC8-NEXT: [[TMP25:%.*]] = and <4 x i32> [[STEP_ADD_7]], splat (i32 15)
; IC8-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP10]], i64 0
; IC8-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; IC8: [[PRED_LOAD_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/bounded-load-user-ic.ll b/llvm/test/Transforms/LoopVectorize/bounded-load-user-ic.ll
index 280613d6b04ec..bd3a906eca0da 100644
--- a/llvm/test/Transforms/LoopVectorize/bounded-load-user-ic.ll
+++ b/llvm/test/Transforms/LoopVectorize/bounded-load-user-ic.ll
@@ -22,7 +22,7 @@ define i32 @bounded_user_ic_exceeds_window(ptr %A, i32 %N) {
; IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; IC4-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 8
+; IC4-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 7
; IC4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; IC4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 4
; IC4-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 8
@@ -81,7 +81,7 @@ define i32 @bounded_user_ic_exceeds_window(ptr %A, i32 %N) {
; IC3-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
; IC3-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; IC3-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; IC3-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 8
+; IC3-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 7
; IC3-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; IC3-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 4
; IC3-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 8
diff --git a/llvm/test/Transforms/LoopVectorize/bounded-load-vf-ranges.ll b/llvm/test/Transforms/LoopVectorize/bounded-load-vf-ranges.ll
index adeed75278ba6..bc596a4ea152e 100644
--- a/llvm/test/Transforms/LoopVectorize/bounded-load-vf-ranges.ll
+++ b/llvm/test/Transforms/LoopVectorize/bounded-load-vf-ranges.ll
@@ -248,7 +248,7 @@ define void @bounded_rmw_vf_capped(ptr noalias %A, ptr noalias %B, i32 %n) {
; VF2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2: [[VECTOR_BODY]]:
; VF2-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VF2-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
+; VF2-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 3
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
@@ -294,7 +294,7 @@ define void @bounded_rmw_vf_capped(ptr noalias %A, ptr noalias %B, i32 %n) {
; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VF4-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
+; VF4-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 3
; VF4-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
; VF4-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
@@ -340,7 +340,7 @@ define void @bounded_rmw_vf_capped(ptr noalias %A, ptr noalias %B, i32 %n) {
; VF8-NEXT: br label %[[VECTOR_BODY:.*]]
; VF8: [[VECTOR_BODY]]:
; VF8-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VF8-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
+; VF8-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 3
; VF8-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; VF8-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
; VF8-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/bounded-store.ll b/llvm/test/Transforms/LoopVectorize/bounded-store.ll
index 0ac378a4ec54f..3160e5c1e731b 100644
--- a/llvm/test/Transforms/LoopVectorize/bounded-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/bounded-store.ll
@@ -87,7 +87,7 @@ define void @bounded_load_with_store_optsize(ptr noalias %A, ptr noalias %B) opt
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = urem <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP0:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 3)
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[TMP0]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i64 1
diff --git a/llvm/test/Transforms/LoopVectorize/debugloc.ll b/llvm/test/Transforms/LoopVectorize/debugloc.ll
index 0db47786fd03c..fd8f023d8b6ce 100644
--- a/llvm/test/Transforms/LoopVectorize/debugloc.ll
+++ b/llvm/test/Transforms/LoopVectorize/debugloc.ll
@@ -201,47 +201,6 @@ exit:
ret void
}
-define void @test_select_chain_debugloc(ptr noalias %dst, ptr noalias %a, ptr noalias %b, i64 %n) !dbg !42 {
-; CHECK-LABEL: define void @test_select_chain_debugloc(
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr %a, i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr %b, i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD1]], splat (i32 100)
-; CHECK-NEXT: [[TMP4:%.*]] = select <2 x i1> [[TMP2]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]], !dbg [[LOC9:![0-9]+]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr %dst, i64 [[INDEX]]
-; CHECK-NEXT: store <2 x i32> [[TMP5]], ptr [[TMP6]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]],
-; CHECK-NEXT: br i1 [[TMP7]], label %middle.block, label %vector.body
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %la = load i32, ptr %gep.a, align 4
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- %lb = load i32, ptr %gep.b, align 4
- %m0 = icmp sgt i32 %la, 0
- %m1 = icmp slt i32 %lb, 100
- %inner = select i1 %m1, i32 %la, i32 %lb, !dbg !44
- %outer = select i1 %m0, i32 %inner, i32 %lb, !dbg !45
- %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
- store i32 %outer, ptr %gep.dst, align 4
- %iv.next = add nuw nsw i64 %iv, 1
- %ec = icmp eq i64 %iv.next, %n
- br i1 %ec, label %exit, label %loop
-
-exit:
- ret void
-}
-
; CHECK: ![[LOC2]] = !DILocation(line: 3
; CHECK: ![[BR_LOC]] = !DILocation(line: 5,
; CHECK: ![[LOC1]] = !DILocation(line: 6
@@ -251,7 +210,6 @@ exit:
; CHECK: [[LOC6]] = !DILocation(line: 430
; CHECK: [[LOC7]] = !DILocation(line: 540
; CHECK: [[LOC8]] = !DILocation(line: 650
-; CHECK: [[LOC9]] = !DILocation(line: 761
@@ -297,7 +255,3 @@ exit:
!39 = distinct !DISubprogram(name: "test_scalar_Steps", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 3, file: !5, scope: !6, type: !7, retainedNodes: !2)
!40 = distinct !DILexicalBlock(scope: !39, file: !5, line: 137, column: 2)
!41 = !DILocation(line: 650, column: 44, scope: !40)
-!42 = distinct !DISubprogram(name: "test_select_chain_debugloc", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 3, file: !5, scope: !6, type: !7, retainedNodes: !2)
-!43 = distinct !DILexicalBlock(scope: !42, file: !5, line: 137, column: 2)
-!44 = !DILocation(line: 760, column: 44, scope: !43)
-!45 = !DILocation(line: 761, column: 44, scope: !43)
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-small-bounded-ranges.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-small-bounded-ranges.ll
index 1eba63048137c..ff8934375384f 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-small-bounded-ranges.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-small-bounded-ranges.ll
@@ -33,7 +33,7 @@ define void @load_bounded_index(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -102,7 +102,7 @@ define i32 @bounded_load_reduction(ptr %A, i32 %N) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[TMP4]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
@@ -174,7 +174,7 @@ define void @store_bounded_index(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -254,7 +254,7 @@ define void @load_bounded_index_offset_1(ptr %A, ptr %B, i32 %N) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = add nuw i32 1, [[INDEX]]
-; CHECK-NEXT: [[TMP11:%.*]] = urem i32 [[TMP10]], 4
+; CHECK-NEXT: [[TMP11:%.*]] = and i32 [[TMP10]], 3
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP11]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -440,7 +440,7 @@ define void @bounded_index_equal_dependence(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -595,7 +595,7 @@ define void @load_bounded_mod8_i32(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[INDEX]], 7
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -667,7 +667,7 @@ define void @load_bounded_mod4_i64(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i32 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[WIDE_LOAD]], splat (i64 10)
@@ -740,7 +740,7 @@ define void @load_bounded_mod128_i32(ptr %A, ptr %B, i32 %N) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[INDEX]], 128
+; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[INDEX]], 127
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -815,8 +815,8 @@ define void @two_bounded_loads_mixed_bounds(ptr %A, ptr %C, ptr %B, i32 %N) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = urem i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP7:%.*]] = urem i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[INDEX]], 7
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[C]], i32 [[TMP7]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4
@@ -890,7 +890,7 @@ define i32 @scale_mismatch_rejected(ptr %A, i32 %N) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = urem <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[VEC_IND]], splat (i32 3)
; CHECK-NEXT: [[TMP3:%.*]] = shl <4 x i32> [[TMP2]], splat (i32 1)
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP4]]
More information about the llvm-commits
mailing list