[llvm] [VPlan] Prevent uses of materialized VPSymbolicValues. (NFC) (PR #182318)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 19 08:59:37 PST 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/182318
After VPSymbolicValues (like VF and VFxUF) are materialized via
replaceAllUsesWith, they should not be accessed again. This patch:
1. Tracks materialization state in VPSymbolicValue.
2. Asserts if the materialized VPValue is used again. Currently it
adds asserts to various member functions, preventing calling them
on materialized symbolic values.
Note that this still allows some uses (e.g. comparing VPSymbolicValue
references or pointers), but this should be relatively harmless given
that it is impossible to (re-)add any users. If we want to further
tighten the checks, we could add asserts to the accessors or override
operator&, but that will require more changes and not add much extra
guards I think.
Depends on https://github.com/llvm/llvm-project/pull/182146 to fix a
current access violation (included in PR).
>From c3e6c797b213b552f8e2d48e1ef565859669c68b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 17 Feb 2026 19:01:30 +0000
Subject: [PATCH 1/2] [VPlan] Materialize VectorTripCount in
narrowInterleaveGroups.
When narrowInterleaveGroups transforms a plan, VF and VFxUF are
materialized (replaced with concrete values). This patch also
materializes the VectorTripCount in the same transform.
This ensures that VectorTripCount is properly computed when the
narrow interleave transform is applied, instead of using the original VF
+ UF to compute the vector trip count. The previous behavior generated
correct code, but executed fewer iterations in the vector loop.
The change also enables stricter verification prevent accesses of
VPSymbolicValues after materialization as follow-up.
Note that in some cases we no miss branch folding, but that should be
addressed separately, https://github.com/llvm/llvm-project/pull/181252
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 27 +++-
...interleave-to-widen-memory-constant-ops.ll | 18 +--
...-narrow-interleave-to-widen-memory-cost.ll | 28 ++--
...-interleave-to-widen-memory-derived-ivs.ll | 22 ++-
...row-interleave-to-widen-memory-metadata.ll | 4 +-
...eave-to-widen-memory-remove-loop-region.ll | 17 ++-
...row-interleave-to-widen-memory-scalable.ll | 9 +-
...arrow-interleave-to-widen-memory-unroll.ll | 36 ++++-
...nterleave-to-widen-memory-with-wide-ops.ll | 22 +--
...sform-narrow-interleave-to-widen-memory.ll | 28 ++--
...sform-narrow-interleave-to-widen-memory.ll | 6 +-
...sform-narrow-interleave-to-widen-memory.ll | 142 ++++++++++++++++--
12 files changed, 259 insertions(+), 100 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bb1a91ec8c963..c78895b53e41d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5427,21 +5427,38 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
// original iteration.
auto *CanIV = VectorLoop->getCanonicalIV();
auto *Inc = cast<VPInstruction>(CanIV->getBackedgeValue());
- VPBuilder PHBuilder(Plan.getVectorPreheader());
+ VPBasicBlock *VectorPH = Plan.getVectorPreheader();
+ VPBuilder PHBuilder(VectorPH, VectorPH->begin());
VPValue *UF = &Plan.getUF();
+ VPValue *Step;
if (VFToOptimize->isScalable()) {
VPValue *VScale = PHBuilder.createElementCount(
VectorLoop->getCanonicalIVType(), ElementCount::getScalable(1));
- VPValue *VScaleUF = PHBuilder.createOverflowingOp(
- Instruction::Mul, {VScale, UF}, {true, false});
- Inc->setOperand(1, VScaleUF);
+ Step = PHBuilder.createOverflowingOp(Instruction::Mul, {VScale, UF},
+ {true, false});
Plan.getVF().replaceAllUsesWith(VScale);
} else {
- Inc->setOperand(1, UF);
+ Step = UF;
Plan.getVF().replaceAllUsesWith(
Plan.getConstantInt(CanIV->getScalarType(), 1));
}
+
+ // Materialize vector trip count with the narrowed step: TC - (TC % Step).
+ assert(Plan.getMiddleBlock()->getNumSuccessors() == 2 &&
+ "cannot materialize vector trip count when folding the tail or "
+ "requiring a scalar iteration");
+ VPValue *TC = Plan.getTripCount();
+ VPValue *R =
+ PHBuilder.createNaryOp(Instruction::URem, {TC, Step},
+ DebugLoc::getCompilerGenerated(), "n.mod.vf");
+ VPValue *VectorTC =
+ PHBuilder.createSub(TC, R, DebugLoc::getCompilerGenerated(), "n.vec");
+ Plan.getVectorTripCount().replaceAllUsesWith(VectorTC);
+
+ Inc->setOperand(1, Step);
+ Plan.getVFxUF().replaceAllUsesWith(Step);
+
removeDeadRecipes(Plan);
assert(none_of(*VectorLoop->getEntryBasicBlock(),
IsaPred<VPVectorPointerRecipe>) &&
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
index 52bd8a0a11e35..6d0e0503c49a5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
@@ -28,9 +28,8 @@ define void @test_add_double_same_const_args_1(ptr %res, ptr noalias %A, ptr noa
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -77,11 +76,10 @@ define void @test_add_double_same_const_args_2(ptr %res, ptr noalias %A, ptr noa
; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -191,11 +189,10 @@ define void @test_add_double_same_var_args_1(ptr %res, ptr noalias %A, ptr noali
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -244,11 +241,10 @@ define void @test_add_double_same_var_args_2(ptr %res, ptr noalias %A, ptr noali
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
index 5e37f9eff4ba2..ccbcfff7fda7b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
@@ -87,7 +87,7 @@ define void @test_complex_add_double(ptr %res, ptr noalias %A, ptr noalias %B, i
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -314,8 +314,8 @@ define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr
; CHECK-NEXT: [[MIN_ITERS_CHECK11:%.*]] = icmp ult i64 [[TMP0]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK11]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP41:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[N_VEC12:%.*]] = sub i64 [[TMP0]], [[TMP41]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -359,18 +359,16 @@ define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr
; CHECK-NEXT: store <2 x double> [[TMP30]], ptr [[TMP37]], align 8
; CHECK-NEXT: store <2 x double> [[TMP31]], ptr [[TMP43]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC12]]
; CHECK-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC12]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP41]], 2
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[N_MOD_VF22:%.*]] = urem i64 [[TMP0]], 2
-; CHECK-NEXT: [[N_VEC23:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF22]]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX24:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT25:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
@@ -384,13 +382,12 @@ define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr
; CHECK-NEXT: [[TMP50:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX24]]
; CHECK-NEXT: store <2 x double> [[TMP48]], ptr [[TMP50]], align 8
; CHECK-NEXT: [[INDEX_NEXT25]] = add nuw i64 [[INDEX24]], 1
-; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT25]], [[N_VEC23]]
-; CHECK-NEXT: br i1 [[TMP51]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT25]], [[TMP0]]
+; CHECK-NEXT: br i1 [[TMP46]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N26:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC23]]
-; CHECK-NEXT: br i1 [[CMP_N26]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC12]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -462,9 +459,8 @@ define void @test_interleave_after_narrowing(i32 %n, ptr %x, ptr noalias %y) {
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
index fab0369de8aa0..6428fe6a29445 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
@@ -16,7 +16,7 @@ define void @derived_int_ivs(ptr noalias %a, ptr noalias %b, i64 %end) {
; VF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
; VF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF2: [[VECTOR_PH]]:
-; VF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2
+; VF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 1
; VF2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; VF2-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], 16
; VF2-NEXT: [[TMP4:%.*]] = add i64 16, [[TMP3]]
@@ -30,8 +30,8 @@ define void @derived_int_ivs(ptr noalias %a, ptr noalias %b, i64 %end) {
; VF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OFFSET_IDX]]
; VF2-NEXT: store <2 x double> [[WIDE_LOAD]], ptr [[TMP7]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
-; VF2-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VF2-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
; VF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; VF2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -46,7 +46,7 @@ define void @derived_int_ivs(ptr noalias %a, ptr noalias %b, i64 %end) {
; VF2IC2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
; VF2IC2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF2IC2: [[VECTOR_PH]]:
-; VF2IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; VF2IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2
; VF2IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; VF2IC2-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], 16
; VF2IC2-NEXT: [[TMP4:%.*]] = add i64 16, [[TMP3]]
@@ -154,7 +154,7 @@ define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
; VF2-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; VF2-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; VF2: [[VECTOR_PH]]:
-; VF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; VF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 1
; VF2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; VF2-NEXT: [[TMP9:%.*]] = mul i64 [[N_VEC]], 16
; VF2-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
@@ -170,8 +170,8 @@ define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[NEXT_GEP]], align 8
; VF2-NEXT: store <2 x double> [[WIDE_LOAD]], ptr [[NEXT_GEP7]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
-; VF2-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; VF2-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
; VF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; VF2-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -203,7 +203,7 @@ define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
; VF2IC2-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; VF2IC2-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; VF2IC2: [[VECTOR_PH]]:
-; VF2IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; VF2IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
; VF2IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; VF2IC2-NEXT: [[TMP9:%.*]] = mul i64 [[N_VEC]], 16
; VF2IC2-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
@@ -324,9 +324,8 @@ define void @narrow_with_uniform_add_and_gep(ptr noalias %p) {
; VF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br label %[[EXIT:.*]]
+; VF2-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; VF2: [[EXIT]]:
-; VF2-NEXT: ret void
;
; VF2IC2-LABEL: define void @narrow_with_uniform_add_and_gep(
; VF2IC2-SAME: ptr noalias [[P:%.*]]) {
@@ -350,9 +349,8 @@ define void @narrow_with_uniform_add_and_gep(ptr noalias %p) {
; VF2IC2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
; VF2IC2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2-NEXT: br label %[[EXIT:.*]]
+; VF2IC2-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; VF2IC2: [[EXIT]]:
-; VF2IC2-NEXT: ret void
;
; VF4-LABEL: define void @narrow_with_uniform_add_and_gep(
; VF4-SAME: ptr noalias [[P:%.*]]) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-metadata.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-metadata.ll
index 442574f298c3f..773e6f0bfc4a5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-metadata.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-metadata.ll
@@ -23,8 +23,8 @@ define void @load_store_interleave_group_with_metadata(ptr noalias %data) {
; VF2-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF2-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br [[EXIT:label %.*]]
-; VF2: [[SCALAR_PH:.*:]]
+; VF2-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; VF2: [[SCALAR_PH]]:
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
index 7d77f2f6b5b9c..d4fb7f7e7685c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
@@ -22,8 +22,23 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) {
; VF2-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 2
; VF2-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
+; VF2-NEXT: br i1 true, label %[[EXIT1:.*]], label %[[SCALAR_PH:.*]]
+; VF2: [[SCALAR_PH]]:
; VF2-NEXT: br label %[[EXIT:.*]]
; VF2: [[EXIT]]:
+; VF2-NEXT: [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT]] ]
+; VF2-NEXT: [[MUL_2:%.*]] = shl nsw i64 [[IV]], 1
+; VF2-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[MUL_2]]
+; VF2-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
+; VF2-NEXT: store i64 [[L_0]], ptr [[DATA_0]], align 8
+; VF2-NEXT: [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 1
+; VF2-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[ADD_1]]
+; VF2-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8
+; VF2-NEXT: store i64 [[L_1]], ptr [[DATA_1]], align 8
+; VF2-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; VF2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 2
+; VF2-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF2: [[EXIT1]]:
; VF2-NEXT: ret void
;
; VF4-LABEL: define void @load_store_interleave_group_tc_2(
@@ -205,7 +220,7 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
; VF2-NEXT: store <4 x float> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 4
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
-; VF2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
; VF2-NEXT: br label %[[EXIT:.*]]
; VF2: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
index fa60f8eec28f1..1436f8012edc4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
@@ -13,8 +13,7 @@ define void @load_store_interleave_group(ptr noalias %data) {
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP2]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -62,8 +61,7 @@ define void @test_2xi64_unary_op_load_interleave_group(ptr noalias %data, ptr no
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1111, [[TMP3]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1111, [[TMP2]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1111, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -114,8 +112,7 @@ define void @narrow_interleave_tc_16_vf_4_if_4(ptr noalias %data) vscale_range(2
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 16, [[TMP3]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 16, [[TMP2]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 16, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll
index 22291203b8c71..f2edd3712aa7c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll
@@ -26,8 +26,23 @@ define void @load_store_interleave_group(ptr noalias %data) {
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 true, label %[[EXIT1:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[MUL_2:%.*]] = shl nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[MUL_2]]
+; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
+; CHECK-NEXT: store i64 [[L_0]], ptr [[DATA_0]], align 8
+; CHECK-NEXT: [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 1
+; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[ADD_1]]
+; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8
+; CHECK-NEXT: store i64 [[L_1]], ptr [[DATA_1]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -81,10 +96,29 @@ define void @test_2xi64_with_wide_load(ptr noalias %data, ptr noalias %factor) {
; CHECK-NEXT: store <2 x i64> [[TMP16]], ptr [[TMP9]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 true, label %[[EXIT1:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]
+; CHECK-NEXT: [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = shl nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP14]]
+; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
+; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR]], [[L_0]]
+; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_0]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = or disjoint i64 [[TMP14]], 1
+; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP13]]
+; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8
+; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR]], [[L_1]]
+; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
index 6d0c55b1d246c..f30509472a25f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
@@ -23,8 +23,8 @@ define void @test_2xi64_unary_op_load_interleave_group(ptr noalias %data, ptr no
; VF2-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF2-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br [[EXIT:label %.*]]
-; VF2: [[SCALAR_PH:.*:]]
+; VF2-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; VF2: [[SCALAR_PH]]:
;
; VF4-LABEL: define void @test_2xi64_unary_op_load_interleave_group(
; VF4-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]]) {
@@ -172,10 +172,10 @@ define void @test_2xi64(ptr noalias %data, ptr noalias %factor) {
; VF2-NEXT: store <2 x i64> [[TMP8]], ptr [[TMP7]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VF2-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br [[EXIT:label %.*]]
-; VF2: [[SCALAR_PH:.*:]]
+; VF2-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; VF2: [[SCALAR_PH]]:
;
; VF4-LABEL: define void @test_2xi64(
; VF4-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]]) {
@@ -952,10 +952,10 @@ define void @test_2xi64_sub_of_wide_loads(ptr noalias %data, ptr noalias %A, ptr
; VF2-NEXT: store <2 x i64> [[TMP13]], ptr [[DATA_1]], align 8
; VF2-NEXT: [[IV_NEXT]] = add nuw i64 [[TMP0]], 1
; VF2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; VF2-NEXT: br i1 [[EC]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF2-NEXT: br i1 [[EC]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br [[EXIT:label %.*]]
-; VF2: [[SCALAR_PH:.*:]]
+; VF2-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; VF2: [[SCALAR_PH]]:
;
; VF4-LABEL: define void @test_2xi64_sub_of_wide_loads(
; VF4-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1189,10 +1189,10 @@ define void @multiple_store_groups_storing_same_wide_bin_op(ptr noalias %A, ptr
; VF2-NEXT: store <2 x double> [[TMP2]], ptr [[TMP4]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br [[EXIT:label %.*]]
-; VF2: [[SCALAR_PH:.*:]]
+; VF2-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; VF2: [[SCALAR_PH]]:
;
; VF4-LABEL: define void @multiple_store_groups_storing_same_wide_bin_op(
; VF4-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index 5c9a3eac44fa0..d45edc17faf27 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -22,9 +22,8 @@ define void @load_store_interleave_group(ptr noalias %data) {
; VF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF2-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br label %[[EXIT:.*]]
+; VF2-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; VF2: [[EXIT]]:
-; VF2-NEXT: ret void
;
; VF4-LABEL: define void @load_store_interleave_group(
; VF4-SAME: ptr noalias [[DATA:%.*]]) {
@@ -87,11 +86,10 @@ define void @load_store_interleave_group_different_objecs(ptr noalias %src, ptr
; VF2-NEXT: store <2 x i64> [[WIDE_LOAD1]], ptr [[TMP7]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP6]], 1
; VF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br label %[[EXIT:.*]]
+; VF2-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; VF2: [[EXIT]]:
-; VF2-NEXT: ret void
;
; VF4-LABEL: define void @load_store_interleave_group_different_objecs(
; VF4-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
@@ -223,11 +221,10 @@ define void @same_constant_store_interleave_group(i64 %x, ptr noalias %dst) {
; VF2-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VF2-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br label %[[EXIT:.*]]
+; VF2-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; VF2: [[EXIT]]:
-; VF2-NEXT: ret void
;
; VF4-LABEL: define void @same_constant_store_interleave_group(
; VF4-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
@@ -341,11 +338,10 @@ define void @same_live_in_store_interleave_group(i64 %x, ptr noalias %dst) {
; VF2-NEXT: store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br label %[[EXIT:.*]]
+; VF2-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; VF2: [[EXIT]]:
-; VF2-NEXT: ret void
;
; VF4-LABEL: define void @same_live_in_store_interleave_group(
; VF4-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
@@ -476,11 +472,10 @@ define void @single_uniform_load_store_interleave_group(ptr noalias %src, ptr no
; VF2-NEXT: store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VF2-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br label %[[EXIT:.*]]
+; VF2-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; VF2: [[EXIT]]:
-; VF2-NEXT: ret void
;
; VF4-LABEL: define void @single_uniform_load_store_interleave_group(
; VF4-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
@@ -620,11 +615,10 @@ define void @multiple_store_groups_storing_same_load_group(ptr noalias %A, ptr n
; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
-; VF2-NEXT: br label %[[EXIT:.*]]
+; VF2-NEXT: br i1 true, [[EXIT1:label %.*]], label %[[EXIT:.*]]
; VF2: [[EXIT]]:
-; VF2-NEXT: ret void
;
; VF4-LABEL: define void @multiple_store_groups_storing_same_load_group(
; VF4-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
index ea49a27b363db..86c4a370c0427 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll
@@ -42,8 +42,7 @@ define void @load_store_interleave_group(ptr noalias %data) {
; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; EPILOGUE: [[VECTOR_PH]]:
; EPILOGUE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; EPILOGUE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]
+; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP2]]
; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
; EPILOGUE: [[VECTOR_BODY]]:
@@ -141,8 +140,7 @@ define void @load_store_interleave_group_i32(ptr noalias %data) {
; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; EPILOGUE: [[VECTOR_PH]]:
; EPILOGUE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; EPILOGUE-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
-; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]
+; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP2]]
; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
; EPILOGUE: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
index cfa601469464f..1921b7346ecf8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
@@ -15,8 +15,8 @@ define void @test_4xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
; CHECK: [[VECTOR_PH1]]:
-; CHECK-NEXT: [[N_MOD_VF1:%.*]] = urem i64 [[N]], 16
-; CHECK-NEXT: [[N_VEC1:%.*]] = sub i64 [[N]], [[N_MOD_VF1]]
+; CHECK-NEXT: [[TMP14:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC2:%.*]] = sub i64 [[N]], [[TMP14]]
; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
; CHECK: [[VECTOR_BODY1]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY1]] ]
@@ -56,18 +56,16 @@ define void @test_4xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP13]], align 8
; CHECK-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP23]], align 8
; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC1]]
-; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC2]]
+; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK1]]:
-; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC1]]
+; CHECK-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N]], [[N_VEC2]]
; CHECK-NEXT: br i1 [[CMP_N1]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF1]], 4
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP14]], 4
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC2]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -80,13 +78,12 @@ define void @test_4xi64(ptr noalias %data, ptr noalias %factor, i64 noundef %n)
; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP3]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ [[N_VEC1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N]], %[[MIDDLE_BLOCK]] ], [ [[N_VEC2]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -761,6 +758,121 @@ loop:
exit:
ret void
}
+
+; Test with induction live-out to verify exit value is correctly computed.
+define i64 @test_4xi64_induction_live_out(ptr noalias %data, ptr noalias %factor, i64 noundef %n) {
+; CHECK-LABEL: define i64 @test_4xi64_induction_live_out(
+; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT8]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[INDEX]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[TMP2]], i32 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP14]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP16:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT5]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP17:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT7]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul <4 x i64> [[BROADCAST_SPLAT9]], [[WIDE_LOAD3]]
+; CHECK-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP11]], align 8
+; CHECK-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP12]], align 8
+; CHECK-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP13]], align 8
+; CHECK-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP14]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]
+; CHECK-NEXT: [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[DATA_0:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV]], i32 0
+; CHECK-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
+; CHECK-NEXT: [[MUL_0:%.*]] = mul i64 [[L_FACTOR]], [[L_0]]
+; CHECK-NEXT: store i64 [[MUL_0]], ptr [[DATA_0]], align 8
+; CHECK-NEXT: [[DATA_1:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV]], i32 1
+; CHECK-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8
+; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[L_FACTOR]], [[L_1]]
+; CHECK-NEXT: store i64 [[MUL_1]], ptr [[DATA_1]], align 8
+; CHECK-NEXT: [[DATA_2:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV]], i32 2
+; CHECK-NEXT: [[L_2:%.*]] = load i64, ptr [[DATA_2]], align 8
+; CHECK-NEXT: [[MUL_2:%.*]] = mul i64 [[L_FACTOR]], [[L_2]]
+; CHECK-NEXT: store i64 [[MUL_2]], ptr [[DATA_2]], align 8
+; CHECK-NEXT: [[DATA_3:%.*]] = getelementptr inbounds { i64, i64, i64, i64 }, ptr [[DATA]], i64 [[IV]], i32 3
+; CHECK-NEXT: [[L_3:%.*]] = load i64, ptr [[DATA_3]], align 8
+; CHECK-NEXT: [[MUL_3:%.*]] = mul i64 [[L_FACTOR]], [[L_3]]
+; CHECK-NEXT: store i64 [[MUL_3]], ptr [[DATA_3]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i64 [ [[IV_NEXT]], %[[LOOP]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[IV_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %arrayidx = getelementptr inbounds i64, ptr %factor, i64 %iv
+ %l.factor = load i64, ptr %arrayidx, align 8
+ %data.0 = getelementptr inbounds { i64 , i64, i64, i64 }, ptr %data, i64 %iv, i32 0
+ %l.0 = load i64, ptr %data.0, align 8
+ %mul.0 = mul i64 %l.factor, %l.0
+ store i64 %mul.0, ptr %data.0, align 8
+ %data.1 = getelementptr inbounds { i64 , i64, i64, i64 }, ptr %data, i64 %iv, i32 1
+ %l.1 = load i64, ptr %data.1, align 8
+ %mul.1 = mul i64 %l.factor, %l.1
+ store i64 %mul.1, ptr %data.1, align 8
+ %data.2 = getelementptr inbounds { i64 , i64, i64, i64 }, ptr %data, i64 %iv, i32 2
+ %l.2 = load i64, ptr %data.2, align 8
+ %mul.2 = mul i64 %l.factor, %l.2
+ store i64 %mul.2, ptr %data.2, align 8
+ %data.3 = getelementptr inbounds { i64 , i64, i64, i64 }, ptr %data, i64 %iv, i32 3
+ %l.3 = load i64, ptr %data.3, align 8
+ %mul.3 = mul i64 %l.factor, %l.3
+ store i64 %mul.3, ptr %data.3, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i64 %iv.next
+}
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -782,4 +894,6 @@ exit:
; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}
; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META1]]}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]], [[META2]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META2]], [[META1]]}
;.
>From de680f9503d1fd229d68e582fe71888bee396282 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 17 Feb 2026 19:11:35 +0000
Subject: [PATCH 2/2] [VPlan] Prevent uses of materialized VPSymbolicValues.
After VPSymbolicValues (like VF and VFxUF) are materialized via
replaceAllUsesWith, they should not be accessed again. This patch:
1. Tracks materialization state in VPSymbolicValue.
2. Asserts if the materialized VPValue is used again. Currently it
adds asserts to various member functions, preventing calling them
on materialized symbolic values.
Note that this still allows some uses (e.g. comparing VPSymbolicValue
references or pointers), but this should be relatively harmless given
that it is impossible to (re-)add any users. If we want to further
tighten the checks, we could add asserts to the accessors or override
operator&, but that will require more changes and not add much extra
guards I think.
Depends on https://github.com/llvm/llvm-project/pull/182146 to fix a
current access violation (included in PR).
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 3 +
llvm/lib/Transforms/Vectorize/VPlan.h | 8 +--
.../Transforms/Vectorize/VPlanAnalysis.cpp | 3 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 18 ++++--
llvm/lib/Transforms/Vectorize/VPlanValue.h | 56 +++++++++++++++++--
.../Transforms/Vectorize/VPlanTest.cpp | 43 ++++++++++++++
6 files changed, 114 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index f233f0dc1b025..e1ab8f6178cb7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1402,11 +1402,14 @@ bool VPValue::isDefinedOutsideLoopRegions() const {
}
void VPValue::replaceAllUsesWith(VPValue *New) {
replaceUsesWithIf(New, [](VPUser &, unsigned) { return true; });
+ if (auto *SV = dyn_cast<VPSymbolicValue>(this))
+ SV->markMaterialized();
}
void VPValue::replaceUsesWithIf(
VPValue *New,
llvm::function_ref<bool(VPUser &U, unsigned Idx)> ShouldReplace) {
+ assertNotMaterialized();
// Note that this early exit is required for correctness; the implementation
// below relies on the number of users for this VPValue to decrease, which
// isn't the case if this == New.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index e05da74125d1c..a12ded1580486 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4668,14 +4668,14 @@ class VPlan {
VPSymbolicValue &getVectorTripCount() { return VectorTripCount; }
/// Returns the VF of the vector loop region.
- VPValue &getVF() { return VF; };
- const VPValue &getVF() const { return VF; };
+ VPSymbolicValue &getVF() { return VF; };
+ const VPSymbolicValue &getVF() const { return VF; };
/// Returns the UF of the vector loop region.
- VPValue &getUF() { return UF; };
+ VPSymbolicValue &getUF() { return UF; };
/// Returns VF * UF of the vector loop region.
- VPValue &getVFxUF() { return VFxUF; }
+ VPSymbolicValue &getVFxUF() { return VFxUF; }
LLVMContext &getContext() const {
return getScalarHeader()->getIRBasicBlock()->getContext();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 4b744b9128171..54c02bf3156b9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -433,7 +433,8 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
// the loop (not including non-recipe values such as arguments and
// constants).
SmallSetVector<VPValue *, 8> LoopInvariants;
- LoopInvariants.insert(&Plan.getVectorTripCount());
+ if (Plan.getVectorTripCount().getNumUsers() > 0)
+ LoopInvariants.insert(&Plan.getVectorTripCount());
// We scan the loop in a topological order in order and assign a number to
// each recipe. We use RPO to ensure that defs are met before their users. We
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c78895b53e41d..52325ee43e8da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4887,6 +4887,9 @@ void VPlanTransforms::materializeConstantVectorTripCount(
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
VPValue *TC = Plan.getTripCount();
+ if (TC->getNumUsers() == 0)
+ return;
+
// Skip cases for which the trip count may be non-trivial to materialize.
// I.e., when a scalar tail is absent - due to tail folding, or when a scalar
// tail is required.
@@ -5078,12 +5081,19 @@ void VPlanTransforms::materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
ElementCount VFEC) {
VPBuilder Builder(VectorPH, VectorPH->begin());
Type *TCTy = VPTypeAnalysis(Plan).inferScalarType(Plan.getTripCount());
- VPValue &VF = Plan.getVF();
- VPValue &VFxUF = Plan.getVFxUF();
VPValue *UF =
Plan.getOrAddLiveIn(ConstantInt::get(TCTy, Plan.getConcreteUF()));
Plan.getUF().replaceAllUsesWith(UF);
+ // If VF and VFxUF are already materialized, there's nothing more to do.
+ if (Plan.getVF().isMaterialized()) {
+ assert(Plan.getVFxUF().isMaterialized() &&
+ "VF and VFxUF must be materialized together");
+ return;
+ }
+
+ VPValue &VF = Plan.getVF();
+ VPValue &VFxUF = Plan.getVFxUF();
// If there are no users of the runtime VF, compute VFxUF by constant folding
// the multiplication of VF and UF.
if (VF.getNumUsers() == 0) {
@@ -5106,10 +5116,6 @@ void VPlanTransforms::materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
VPValue *MulByUF = Builder.createOverflowingOp(
Instruction::Mul, {RuntimeVF, UF}, {true, false});
VFxUF.replaceAllUsesWith(MulByUF);
-
- assert(Plan.getVF().getNumUsers() == 0 && Plan.getUF().getNumUsers() == 0 &&
- Plan.getVFxUF().getNumUsers() == 0 &&
- "VF, UF, and VFxUF not expected to be used");
}
DenseMap<const SCEV *, Value *>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 4ef78341e0654..b7d923e302369 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -101,11 +101,24 @@ class LLVM_ABI_FOR_TEST VPValue {
void dump() const;
#endif
- unsigned getNumUsers() const { return Users.size(); }
- void addUser(VPUser &User) { Users.push_back(&User); }
+ /// Assert that this VPValue has not been materialized, if it is a
+ /// VPSymbolicValue.
+ void assertNotMaterialized() const;
+
+ unsigned getNumUsers() const {
+ if (Users.empty())
+ return 0;
+ assertNotMaterialized();
+ return Users.size();
+ }
+ void addUser(VPUser &User) {
+ assertNotMaterialized();
+ Users.push_back(&User);
+ }
/// Remove a single \p User from the list of users.
void removeUser(VPUser &User) {
+ assertNotMaterialized();
// The same user can be added multiple times, e.g. because the same VPValue
// is used twice by the same VPUser. Remove a single one.
auto *I = find(Users, &User);
@@ -118,10 +131,22 @@ class LLVM_ABI_FOR_TEST VPValue {
typedef iterator_range<user_iterator> user_range;
typedef iterator_range<const_user_iterator> const_user_range;
- user_iterator user_begin() { return Users.begin(); }
- const_user_iterator user_begin() const { return Users.begin(); }
- user_iterator user_end() { return Users.end(); }
- const_user_iterator user_end() const { return Users.end(); }
+ user_iterator user_begin() {
+ assertNotMaterialized();
+ return Users.begin();
+ }
+ const_user_iterator user_begin() const {
+ assertNotMaterialized();
+ return Users.begin();
+ }
+ user_iterator user_end() {
+ assertNotMaterialized();
+ return Users.end();
+ }
+ const_user_iterator user_end() const {
+ assertNotMaterialized();
+ return Users.end();
+ }
user_range users() { return user_range(user_begin(), user_end()); }
const_user_range users() const {
return const_user_range(user_begin(), user_end());
@@ -226,6 +251,19 @@ struct VPSymbolicValue : public VPValue {
static bool classof(const VPValue *V) {
return V->getVPValueID() == VPVSymbolicSC;
}
+
+#if !defined(NDEBUG)
+ /// Returns true if this symbolic value has been materialized.
+ bool isMaterialized() const { return Materialized; }
+
+ /// Mark this symbolic value as materialized.
+ void markMaterialized() { Materialized = true; }
+
+private:
+ /// Track whether this symbolic value has been materialized (replaced).
+ /// After materialization, accessing users should trigger an assertion.
+ bool Materialized = false;
+#endif
};
/// A VPValue defined by a recipe that produces one or more values.
@@ -427,6 +465,12 @@ class VPDef {
unsigned getNumDefinedValues() const { return DefinedValues.size(); }
};
+inline void VPValue::assertNotMaterialized() const {
+ assert((!isa<VPSymbolicValue>(this) ||
+ !cast<VPSymbolicValue>(this)->isMaterialized()) &&
+ "accessing materialized symbolic value");
+}
+
} // namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_VALUE_H
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 254755bb2bf4c..e526114589913 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1745,5 +1745,48 @@ TEST_F(VPRecipeTest, CastToVPSingleDefRecipe) {
// TODO: check other VPSingleDefRecipes.
}
+TEST_F(VPInstructionTest, VPSymbolicValueMaterialization) {
+ IntegerType *Int64 = IntegerType::get(C, 64);
+ VPlan &Plan = getPlan();
+
+ // Initially, VF is not materialized.
+ EXPECT_FALSE(Plan.getVF().isMaterialized());
+
+ // Create a recipe that uses VF.
+ VPValue *VF = &Plan.getVF();
+ VPInstruction *I = new VPInstruction(VPInstruction::StepVector, {});
+ VPBasicBlock &VPBB = *Plan.createVPBasicBlock("");
+ VPBB.appendRecipe(I);
+ I->addOperand(VF);
+
+ // Replace VF with a constant.
+ VPValue *Const1 = Plan.getOrAddLiveIn(ConstantInt::get(Int64, 1));
+ VF->replaceAllUsesWith(Const1);
+
+ // Now VF should be materialized.
+ EXPECT_TRUE(Plan.getVF().isMaterialized());
+}
+
+#if GTEST_HAS_DEATH_TEST
+#ifndef NDEBUG
+TEST_F(VPInstructionTest, VPSymbolicValueAddUserAfterMaterialization) {
+ IntegerType *Int64 = IntegerType::get(C, 64);
+ VPlan &Plan = getPlan();
+
+ // Materialize VF by replacing all uses.
+ VPValue *VF = &Plan.getVF();
+ VPValue *Const1 = Plan.getOrAddLiveIn(ConstantInt::get(Int64, 1));
+ VF->replaceAllUsesWith(Const1);
+ EXPECT_TRUE(Plan.getVF().isMaterialized());
+
+ // Adding a new user to a materialized value should crash.
+ VPInstruction *I = new VPInstruction(VPInstruction::StepVector, {});
+ VPBasicBlock &VPBB = *Plan.createVPBasicBlock("");
+ VPBB.appendRecipe(I);
+ EXPECT_DEATH(I->addOperand(VF), "accessing materialized symbolic value");
+}
+#endif
+#endif
+
} // namespace
} // namespace llvm
More information about the llvm-commits
mailing list