[llvm] [VPlan] Scalarize to first-lane-only directly on VPlan (PR #184267)
Andrei Elovikov via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 09:18:43 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/184267
>From 1c504b3dbf3b18272ba912f2da60a3648a4d77e7 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 27 Feb 2026 11:02:19 -0800
Subject: [PATCH 01/19] [VPlan] Scalarize to first-lane-only directly on VPlan
This is needed to enable subsequent https://github.com/llvm/llvm-project/pull/182595.
I don't think we can fully port all scalarization logic from the legacy
path to VPlan-based right now because that would require us to introduce
interleave groups much earlier in VPlan pipeline, and without that we
can't really `assert` this new decision matches the previous CM-based
one. And without those `assert`s it's really hard to ensure we properly
port all the previous logic.
As such, I decided just to implement something much simpler that would
be enough for #182595. However, we perform this transformation before
delegating to the old CM-based decision, so it **is** effective
immediately and taking precedence even for consecutive loads/stores
right away.
Depends on https://github.com/llvm/llvm-project/pull/182592 but is stacked on
top of https://github.com/llvm/llvm-project/pull/182594 to enable linear
stacking for https://github.com/llvm/llvm-project/pull/182595.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 65 +++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 6 ++
.../LoopVectorize/AArch64/binop-costs.ll | 10 +--
.../VPlan/vplan-print-after-all.ll | 1 +
.../X86/drop-poison-generating-flags.ll | 4 +-
.../Transforms/LoopVectorize/X86/funclet.ll | 6 +-
llvm/test/Transforms/LoopVectorize/pr37248.ll | 2 +-
.../runtime-check-needed-but-empty.ll | 2 +-
9 files changed, 88 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2f70238415beb..6fee8c3476723 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7107,6 +7107,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeMemOpWideningDecisions, *Plan,
Range, RecipeBuilder);
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeScalarizationDecisions, *Plan,
+ Range, RecipeBuilder);
+
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
// Convert input VPInstructions to widened recipes.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5e6d1bbcd5a7c..fd1bf1fb5537d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6405,3 +6405,68 @@ void VPlanTransforms::makeMemOpWideningDecisions(
ReplaceWith(Recipe);
}
}
+
+void VPlanTransforms::makeScalarizationDecisions(
+ VPlan &Plan, VFRange &Range, VPRecipeBuilder &RecipeBuilder) {
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) { return VF.isScalar(); }, Range))
+ return;
+
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ post_order<VPBlockShallowTraversalWrapper<VPBlockBase *>>(
+ HeaderVPBB))) {
+ for (VPRecipeBase &R :
+ make_early_inc_range(make_range(VPBB->rbegin(), VPBB->rend()))) {
+ auto *VPI = dyn_cast<VPInstruction>(&R);
+ if (!VPI)
+ continue;
+
+ auto *I = cast_or_null<Instruction>(VPI->getUnderlyingValue());
+ if (!I)
+ // Wouldn't be able to create a `VPReplicateRecipe` anyway.
+ continue;
+
+ bool CanTransformToFirstLaneOnly = [&]() {
+ if (VPI->mayHaveSideEffects())
+ return false;
+
+ if (is_contained({Instruction::SDiv, Instruction::UDiv,
+ Instruction::SRem, Instruction::URem},
+ VPI->getOpcode()) &&
+ VPI->getMask())
+ return false;
+
+ // Avoid rewriting IV increment as that interferes with
+ // `removeRedundantCanonicalIVs`.
+ if (VPI->getOpcode() == Instruction::Add &&
+ any_of(VPI->operands(),
+ [&](auto *Op) { return isa<VPWidenInductionRecipe>(Op); }))
+ return false;
+
+ if (!all_of(VPI->users(), [&](auto *U) {
+ // TODO: This "ScalarCast" is bonkers...
+ if (VPI->isScalarCast() && isa<VPWidenGEPRecipe>(U))
+ return false;
+
+ return U->usesFirstLaneOnly(VPI);
+ }))
+ return false;
+
+ return true;
+ }();
+
+ if (CanTransformToFirstLaneOnly) {
+ auto *Recipe =
+ new VPReplicateRecipe(I, VPI->operandsWithoutMask(), true, nullptr,
+ *VPI, *VPI, VPI->getDebugLoc());
+ Recipe->insertBefore(VPI);
+ VPI->replaceAllUsesWith(Recipe);
+ RecipeBuilder.setRecipe(I, Recipe);
+ VPI->eraseFromParent();
+ continue;
+ }
+ }
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index b7071462ab8ac..0d4aac11a798b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -532,6 +532,12 @@ struct VPlanTransforms {
/// recipes. Non load/store input instructions are left unchanged.
static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder);
+
+ /// Make VPlan-based scalarization decision prior to delegating to the ones
+ /// made by the legacy CM. Only transforms "usesFirstLaneOnly` def-use chains
+ /// enabled by prior widening of consecutive memory operations for now.
+ static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range,
+ VPRecipeBuilder &RecipeBuilder);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll
index 8e4c6d470c9be..343276bc9fe30 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/binop-costs.ll
@@ -8,14 +8,14 @@ target triple = "arm64-apple-macosx"
define void @udiv_rhs_opt_cost(ptr %dst) #0 {
; CHECK-LABEL: 'udiv_rhs_opt_cost'
; CHECK: LV: Found an estimated cost of 5 for VF 1 For instruction: %div = udiv i8 %iv.trunc, 3
-; CHECK: Cost of 5 for VF 2: CLONE ir<%div> = udiv vp<[[VP7:%[0-9]+]]>, ir<3>
+; CHECK: Cost of 5 for VF 2: CLONE ir<%div> = udiv ir<%iv.trunc>, ir<3>
; CHECK: Cost of 0 for VF 2: IR %div = udiv i8 %iv.trunc, 3
-; CHECK: Cost of 5 for VF 4: CLONE ir<%div> = udiv vp<[[VP7]]>, ir<3>
+; CHECK: Cost of 5 for VF 4: CLONE ir<%div> = udiv ir<%iv.trunc>, ir<3>
; CHECK: Cost of 0 for VF 4: IR %div = udiv i8 %iv.trunc, 3
-; CHECK: Cost of 5 for VF vscale x 1: CLONE ir<%div> = udiv vp<[[VP7]]>, ir<3>
-; CHECK: Cost of 5 for VF vscale x 2: CLONE ir<%div> = udiv vp<[[VP7]]>, ir<3>
+; CHECK: Cost of 5 for VF vscale x 1: CLONE ir<%div> = udiv ir<%iv.trunc>, ir<3>
+; CHECK: Cost of 5 for VF vscale x 2: CLONE ir<%div> = udiv ir<%iv.trunc>, ir<3>
; CHECK: Cost of 0 for VF vscale x 2: IR %div = udiv i8 %iv.trunc, 3
-; CHECK: Cost of 5 for VF vscale x 4: CLONE ir<%div> = udiv vp<[[VP7]]>, ir<3>
+; CHECK: Cost of 5 for VF vscale x 4: CLONE ir<%div> = udiv ir<%iv.trunc>, ir<3>
; CHECK: Cost of 0 for VF vscale x 4: IR %div = udiv i8 %iv.trunc, 3
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index fe8bbcb11ff88..def4c032d9f83 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -12,6 +12,7 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::introduceMasksAndLinearize
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInLoopReductionRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeMemOpWideningDecisions
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeScalarizationDecisions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimizeFindIVReductions
diff --git a/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
index 4481440bb623c..e4e7cbeea94eb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
@@ -425,9 +425,9 @@ define void @drop_zext_nneg(ptr noalias %p, ptr noalias %p1) #0 {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <4 x i32> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i32> [[VEC_IND]] to <4 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[P]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP0]], <4 x double> poison)
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x double> [[WIDE_MASKED_LOAD]], <4 x double> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/X86/funclet.ll b/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
index 2ccb0428b9269..8d6d169c2d948 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
@@ -2,7 +2,7 @@
target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
target triple = "i686-pc-windows-msvc18.0.0"
-define void @test1() #0 personality ptr @__CxxFrameHandler3 {
+define void @test1(ptr noalias %p) #0 personality ptr @__CxxFrameHandler3 {
entry:
invoke void @_CxxThrowException(ptr null, ptr null)
to label %unreachable unwind label %catch.dispatch
@@ -19,7 +19,9 @@ for.cond.cleanup:
for.body:
%i.07 = phi i32 [ 0, %catch ], [ %inc, %for.body ]
+ %gep = getelementptr double, ptr %p, i32 %i.07
%call = call double @floor(double 1.0) #1 [ "funclet"(token %1) ]
+ store double %call, ptr %gep
%inc = add nuw nsw i32 %i.07, 1
%exitcond = icmp eq i32 %inc, 1024
br i1 %exitcond, label %for.cond.cleanup, label %for.body
@@ -33,7 +35,7 @@ unreachable:
; CHECK-LABEL: define void @test1(
; CHECK: %[[cpad:.*]] = catchpad within {{.*}} [ptr null, i32 64, ptr null]
-; CHECK: call <16 x double> @llvm.floor.v16f64(<16 x double> {{.*}}) [ "funclet"(token %[[cpad]]) ]
+; CHECK: call <2 x double> @llvm.floor.v2f64(<2 x double> {{.*}}) [ "funclet"(token %[[cpad]]) ]
declare x86_stdcallcc void @_CxxThrowException(ptr, ptr)
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
index 60736f5f126c6..03e3d1f222399 100644
--- a/llvm/test/Transforms/LoopVectorize/pr37248.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -43,7 +43,6 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]]
-; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: store i32 10, ptr [[B]], align 1
@@ -54,6 +53,7 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
; CHECK-NEXT: store i32 10, ptr [[B]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE3]]:
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP13]]
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[TMP15]], i64 -1
; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP17]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll
index dd7a8a87a921b..7548a783fb4dd 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-needed-but-empty.ll
@@ -13,12 +13,12 @@ define void @test(ptr %A, i32 %x) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[OFFSET_IDX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[OFFSET_IDX]], 1
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[OFFSET_IDX]] to i32
; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]]
; CHECK-NEXT: store <4 x float> [[WIDE_LOAD]], ptr [[TMP9]], align 4
>From 385003306f77358f1bec4a3d852ac6e1b86e6bad Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 3 Mar 2026 09:48:37 -0800
Subject: [PATCH 02/19] Use `reverse`/`IsaPred`
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index fd1bf1fb5537d..7aaf8310a6cb9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6417,8 +6417,7 @@ void VPlanTransforms::makeScalarizationDecisions(
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
post_order<VPBlockShallowTraversalWrapper<VPBlockBase *>>(
HeaderVPBB))) {
- for (VPRecipeBase &R :
- make_early_inc_range(make_range(VPBB->rbegin(), VPBB->rend()))) {
+ for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
continue;
@@ -6441,8 +6440,7 @@ void VPlanTransforms::makeScalarizationDecisions(
// Avoid rewriting IV increment as that interferes with
// `removeRedundantCanonicalIVs`.
if (VPI->getOpcode() == Instruction::Add &&
- any_of(VPI->operands(),
- [&](auto *Op) { return isa<VPWidenInductionRecipe>(Op); }))
+ any_of(VPI->operands(), IsaPred<VPWidenInductionRecipe>))
return false;
if (!all_of(VPI->users(), [&](auto *U) {
>From ada4b13029429ddaa29e3f9f527bcc124dea402e Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 3 Mar 2026 09:57:48 -0800
Subject: [PATCH 03/19] Don't pass RecipeBuilder
Legacy calls `setRecipe` on all processed recipes but really queries `getRecipe`
for memory operations only, that we don't touch in the scalarization as that
happens after all memory recipes has been processed.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 +---
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 3 +--
3 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6fee8c3476723..d983fde028dc3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7108,7 +7108,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
Range, RecipeBuilder);
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeScalarizationDecisions, *Plan,
- Range, RecipeBuilder);
+ Range);
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7aaf8310a6cb9..a8726134e789c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6406,8 +6406,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(
}
}
-void VPlanTransforms::makeScalarizationDecisions(
- VPlan &Plan, VFRange &Range, VPRecipeBuilder &RecipeBuilder) {
+void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) { return VF.isScalar(); }, Range))
return;
@@ -6461,7 +6460,6 @@ void VPlanTransforms::makeScalarizationDecisions(
*VPI, *VPI, VPI->getDebugLoc());
Recipe->insertBefore(VPI);
VPI->replaceAllUsesWith(Recipe);
- RecipeBuilder.setRecipe(I, Recipe);
VPI->eraseFromParent();
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 0d4aac11a798b..aeaf075dbc7bf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -536,8 +536,7 @@ struct VPlanTransforms {
/// Make VPlan-based scalarization decision prior to delegating to the ones
/// made by the legacy CM. Only transforms "usesFirstLaneOnly` def-use chains
/// enabled by prior widening of consecutive memory operations for now.
- static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range,
- VPRecipeBuilder &RecipeBuilder);
+ static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range);
};
} // namespace llvm
>From ff7e66d53726d472a6c5d87cfca9183e29039dde Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 3 Mar 2026 13:08:30 -0800
Subject: [PATCH 04/19] Just use `vputils::onlyFirstLaneUsed`
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a8726134e789c..068ee7e6775e0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6442,13 +6442,7 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
any_of(VPI->operands(), IsaPred<VPWidenInductionRecipe>))
return false;
- if (!all_of(VPI->users(), [&](auto *U) {
- // TODO: This "ScalarCast" is bonkers...
- if (VPI->isScalarCast() && isa<VPWidenGEPRecipe>(U))
- return false;
-
- return U->usesFirstLaneOnly(VPI);
- }))
+ if (!vputils::onlyFirstLaneUsed(VPI))
return false;
return true;
>From 7722a1f02a73c2e28b36d4a6b36fdd3567773082 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 15 Apr 2026 09:19:46 -0700
Subject: [PATCH 05/19] Extend post_order's lifetime
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 068ee7e6775e0..3dad455596876 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6413,9 +6413,11 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- post_order<VPBlockShallowTraversalWrapper<VPBlockBase *>>(
- HeaderVPBB))) {
+
+ // Extend lifetime per `llvm::PostOrderTraversal` documentation:
+ auto PO = vp_post_order_shallow(HeaderVPBB);
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(PO)) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
>From c742b3ca847b5435b09cf97435a9513691ad0c94 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 15 Apr 2026 10:39:08 -0700
Subject: [PATCH 06/19] Minor stylistic cleanup
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 24 +++++++++----------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3dad455596876..69d2b01ecee8d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6411,11 +6411,9 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
[&](ElementCount VF) { return VF.isScalar(); }, Range))
return;
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
-
// Extend lifetime per `llvm::PostOrderTraversal` documentation:
- auto PO = vp_post_order_shallow(HeaderVPBB);
+ auto PO =
+ vp_post_order_shallow(Plan.getVectorLoopRegion()->getEntryBasicBlock());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(PO)) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
@@ -6432,6 +6430,8 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (VPI->mayHaveSideEffects())
return false;
+ // We want to drop the mask operand, doing that for integer division
+ // isn't safe.
if (is_contained({Instruction::SDiv, Instruction::UDiv,
Instruction::SRem, Instruction::URem},
VPI->getOpcode()) &&
@@ -6450,15 +6450,15 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
return true;
}();
- if (CanTransformToFirstLaneOnly) {
- auto *Recipe =
- new VPReplicateRecipe(I, VPI->operandsWithoutMask(), true, nullptr,
- *VPI, *VPI, VPI->getDebugLoc());
- Recipe->insertBefore(VPI);
- VPI->replaceAllUsesWith(Recipe);
- VPI->eraseFromParent();
+ if (!CanTransformToFirstLaneOnly)
continue;
- }
+
+ auto *Recipe =
+ new VPReplicateRecipe(I, VPI->operandsWithoutMask(), true, nullptr,
+ *VPI, *VPI, VPI->getDebugLoc());
+ Recipe->insertBefore(VPI);
+ VPI->replaceAllUsesWith(Recipe);
+ VPI->eraseFromParent();
}
}
}
>From e192fc24e9a9474b565545ab07544f79f037cbda Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 16 Apr 2026 15:28:13 -0700
Subject: [PATCH 07/19] Addressing code review comments
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 43 +++++++++----------
1 file changed, 21 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 69d2b01ecee8d..2805708839bb1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6421,36 +6421,35 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (!VPI)
continue;
+ // For now we (effectively) only scalarize to first-lane-only address
+ // computation chain for the memory operations such that
+ // MemOp->usesFirstLaneOnly(MemOpAddressOperand).
+
auto *I = cast_or_null<Instruction>(VPI->getUnderlyingValue());
if (!I)
// Wouldn't be able to create a `VPReplicateRecipe` anyway.
continue;
- bool CanTransformToFirstLaneOnly = [&]() {
- if (VPI->mayHaveSideEffects())
- return false;
-
- // We want to drop the mask operand, doing that for integer division
- // isn't safe.
- if (is_contained({Instruction::SDiv, Instruction::UDiv,
- Instruction::SRem, Instruction::URem},
- VPI->getOpcode()) &&
- VPI->getMask())
- return false;
-
- // Avoid rewriting IV increment as that interferes with
- // `removeRedundantCanonicalIVs`.
- if (VPI->getOpcode() == Instruction::Add &&
- any_of(VPI->operands(), IsaPred<VPWidenInductionRecipe>))
- return false;
+ // If "executing" other lanes produces side-effects we can't avoid them.
+ if (VPI->mayHaveSideEffects())
+ continue;
- if (!vputils::onlyFirstLaneUsed(VPI))
- return false;
+ // We want to drop the mask operand, doing that for integer division
+ // isn't safe if it's predicated.
+ if (is_contained({Instruction::SDiv, Instruction::UDiv, Instruction::SRem,
+ Instruction::URem},
+ VPI->getOpcode()) &&
+ VPI->isMasked())
+ continue;
- return true;
- }();
+ // Avoid rewriting IV increment as that interferes with
+ // `removeRedundantCanonicalIVs`.
+ if (VPI->getOpcode() == Instruction::Add &&
+ any_of(VPI->operands(), IsaPred<VPWidenInductionRecipe>))
+ continue;
- if (!CanTransformToFirstLaneOnly)
+ // Other lanes are needed - can't drop them.
+ if (!vputils::onlyFirstLaneUsed(VPI))
continue;
auto *Recipe =
>From 65370d66900b321c2818d3c6e6bc366045a29a1c Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 20 Apr 2026 10:38:46 -0700
Subject: [PATCH 08/19] Use `post_order` directly as `vp_post_order_shallow`
has been removed
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2805708839bb1..73eb65825ad17 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6412,8 +6412,8 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
return;
// Extend lifetime per `llvm::PostOrderTraversal` documentation:
- auto PO =
- vp_post_order_shallow(Plan.getVectorLoopRegion()->getEntryBasicBlock());
+ auto PO = post_order<VPBlockShallowTraversalWrapper<VPBlockBase *>>(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(PO)) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
>From b9785fc765ebf0063db29ea39026262227940267 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 20 Apr 2026 10:53:27 -0700
Subject: [PATCH 09/19] use VPWidenIntOrFpInductionRecipe for canonical IV
bailouts
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 73eb65825ad17..f0490f0e49e1c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6445,7 +6445,7 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
// Avoid rewriting IV increment as that interferes with
// `removeRedundantCanonicalIVs`.
if (VPI->getOpcode() == Instruction::Add &&
- any_of(VPI->operands(), IsaPred<VPWidenInductionRecipe>))
+ any_of(VPI->operands(), IsaPred<VPWidenIntOrFpInductionRecipe>))
continue;
// Other lanes are needed - can't drop them.
>From 09a556e5c199d8c9c856db023fbf9978a7f42058 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 20 Apr 2026 11:07:49 -0700
Subject: [PATCH 10/19] Update
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
Co-authored-by: Ramkumar Ramachandra <artagnon at tenstorrent.com>
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f0490f0e49e1c..ecfd8d226ab6b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6411,11 +6411,9 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
[&](ElementCount VF) { return VF.isScalar(); }, Range))
return;
- // Extend lifetime per `llvm::PostOrderTraversal` documentation:
- auto PO = post_order<VPBlockShallowTraversalWrapper<VPBlockBase *>>(
- Plan.getVectorLoopRegion()->getEntryBasicBlock());
-
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(PO)) {
+ PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
+ Plan.getEntry());
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(POT)) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
>From 9bb907bae1c6881b5649304bf0787e6c0eecf6d0 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 20 Apr 2026 11:21:46 -0700
Subject: [PATCH 11/19] Add VPRecipeBase::isSafeToSpeculativelyExecute
---
llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++++
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 ++++++++++++++++
.../lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 ++------
3 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 934cca006e91c..b7d78d7ea7ec2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -536,6 +536,10 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Returns true if the recipe may have side-effects.
bool mayHaveSideEffects() const;
+ /// Return true if we can safely execute this recipe unconditionally even if
+ /// it is masked originally.
+ bool isSafeToSpeculativelyExecute() const;
+
/// Returns true for PHI-like recipes.
bool isPhi() const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 25fe37124b017..fee478b184f22 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -225,6 +225,22 @@ bool VPRecipeBase::mayHaveSideEffects() const {
}
}
+bool VPRecipeBase::isSafeToSpeculativelyExecute() const {
+ if (mayHaveSideEffects())
+ return false;
+
+ auto *VPI = dyn_cast<VPInstruction>(this);
+ if (!VPI)
+ return true;
+
+ if (is_contained({Instruction::SDiv, Instruction::UDiv, Instruction::SRem,
+ Instruction::URem},
+ VPI->getOpcode()))
+ return false;
+
+ return true;
+}
+
void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) {
assert(!Parent && "Recipe already in some VPBasicBlock");
assert(InsertPos->getParent() &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ecfd8d226ab6b..77a88bcf3c6f2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6432,12 +6432,8 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (VPI->mayHaveSideEffects())
continue;
- // We want to drop the mask operand, doing that for integer division
- // isn't safe if it's predicated.
- if (is_contained({Instruction::SDiv, Instruction::UDiv, Instruction::SRem,
- Instruction::URem},
- VPI->getOpcode()) &&
- VPI->isMasked())
+ // We want to drop the mask operand, verify we can safely do that.
+ if (VPI->isMasked() && !VPI->isSafeToSpeculativelyExecute())
continue;
// Avoid rewriting IV increment as that interferes with
>From c309f17e58cda968fdba393899cede7e25fe1448 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 20 Apr 2026 14:17:43 -0700
Subject: [PATCH 12/19] More conservative isSafeToSpeculativelyExecute
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 29 ++++++++++++-------
1 file changed, 19 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fee478b184f22..dbcbe27c9aec4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -226,19 +226,28 @@ bool VPRecipeBase::mayHaveSideEffects() const {
}
bool VPRecipeBase::isSafeToSpeculativelyExecute() const {
- if (mayHaveSideEffects())
+ switch (getVPRecipeID()) {
+ default:
return false;
+ case VPInstructionSC: {
+ unsigned Opcode = cast<VPInstruction>(this)->getOpcode();
+ if (Instruction::isCast(Opcode))
+ return true;
- auto *VPI = dyn_cast<VPInstruction>(this);
- if (!VPI)
+ switch (Opcode) {
+ default:
+ return false;
+ case Instruction::Add:
+ case Instruction::Sub:
+ case Instruction::Mul:
+ case Instruction::GetElementPtr:
+ return true;
+ }
+ }
+ case VPVectorPointerSC:
+ case VPVectorEndPointerSC:
return true;
-
- if (is_contained({Instruction::SDiv, Instruction::UDiv, Instruction::SRem,
- Instruction::URem},
- VPI->getOpcode()))
- return false;
-
- return true;
+ }
}
void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) {
>From 0e9ccc1d47f941d5fff3717daec2f843ef4fab62 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 24 Apr 2026 13:38:01 -0700
Subject: [PATCH 13/19] Update test after rebase
---
llvm/test/Transforms/LoopVectorize/extract-value-widen.ll | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/extract-value-widen.ll b/llvm/test/Transforms/LoopVectorize/extract-value-widen.ll
index ecf1ed9441dc4..2e8b99e31a6a1 100644
--- a/llvm/test/Transforms/LoopVectorize/extract-value-widen.ll
+++ b/llvm/test/Transforms/LoopVectorize/extract-value-widen.ll
@@ -12,14 +12,11 @@ define void @func(ptr %p) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> [[VEC_IND]], <2 x i64> splat (i64 1))
-; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i1> } [[TMP0]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[INDEX]], i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { i64, i1 } [[TMP0]], 0
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [8 x i8], ptr [[P]], i64 [[TMP2]]
; CHECK-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP3]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], -9223372036854775808
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
>From 57dcfa38385609fed2aeafd6da27dd89e5a7a7b1 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 28 Apr 2026 09:02:55 -0700
Subject: [PATCH 14/19] Drop Vector[End]Pointer from
isSafeToSpeculativelyExecute
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index dbcbe27c9aec4..dfd7a75bb9b81 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -244,9 +244,6 @@ bool VPRecipeBase::isSafeToSpeculativelyExecute() const {
return true;
}
}
- case VPVectorPointerSC:
- case VPVectorEndPointerSC:
- return true;
}
}
>From 90284ac3c3878cc3d0f0549b10ec09a591fc6408 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 28 Apr 2026 09:06:44 -0700
Subject: [PATCH 15/19] Address code review comments
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 77a88bcf3c6f2..547ccc5b035e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6420,12 +6420,11 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
continue;
// For now we (effectively) only scalarize to first-lane-only address
- // computation chain for the memory operations such that
- // MemOp->usesFirstLaneOnly(MemOpAddressOperand).
+ // computation chain for the memory operations.
auto *I = cast_or_null<Instruction>(VPI->getUnderlyingValue());
+ // Wouldn't be able to create a `VPReplicateRecipe` anyway.
if (!I)
- // Wouldn't be able to create a `VPReplicateRecipe` anyway.
continue;
// If "executing" other lanes produces side-effects we can't avoid them.
@@ -6446,9 +6445,9 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (!vputils::onlyFirstLaneUsed(VPI))
continue;
- auto *Recipe =
- new VPReplicateRecipe(I, VPI->operandsWithoutMask(), true, nullptr,
- *VPI, *VPI, VPI->getDebugLoc());
+ auto *Recipe = new VPReplicateRecipe(
+ I, VPI->operandsWithoutMask(), /*IsSingleScalar=*/true,
+ /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc());
Recipe->insertBefore(VPI);
VPI->replaceAllUsesWith(Recipe);
VPI->eraseFromParent();
>From 474ebacb1626cde32e44be5014c1d26fc62883f0 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 28 Apr 2026 09:25:19 -0700
Subject: [PATCH 16/19] Add suggested test
---
.../make_scalarization_decisions.ll | 43 +++++++++++++++++++
1 file changed, 43 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/make_scalarization_decisions.ll
diff --git a/llvm/test/Transforms/LoopVectorize/make_scalarization_decisions.ll b/llvm/test/Transforms/LoopVectorize/make_scalarization_decisions.ll
new file mode 100644
index 0000000000000..dea758e9114a3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/make_scalarization_decisions.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 | FileCheck %s
+
+define void @preserve_trunc_flags(ptr %p) {
+; CHECK-LABEL: define void @preserve_trunc_flags(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = trunc nuw i64 [[INDEX]] to i32
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [8 x i8], ptr [[P]], i32 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP1]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ ; We can preserve nuw flags in the vectorized code:
+ %trunc = trunc nuw i64 %iv to i32
+ %gep = getelementptr [8 x i8], ptr %p, i32 %trunc
+ %load = load i64, ptr %gep, align 8
+ %add = add i64 %load, 42
+ store i64 %add, ptr %gep, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 100
+ br i1 %ec, label %exit, label %header
+
+exit:
+ ret void
+}
>From 95fbc3667786ca11f4d7096318f8747a541cee48 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 28 Apr 2026 22:30:05 -0700
Subject: [PATCH 17/19] Rename test
---
...scalarization_decisions.ll => make-scalarization-decisions.ll} | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename llvm/test/Transforms/LoopVectorize/{make_scalarization_decisions.ll => make-scalarization-decisions.ll} (100%)
diff --git a/llvm/test/Transforms/LoopVectorize/make_scalarization_decisions.ll b/llvm/test/Transforms/LoopVectorize/make-scalarization-decisions.ll
similarity index 100%
rename from llvm/test/Transforms/LoopVectorize/make_scalarization_decisions.ll
rename to llvm/test/Transforms/LoopVectorize/make-scalarization-decisions.ll
>From 4fa2e19c8f7a3a3cbc25f38de632f859abc2e04c Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 28 Apr 2026 22:32:49 -0700
Subject: [PATCH 18/19] RUN_VPLAN_PASS_NO_VERIFY->RUN_VPLAN_PASS
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d983fde028dc3..d0707834642a4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7104,11 +7104,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan,
VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, Config.CostKind, CM.PSE,
OrigLoop);
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeMemOpWideningDecisions, *Plan,
- Range, RecipeBuilder);
+ RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
+ RecipeBuilder);
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::makeScalarizationDecisions, *Plan,
- Range);
+ RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
// Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
>From 5f6bbb7fec6f41829f81251a491a0fa8a18e3d5e Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 4 May 2026 09:22:42 -0700
Subject: [PATCH 19/19] Address some code review feedback
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index be0a3cb853d05..cac9574a673a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6510,15 +6510,12 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (!VPI)
continue;
- // For now we (effectively) only scalarize to first-lane-only address
- // computation chain for the memory operations.
-
auto *I = cast_or_null<Instruction>(VPI->getUnderlyingValue());
// Wouldn't be able to create a `VPReplicateRecipe` anyway.
if (!I)
continue;
- // If "executing" other lanes produces side-effects we can't avoid them.
+ // If executing other lanes produces side-effects we can't avoid them.
if (VPI->mayHaveSideEffects())
continue;
More information about the llvm-commits
mailing list