[llvm] [VPlan] Deep-traverse in narrowToSingleScalars (PR #194680)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 01:14:59 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/194680
>From bffbef7c78eeca4d08381f2e779b56eef97efc48 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 28 Apr 2026 18:01:08 +0100
Subject: [PATCH 1/2] [VPlan] Deep-traverse in narrowToSingleScalars
In VPlanUnroll, cloneForLane now handles single-scalar replicates in
replicate regions, and the limitation can be lifted.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 +-----
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 9 +--------
.../LoopVectorize/vplan-widen-select-instruction.ll | 8 +++-----
3 files changed, 5 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3dadd9253ad57..bcdb91a54e305 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1849,12 +1849,8 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
if (Plan.hasScalarVFOnly())
return;
- // Try to narrow wide and replicating recipes to single scalar recipes,
- // based on VPlan analysis. Only process blocks in the loop region for now,
- // without traversing into nested regions, as recipes in replicate regions
- // cannot be converted yet.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+ vp_depth_first_deep(Plan.getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe>(&R))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 3327c4b188bb3..8858ba44a1d7b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -383,16 +383,9 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(VPV))
return true;
- if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
- const VPRegionBlock *RegionOfR = Rep->getRegion();
- // Don't consider recipes in replicate regions as uniform yet; their first
- // lane cannot be accessed when executing the replicate region for other
- // lanes.
- if (RegionOfR && RegionOfR->isReplicator())
- return false;
+ if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV))
return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
all_of(Rep->operands(), isSingleScalar));
- }
if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll b/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
index 9f1020fb51c79..75f56a74986c4 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
@@ -16,10 +16,6 @@ define void @loop_invariant_select(ptr noalias nocapture %out, i1 %select, doubl
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
@@ -28,7 +24,9 @@ define void @loop_invariant_select(ptr noalias nocapture %out, i1 %select, doubl
; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]
; CHECK: [[FOR2_HEADER3]]:
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP2:%.*]], %[[FOR2_HEADER3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SELECT]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[SELECT]], double [[A]], double [[B]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP1]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))
; CHECK-NEXT: [[TMP2]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[TMP2]], splat (i64 10000)
>From 63c98c7e7c646132589380204b94cad9040d3be4 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 29 Apr 2026 15:00:03 +0100
Subject: [PATCH 2/2] [VPlan] Revert incorrect change
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 8858ba44a1d7b..3327c4b188bb3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -383,9 +383,16 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(VPV))
return true;
- if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV))
+ if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
+ const VPRegionBlock *RegionOfR = Rep->getRegion();
+ // Don't consider recipes in replicate regions as uniform yet; their first
+ // lane cannot be accessed when executing the replicate region for other
+ // lanes.
+ if (RegionOfR && RegionOfR->isReplicator())
+ return false;
return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
all_of(Rep->operands(), isSingleScalar));
+ }
if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
More information about the llvm-commits
mailing list