[llvm] [VPlan] Sink single-scalar VPI to replicate regions (PR #208906)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 10:06:43 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/208906
>From 464e5bc40b0337e2148ae064707ee39d647baa95 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sat, 11 Jul 2026 12:32:25 +0100
Subject: [PATCH] [VPlan] Sink single-scalar VPI to replicate regions
The newly-introduced scalar-casts are sunk. The patch also generalizes
the existing special cases correctly using
vputils::doesGeneratePerAllLanes.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 21 ++++++++++---------
llvm/test/Transforms/LoopVectorize/as_cast.ll | 16 +++++++-------
.../LoopVectorize/cast-induction.ll | 4 ++--
3 files changed, 21 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 0f76041a9cd7c..baf1716c078cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -334,22 +334,23 @@ static bool sinkScalarOperands(VPlan &Plan) {
SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
VPBasicBlock *SinkTo, VPValue *Op) {
- auto *Candidate =
- dyn_cast_or_null<VPSingleDefRecipe>(Op->getDefiningRecipe());
- if (!Candidate)
- return;
-
- // We only know how to sink VPReplicateRecipes and VPScalarIVStepsRecipes
- // for now.
- if (!isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(Candidate))
+ auto *Candidate = dyn_cast<VPSingleDefRecipe>(Op);
+ if (!isa_and_nonnull<VPReplicateRecipe, VPScalarIVStepsRecipe,
+ VPInstruction>(Candidate))
return;
if (Candidate->getParent() == SinkTo ||
+ all_of(Candidate->operands(),
+ [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); }) ||
vputils::cannotHoistOrSinkRecipe(*Candidate, /*Sinking=*/true))
return;
- if (auto *RepR = dyn_cast<VPReplicateRecipe>(Candidate))
- if (!ScalarVFOnly && RepR->isSingleScalar())
+ if (!ScalarVFOnly && !vputils::doesGeneratePerAllLanes(Candidate))
+ return;
+
+ // Only single-scalar VPInstructions can be sunk.
+ if (auto *VPI = dyn_cast<VPInstruction>(Candidate))
+ if (!vputils::isSingleScalar(VPI))
return;
WorkList.insert({SinkTo, Candidate});
diff --git a/llvm/test/Transforms/LoopVectorize/as_cast.ll b/llvm/test/Transforms/LoopVectorize/as_cast.ll
index 7d93501da4411..81b09121f4077 100644
--- a/llvm/test/Transforms/LoopVectorize/as_cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/as_cast.ll
@@ -64,20 +64,20 @@ define void @loop_varying_as_cast(ptr addrspace(1) %in) {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[INDEX]], 6
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[TMP0]], 6
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[IN]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[IN]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[TMP4]] to ptr
-; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr addrspace(1) [[TMP7]] to ptr
; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[IN]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr addrspace(1) [[TMP7]] to ptr
+; CHECK-NEXT: store i64 [[TMP6]], ptr [[TMP8]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_IF1]]:
-; CHECK-NEXT: store i64 [[TMP6]], ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[IN]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = addrspacecast ptr addrspace(1) [[TMP11]] to ptr
+; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP12]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_CONTINUE2]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/cast-induction.ll b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
index a0e2337a4a4cf..b7cbf0ff57787 100644
--- a/llvm/test/Transforms/LoopVectorize/cast-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
@@ -276,16 +276,16 @@ define void @cast_induction_tail_folding(ptr %A) {
; IC2-NEXT: [[INDEX1:%.*]] = add i32 [[INDEX]], 1
; IC2-NEXT: [[TMP2:%.*]] = icmp ule i32 [[INDEX]], 2
; IC2-NEXT: [[TMP3:%.*]] = icmp ule i32 [[INDEX1]], 2
-; IC2-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64
-; IC2-NEXT: [[TMP6:%.*]] = sext i32 [[INDEX1]] to i64
; IC2-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; IC2: [[PRED_STORE_IF]]:
+; IC2-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64
; IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
; IC2-NEXT: store i32 [[INDEX]], ptr [[TMP5]], align 4
; IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
; IC2: [[PRED_STORE_CONTINUE]]:
; IC2-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; IC2: [[PRED_STORE_IF1]]:
+; IC2-NEXT: [[TMP6:%.*]] = sext i32 [[INDEX1]] to i64
; IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; IC2-NEXT: store i32 [[INDEX1]], ptr [[TMP7]], align 4
; IC2-NEXT: br label %[[PRED_STORE_CONTINUE2]]
More information about the llvm-commits
mailing list