[llvm] [SLP]Fix scheduling of copyable bundle with commutative op used outside parent PHI (PR #193566)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 22 11:58:40 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/193566
The previous (V, Op) pair insert was a no-op since V is unique per iteration.
Replace it with a hasOneUse() fast path plus a check that bails only when I
has a user outside the grandparent PHI's Scalars. Uses within the same
vectorized PHI are tracked by the existing dep machinery; an external user
(e.g. a scalar PHI in a different block) is what trips scheduleBlock's
"must be scheduled at this point" assertion.
Fixes #193315.
>From c9a77ea85cbd22b01bf870983d6f89f135a4ce91 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Wed, 22 Apr 2026 11:58:27 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 17 ++++--
.../commutative-copyable-external-phi-use.ll | 55 +++++++++++++++++++
...copyable-used-outside-with-immediate-op.ll | 10 ++--
3 files changed, 74 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/commutative-copyable-external-phi-use.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 949e08501a86d..f904bc9feb5c1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23886,15 +23886,24 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
EI.UserTE->UserTreeIndex.UserTE->State != TreeEntry::SplitVectorize &&
EI.UserTE->UserTreeIndex.UserTE->getOpcode() == Instruction::PHI;
if (IsNonSchedulableWithParentPhiNode) {
- SmallSet<std::pair<Value *, Value *>, 4> Values;
- for (const auto [Idx, V] :
- enumerate(EI.UserTE->UserTreeIndex.UserTE->Scalars)) {
+ ArrayRef<Value *> PhiScalars = EI.UserTE->UserTreeIndex.UserTE->Scalars;
+ for (const auto [Idx, V] : enumerate(PhiScalars)) {
Value *Op = EI.UserTE->UserTreeIndex.UserTE->getOperand(
EI.UserTE->UserTreeIndex.EdgeIdx)[Idx];
auto *I = dyn_cast<Instruction>(Op);
if (!I || !isCommutative(I))
continue;
- if (!Values.insert(std::make_pair(V, Op)).second)
+ // Bail out only when I has a user that is an instruction outside the
+ // grandparent PHI's Scalars. Multiple uses that all land in the same
+ // vectorized PHI are tracked by the existing dependency machinery;
+ // uses outside it (e.g. a scalar PHI in a different block) break the
+ // scheduler's dep accounting for non-schedulable copyable bundles.
+ if (I->hasOneUse())
+ continue;
+ if (any_of(I->users(), [&](User *U) {
+ auto *UI = dyn_cast<Instruction>(U);
+ return UI && !is_contained(PhiScalars, UI);
+ }))
return std::nullopt;
}
} else {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/commutative-copyable-external-phi-use.ll b/llvm/test/Transforms/SLPVectorizer/X86/commutative-copyable-external-phi-use.ll
new file mode 100644
index 0000000000000..263fff4c64fa9
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/commutative-copyable-external-phi-use.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(i32 %arg) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[ARG]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> zeroinitializer, [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl <2 x i32> zeroinitializer, [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP3]], <i32 1, i32 0>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP5]], splat (i32 1)
+; CHECK-NEXT: [[TMP7:%.*]] = ashr <2 x i32> [[TMP5]], splat (i32 1)
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP7]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: br label %[[BB1:.*]]
+; CHECK: [[BB1]]:
+; CHECK-NEXT: br i1 false, label %[[BB2:.*]], label %[[BB4:.*]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP8]], %[[BB1]] ], [ [[TMP14:%.*]], %[[BB4]] ]
+; CHECK-NEXT: ret void
+; CHECK: [[BB4]]:
+; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i32> [ [[TMP4]], %[[BB1]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = or <2 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> [[TMP12]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP14]] = or <2 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT: br label %[[BB2]]
+;
+bb:
+ %lshr = lshr i32 0, %arg
+ %add = add i32 %lshr, 1
+ %shl = shl i32 0, 0
+ %or = or i32 %shl, 0
+ %ashr = ashr i32 %or, 1
+ br label %bb1
+
+bb1:
+ br i1 false, label %bb2, label %bb4
+
+bb2:
+ %phi = phi i32 [ %add, %bb1 ], [ %or8, %bb4 ]
+ %phi3 = phi i32 [ %ashr, %bb1 ], [ %ashr10, %bb4 ]
+ ret void
+
+bb4:
+ %phi5 = phi i32 [ %add, %bb1 ]
+ %phi6 = phi i32 [ %or, %bb1 ]
+ %lshr7 = lshr i32 %phi5, 0
+ %or8 = or i32 %lshr7, 0
+ %or9 = or i32 %phi6, 0
+ %ashr10 = ashr i32 %or9, 0
+ br label %bb2
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/copyable-used-outside-with-immediate-op.ll b/llvm/test/Transforms/SLPVectorizer/X86/copyable-used-outside-with-immediate-op.ll
index d4aef24962313..1f9fc9efab11f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/copyable-used-outside-with-immediate-op.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/copyable-used-outside-with-immediate-op.ll
@@ -13,16 +13,18 @@ define void @test() {
; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ [[TMP0]], %[[BB1]] ]
; CHECK-NEXT: br label %[[BB10]]
; CHECK: [[BB10]]:
+; CHECK-NEXT: [[PHI11:%.*]] = phi i32 [ 0, %[[BB3]] ], [ [[PHI11]], %[[BB14]] ]
; CHECK-NEXT: [[PHI12:%.*]] = phi float [ 0.000000e+00, %[[BB3]] ], [ 0.000000e+00, %[[BB14]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 0, i32 0>, %[[BB3]] ], [ [[TMP7:%.*]], %[[BB14]] ]
+; CHECK-NEXT: [[PHI13:%.*]] = phi i32 [ 0, %[[BB3]] ], [ [[OR15:%.*]], %[[BB14]] ]
; CHECK-NEXT: switch i32 0, label %[[BB14]] [
; CHECK-NEXT: i32 0, label %[[BB1]]
; CHECK-NEXT: ]
; CHECK: [[BB14]]:
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, <4 x i32> [[TMP3]], <4 x i32> <i32 poison, i32 poison, i32 2, i32 6>
-; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[PHI13]], [[PHI11]]
+; CHECK-NEXT: [[OR15]] = or i32 [[OR]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[PHI11]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[OR]], i32 3
; CHECK-NEXT: [[TMP6]] = or <4 x i32> [[TMP5]], <i32 poison, i32 poison, i32 0, i32 0>
-; CHECK-NEXT: [[TMP7]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <4 x i32> <i32 poison, i32 poison, i32 2, i32 7>
; CHECK-NEXT: br i1 false, label %[[BB1]], label %[[BB10]]
;
bb:
More information about the llvm-commits
mailing list