[llvm] c98961e - [SLP]Release copyable operand deps on duplicate parent-phi lanes
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 14:54:45 PDT 2026
Author: Alexey Bataev
Date: 2026-08-13T17:54:40-04:00
New Revision: c98961e3053140b0a4444029425c7e53d73694fa
URL: https://github.com/llvm/llvm-project/commit/c98961e3053140b0a4444029425c7e53d73694fa
DIFF: https://github.com/llvm/llvm-project/commit/c98961e3053140b0a4444029425c7e53d73694fa.diff
LOG: [SLP]Release copyable operand deps on duplicate parent-phi lanes
Non-scheduled entries feeding a phi may repeat a def across lanes. The
scheduler skipped duplicate parent-phi user lanes entirely, so per-column
copyable scheduling data was never released and the copyable bundles never
became ready.
Fixes #216146
Reviewers:
Pull Request: https://github.com/llvm/llvm-project/pull/216186
Added:
llvm/test/Transforms/SLPVectorizer/X86/copyable-operand-dup-parent-phi-user.ll
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 85daf23009400..53816d49de722 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4936,7 +4936,8 @@ class slpvectorizer::BoUpSLP {
[&](Instruction *I, TreeEntry *UserTE, unsigned OpIdx,
SmallDenseSet<std::pair<const ScheduleEntity *, unsigned>>
&Checked,
- bool IsExpandedOperand = false) {
+ bool IsExpandedOperand = false,
+ bool CopyableDepsOnly = false) {
if (!ScheduleCopyableDataMap.empty()) {
const EdgeInfo EI = {UserTE, OpIdx};
if (ScheduleCopyableData *CD =
@@ -4947,6 +4948,8 @@ class slpvectorizer::BoUpSLP {
return;
}
}
+ if (CopyableDepsOnly)
+ return;
auto It = OperandsUses.find(I);
if (It == OperandsUses.end()) {
// Column value may be a peeled intermediate, not a direct
@@ -5017,18 +5020,16 @@ class slpvectorizer::BoUpSLP {
Bundle->getTreeEntry()->hasReassocScalars()) &&
"Missed TreeEntry operands?");
- // Count the number of unique phi nodes, which are the parent for
- // parent entry, and exit, if all the unique phis are processed.
- if (IsNonSchedulableWithParentPhiNode) {
- const TreeEntry *ParentTE =
- Bundle->getTreeEntry()->UserTreeIndex.UserTE;
- Value *User = ParentTE->Scalars[Lane];
- if (!ParentsUniqueUsers.insert(User).second) {
- It = std::find(std::next(It),
- Bundle->getTreeEntry()->Scalars.end(), In);
- continue;
- }
- }
+ // Count the number of unique phi nodes, which are the parent
+ // entry, and handle the non-copyable deps only on the first lane
+ // for each such phi. Copyable deps are counted per operand column
+ // lane and are released on every lane.
+ bool CopyableDepsOnly =
+ IsNonSchedulableWithParentPhiNode &&
+ !ParentsUniqueUsers
+ .insert(Bundle->getTreeEntry()
+ ->UserTreeIndex.UserTE->Scalars[Lane])
+ .second;
// A blended-load operand node is the synthetic blend mask, not an
// IR operand of the load. Use the real pointer operand for
@@ -5042,12 +5043,13 @@ class slpvectorizer::BoUpSLP {
IsBlended ? In->getOperand(OpIdx)
: Bundle->getTreeEntry()->getOperand(
OpIdx)[Lane])) {
- FoundInOpColumns |= I == In;
+ FoundInOpColumns |= (I == In) && !CopyableDepsOnly;
LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): "
<< *I << "\n");
DecrUnschedForInst(
I, Bundle->getTreeEntry(), OpIdx, Checked,
- Bundle->getTreeEntry()->isExpandedOperand(In, OpIdx));
+ Bundle->getTreeEntry()->isExpandedOperand(In, OpIdx),
+ /*CopyableDepsOnly=*/CopyableDepsOnly);
}
// If parent node is schedulable, it will be handled correctly.
if (Bundle->getTreeEntry()->isCopyableElement(In))
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/copyable-operand-dup-parent-phi-user.ll b/llvm/test/Transforms/SLPVectorizer/X86/copyable-operand-dup-parent-phi-user.ll
new file mode 100644
index 0000000000000..6adea3fb5c55f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/copyable-operand-dup-parent-phi-user.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=slp-vectorizer < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @k() {
+; CHECK-LABEL: define void @k() {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr null, align 4
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> <float 0.000000e+00, float poison, float poison, float 0.000000e+00>, float [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 3>
+; CHECK-NEXT: br i1 false, label %[[V:.*]], label %[[IF_END9:.*]]
+; CHECK: [[T:.*]]:
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP20:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = fsub <4 x float> zeroinitializer, [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd <4 x float> [[TMP4]], zeroinitializer
+; CHECK-NEXT: br label %[[S:.*]]
+; CHECK: [[IF_END9]]:
+; CHECK-NEXT: [[SUB10:%.*]] = fsub float 0.000000e+00, [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> <float -0.000000e+00, float poison, float poison, float 0.000000e+00>, float [[SUB10]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 3>
+; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x float> <float 0.000000e+00, float 1.000000e+00, float -0.000000e+00, float 0.000000e+00>, [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> <float 0.000000e+00, float poison, float poison, float -0.000000e+00>, <4 x i32> <i32 4, i32 1, i32 1, i32 7>
+; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x float> <float 0.000000e+00, float -0.000000e+00, float 1.000000e+00, float 0.000000e+00>, [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x float> [[TMP8]], [[TMP10]]
+; CHECK-NEXT: br label %[[S]]
+; CHECK: [[S]]:
+; CHECK-NEXT: [[L_2:%.*]] = phi float [ 0.000000e+00, %[[IF_END9]] ], [ 0.000000e+00, %[[T]] ]
+; CHECK-NEXT: [[M_1:%.*]] = phi float [ [[SUB10]], %[[IF_END9]] ], [ 0.000000e+00, %[[T]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x float> [ [[TMP11]], %[[IF_END9]] ], [ [[TMP5]], %[[T]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x float> [[TMP12]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x float> <float 0.000000e+00, float poison, float poison, float poison>, float [[M_1]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x float> [[TMP14]], <4 x float> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[L_2]], i64 3
+; CHECK-NEXT: br i1 false, label %[[V]], label %[[U:.*]]
+; CHECK: [[V]]:
+; CHECK-NEXT: [[P_2:%.*]] = phi float [ [[TMP13]], %[[S]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[TMP17:%.*]] = phi <4 x float> [ [[TMP12]], %[[S]] ], [ [[TMP2]], %[[ENTRY]] ]
+; CHECK-NEXT: [[TMP18:%.*]] = fadd <4 x float> [[TMP17]], <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT: [[TMP19:%.*]] = fadd <4 x float> [[TMP18]], <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT: [[TMP20]] = fadd <4 x float> [[TMP19]], <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT: br i1 false, label %[[U]], label %[[T]]
+; CHECK: [[U]]:
+; CHECK-NEXT: [[P_3:%.*]] = phi float [ [[TMP13]], %[[S]] ], [ [[P_2]], %[[V]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = phi <4 x float> [ [[TMP16]], %[[S]] ], [ [[TMP20]], %[[V]] ]
+; CHECK-NEXT: ret void
+;
+entry:
+ %0 = load float, ptr null, align 4
+ br i1 false, label %v, label %if.end9
+
+t: ; preds = %v
+ %sub = fsub float 0.000000e+00, %sub23
+ %add = fadd float %sub, 0.000000e+00
+ %sub1 = fsub float 0.000000e+00, %sub26
+ %add2 = fadd float %sub1, 0.000000e+00
+ %sub3 = fsub float 0.000000e+00, %sub29
+ %add4 = fadd float %sub3, 0.000000e+00
+ br label %s
+
+if.end9: ; preds = %entry
+ %sub10 = fsub float 0.000000e+00, %0
+ %sub11 = fsub float 0.000000e+00, 0.000000e+00
+ %add12 = fadd float 0.000000e+00, 0.000000e+00
+ %add13 = fadd float %add12, 0.000000e+00
+ %add14 = fadd float %sub10, 1.000000e+00
+ %add15 = fadd float %sub10, %add14
+ %add16 = fadd float 0.000000e+00, 0.000000e+00
+ %add17 = fadd float %sub11, %add16
+ br label %s
+
+s: ; preds = %if.end9, %t
+ %l.2 = phi float [ 0.000000e+00, %if.end9 ], [ 0.000000e+00, %t ]
+ %m.1 = phi float [ %sub10, %if.end9 ], [ 0.000000e+00, %t ]
+ %p.1 = phi float [ %add13, %if.end9 ], [ %add, %t ]
+ %o.0 = phi float [ %add15, %if.end9 ], [ %add2, %t ]
+ %r.1 = phi float [ %add17, %if.end9 ], [ %add4, %t ]
+ br i1 false, label %v, label %u
+
+v: ; preds = %s, %entry
+ %p.2 = phi float [ %p.1, %s ], [ 0.000000e+00, %entry ]
+ %o.1 = phi float [ %o.0, %s ], [ %0, %entry ]
+ %r.2 = phi float [ %r.1, %s ], [ 0.000000e+00, %entry ]
+ %add21 = fadd float 0.000000e+00, %p.2
+ %add22 = fadd float %add21, 0.000000e+00
+ %sub23 = fadd float %add22, 0.000000e+00
+ %add24 = fadd float 0.000000e+00, %o.1
+ %add25 = fadd float %add24, 0.000000e+00
+ %sub26 = fadd float %add25, 0.000000e+00
+ %add27 = fadd float 0.000000e+00, %r.2
+ %add28 = fadd float %add27, 0.000000e+00
+ %sub29 = fadd float %add28, 0.000000e+00
+ br i1 false, label %u, label %t
+
+u: ; preds = %v, %s
+ %l.3 = phi float [ %l.2, %s ], [ %sub23, %v ]
+ %m.2 = phi float [ %m.1, %s ], [ %sub26, %v ]
+ %n.1 = phi float [ 0.000000e+00, %s ], [ %sub29, %v ]
+ %p.3 = phi float [ %p.1, %s ], [ %p.2, %v ]
+ %o.2 = phi float [ %o.0, %s ], [ %o.1, %v ]
+ ret void
+}
More information about the llvm-commits
mailing list