[llvm] 25a492b - [SLP]Keep reuse mask in sync when reordering split node operand
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 6 13:43:37 PDT 2026
Author: Alexey Bataev
Date: 2026-06-06T16:43:33-04:00
New Revision: 25a492be00ef46753d7fa1fb9bae03746dad1441
URL: https://github.com/llvm/llvm-project/commit/25a492be00ef46753d7fa1fb9bae03746dad1441
DIFF: https://github.com/llvm/llvm-project/commit/25a492be00ef46753d7fa1fb9bae03746dad1441.diff
LOG: [SLP]Keep reuse mask in sync when reordering split node operand
When reorderBottomToTop uses an operand order for a split vectorize
node and the operand has both reordered and reused scalars, only the
reorder indices were cleared while the reuse mask was left stale. This
diverged the split node scalars from the operand effective order.
Fixes #202003
Reviewers:
Pull Request: https://github.com/llvm/llvm-project/pull/202084
Added:
llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 64fe45433af30..e1ffcef25280e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9278,6 +9278,14 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
"Expected exactly 2 entries.");
for (const auto &P : Data.first->CombinedEntriesWithIndices) {
TreeEntry &OpTE = *VectorizableTree[P.first];
+ // The order of an operand that has both reordered and reused scalars
+ // cannot be absorbed into the split node cleanly: clearing the
+ // reorder indices while keeping the reuse mask (or vice versa)
+ // desyncs the split node scalars from the operand effective order.
+ // Skip reordering for such operands.
+ if (OpTE.State != TreeEntry::SplitVectorize &&
+ !OpTE.ReorderIndices.empty() && !OpTE.ReuseShuffleIndices.empty())
+ continue;
OrdersType Order = OpTE.ReorderIndices;
if (Order.empty() || !OpTE.ReuseShuffleIndices.empty()) {
if (!OpTE.isGather() && OpTE.ReuseShuffleIndices.empty())
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
new file mode 100644
index 0000000000000..a057d8415cf64
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test(i1 %tobool46.not) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i1 [[TOBOOL46_NOT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[U:.*]]
+; CHECK: [[U]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ <i32 0, i32 0, i32 undef, i32 undef>, %[[ENTRY]] ], [ zeroinitializer, %[[IF_END44:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[ENTRY]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: br i1 false, label %[[IF_END3:.*]], label %[[IF_END51:.*]]
+; CHECK: [[IF_END3]]:
+; CHECK-NEXT: br i1 false, label %[[AB:.*]], label %[[AA:.*]]
+; CHECK: [[AB]]:
+; CHECK-NEXT: br i1 false, label %[[IF_END44]], label %[[AB]]
+; CHECK: [[IF_END44]]:
+; CHECK-NEXT: br i1 [[TOBOOL46_NOT]], label %[[AA]], label %[[U]]
+; CHECK: [[AA]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: br i1 false, label %[[IF_END51]], label %[[V:.*]]
+; CHECK: [[V]]:
+; CHECK-NEXT: br label %[[IF_END51]]
+; CHECK: [[IF_END51]]:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP6]], %[[AA]] ], [ [[TMP7]], %[[V]] ], [ [[TMP0]], %[[U]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <8 x i32> [ [[TMP13]], %[[AA]] ], [ zeroinitializer, %[[V]] ], [ [[TMP5]], %[[U]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP11]])
+; CHECK-NEXT: ret i32 [[TMP12]]
+;
+entry:
+ br label %u
+
+u:
+ %q.sroa.3.0 = phi i32 [ 0, %entry ], [ %sub, %if.end44 ]
+ %q.sroa.7.0 = phi i32 [ 0, %entry ], [ %sub28, %if.end44 ]
+ %q.sroa.11.0 = phi i32 [ 0, %entry ], [ %sub31, %if.end44 ]
+ %q.sroa.15.0 = phi i32 [ 0, %entry ], [ %sub34, %if.end44 ]
+ %q.sroa.19.0 = phi i32 [ undef, %entry ], [ %sub37, %if.end44 ]
+ %q.sroa.23.0 = phi i32 [ undef, %entry ], [ %sub40, %if.end44 ]
+ br i1 false, label %if.end3, label %if.end51
+
+if.end3:
+ br i1 false, label %ab, label %aa
+
+ab:
+ %add19 = or i32 0, 0
+ %add23 = or i32 0, 0
+ %sub = add i32 0, 0
+ %sub28 = add i32 0, 0
+ %sub31 = add i32 0, 0
+ %sub34 = add i32 0, 0
+ %sub37 = add i32 0, 0
+ %sub40 = add i32 0, 0
+ br i1 false, label %if.end44, label %ab
+
+if.end44:
+ br i1 %tobool46.not, label %aa, label %u
+
+aa:
+ %r.sroa.21.2 = phi i32 [ 0, %if.end3 ], [ %add23, %if.end44 ]
+ %r.sroa.17.2 = phi i32 [ 0, %if.end3 ], [ %add19, %if.end44 ]
+ %r.sroa.11.1 = phi i32 [ 0, %if.end3 ], [ 0, %if.end44 ]
+ %r.sroa.7.2 = phi i32 [ 0, %if.end3 ], [ 0, %if.end44 ]
+ %r.sroa.3.2 = phi i32 [ 0, %if.end3 ], [ 0, %if.end44 ]
+ %q.sroa.3.2 = phi i32 [ 0, %if.end3 ], [ %sub, %if.end44 ]
+ %q.sroa.7.2 = phi i32 [ 0, %if.end3 ], [ %sub28, %if.end44 ]
+ %q.sroa.11.2 = phi i32 [ 0, %if.end3 ], [ %sub31, %if.end44 ]
+ %q.sroa.15.2 = phi i32 [ 0, %if.end3 ], [ %sub34, %if.end44 ]
+ %q.sroa.19.2 = phi i32 [ 0, %if.end3 ], [ %sub37, %if.end44 ]
+ %q.sroa.23.2 = phi i32 [ 0, %if.end3 ], [ %sub40, %if.end44 ]
+ br i1 false, label %if.end51, label %v
+
+v:
+ br label %if.end51
+
+if.end51:
+ %r.sroa.21.3 = phi i32 [ %r.sroa.21.2, %aa ], [ %q.sroa.23.2, %v ], [ %q.sroa.23.0, %u ]
+ %r.sroa.17.3 = phi i32 [ %r.sroa.17.2, %aa ], [ %q.sroa.19.2, %v ], [ %q.sroa.19.0, %u ]
+ %r.sroa.13.3 = phi i32 [ %r.sroa.11.1, %aa ], [ %q.sroa.15.2, %v ], [ %q.sroa.15.0, %u ]
+ %r.sroa.11.2 = phi i32 [ %r.sroa.11.1, %aa ], [ %q.sroa.11.2, %v ], [ %q.sroa.11.0, %u ]
+ %r.sroa.7.3 = phi i32 [ %r.sroa.7.2, %aa ], [ 0, %v ], [ %q.sroa.7.0, %u ]
+ %r.sroa.3.3 = phi i32 [ %r.sroa.3.2, %aa ], [ 0, %v ], [ %q.sroa.3.0, %u ]
+ %q.sroa.3.4 = phi i32 [ %q.sroa.3.2, %aa ], [ 0, %v ], [ %q.sroa.3.0, %u ]
+ %q.sroa.7.4 = phi i32 [ %q.sroa.7.2, %aa ], [ 0, %v ], [ %q.sroa.7.0, %u ]
+ %q.sroa.11.4 = phi i32 [ %q.sroa.11.2, %aa ], [ 0, %v ], [ %q.sroa.11.0, %u ]
+ %q.sroa.15.4 = phi i32 [ %q.sroa.15.2, %aa ], [ 0, %v ], [ %q.sroa.15.0, %u ]
+ %q.sroa.19.4 = phi i32 [ %q.sroa.19.2, %aa ], [ 0, %v ], [ %q.sroa.19.0, %u ]
+ %q.sroa.23.4 = phi i32 [ %q.sroa.23.2, %aa ], [ 0, %v ], [ %q.sroa.23.0, %u ]
+ %add62 = or i32 %r.sroa.7.3, %r.sroa.3.3
+ %add64 = or i32 %add62, %q.sroa.3.4
+ %add66 = or i32 %add64, %q.sroa.7.4
+ %add68 = or i32 %add66, %q.sroa.11.4
+ %add70 = or i32 %add68, %q.sroa.15.4
+ %add72 = or i32 %add70, %q.sroa.19.4
+ %add74 = or i32 %add72, %q.sroa.23.4
+ ret i32 %add74
+}
More information about the llvm-commits
mailing list