[llvm] [SLP]Keep reuse mask in sync when reordering split node operand (PR #202084)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 6 13:43:22 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/202084
When reorderBottomToTop uses an operand order for a split vectorize
node and the operand has both reordered and reused scalars, only the
reorder indices were cleared while the reuse mask was left stale. This
diverged the split node scalars from the operand effective order.
Fixes #202003
>From e6300f7cdbcc79f395aeef838a46041f86284417 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 6 Jun 2026 13:43:11 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 8 ++
...split-node-reused-and-reordered-operand.ll | 106 ++++++++++++++++++
2 files changed, 114 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 64fe45433af30..e1ffcef25280e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9278,6 +9278,14 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
"Expected exactly 2 entries.");
for (const auto &P : Data.first->CombinedEntriesWithIndices) {
TreeEntry &OpTE = *VectorizableTree[P.first];
+ // The order of an operand that has both reordered and reused scalars
+ // cannot be absorbed into the split node cleanly: clearing the
+ // reorder indices while keeping the reuse mask (or vice versa)
+ // desyncs the split node scalars from the operand effective order.
+ // Skip reordering for such operands.
+ if (OpTE.State != TreeEntry::SplitVectorize &&
+ !OpTE.ReorderIndices.empty() && !OpTE.ReuseShuffleIndices.empty())
+ continue;
OrdersType Order = OpTE.ReorderIndices;
if (Order.empty() || !OpTE.ReuseShuffleIndices.empty()) {
if (!OpTE.isGather() && OpTE.ReuseShuffleIndices.empty())
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
new file mode 100644
index 0000000000000..a057d8415cf64
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-and-reordered-operand.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test(i1 %tobool46.not) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i1 [[TOBOOL46_NOT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[U:.*]]
+; CHECK: [[U]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ <i32 0, i32 0, i32 undef, i32 undef>, %[[ENTRY]] ], [ zeroinitializer, %[[IF_END44:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[ENTRY]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: br i1 false, label %[[IF_END3:.*]], label %[[IF_END51:.*]]
+; CHECK: [[IF_END3]]:
+; CHECK-NEXT: br i1 false, label %[[AB:.*]], label %[[AA:.*]]
+; CHECK: [[AB]]:
+; CHECK-NEXT: br i1 false, label %[[IF_END44]], label %[[AB]]
+; CHECK: [[IF_END44]]:
+; CHECK-NEXT: br i1 [[TOBOOL46_NOT]], label %[[AA]], label %[[U]]
+; CHECK: [[AA]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[IF_END3]] ], [ zeroinitializer, %[[IF_END44]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: br i1 false, label %[[IF_END51]], label %[[V:.*]]
+; CHECK: [[V]]:
+; CHECK-NEXT: br label %[[IF_END51]]
+; CHECK: [[IF_END51]]:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP6]], %[[AA]] ], [ [[TMP7]], %[[V]] ], [ [[TMP0]], %[[U]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <8 x i32> [ [[TMP13]], %[[AA]] ], [ zeroinitializer, %[[V]] ], [ [[TMP5]], %[[U]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP11]])
+; CHECK-NEXT: ret i32 [[TMP12]]
+;
+entry:
+ br label %u
+
+u:
+ %q.sroa.3.0 = phi i32 [ 0, %entry ], [ %sub, %if.end44 ]
+ %q.sroa.7.0 = phi i32 [ 0, %entry ], [ %sub28, %if.end44 ]
+ %q.sroa.11.0 = phi i32 [ 0, %entry ], [ %sub31, %if.end44 ]
+ %q.sroa.15.0 = phi i32 [ 0, %entry ], [ %sub34, %if.end44 ]
+ %q.sroa.19.0 = phi i32 [ undef, %entry ], [ %sub37, %if.end44 ]
+ %q.sroa.23.0 = phi i32 [ undef, %entry ], [ %sub40, %if.end44 ]
+ br i1 false, label %if.end3, label %if.end51
+
+if.end3:
+ br i1 false, label %ab, label %aa
+
+ab:
+ %add19 = or i32 0, 0
+ %add23 = or i32 0, 0
+ %sub = add i32 0, 0
+ %sub28 = add i32 0, 0
+ %sub31 = add i32 0, 0
+ %sub34 = add i32 0, 0
+ %sub37 = add i32 0, 0
+ %sub40 = add i32 0, 0
+ br i1 false, label %if.end44, label %ab
+
+if.end44:
+ br i1 %tobool46.not, label %aa, label %u
+
+aa:
+ %r.sroa.21.2 = phi i32 [ 0, %if.end3 ], [ %add23, %if.end44 ]
+ %r.sroa.17.2 = phi i32 [ 0, %if.end3 ], [ %add19, %if.end44 ]
+ %r.sroa.11.1 = phi i32 [ 0, %if.end3 ], [ 0, %if.end44 ]
+ %r.sroa.7.2 = phi i32 [ 0, %if.end3 ], [ 0, %if.end44 ]
+ %r.sroa.3.2 = phi i32 [ 0, %if.end3 ], [ 0, %if.end44 ]
+ %q.sroa.3.2 = phi i32 [ 0, %if.end3 ], [ %sub, %if.end44 ]
+ %q.sroa.7.2 = phi i32 [ 0, %if.end3 ], [ %sub28, %if.end44 ]
+ %q.sroa.11.2 = phi i32 [ 0, %if.end3 ], [ %sub31, %if.end44 ]
+ %q.sroa.15.2 = phi i32 [ 0, %if.end3 ], [ %sub34, %if.end44 ]
+ %q.sroa.19.2 = phi i32 [ 0, %if.end3 ], [ %sub37, %if.end44 ]
+ %q.sroa.23.2 = phi i32 [ 0, %if.end3 ], [ %sub40, %if.end44 ]
+ br i1 false, label %if.end51, label %v
+
+v:
+ br label %if.end51
+
+if.end51:
+ %r.sroa.21.3 = phi i32 [ %r.sroa.21.2, %aa ], [ %q.sroa.23.2, %v ], [ %q.sroa.23.0, %u ]
+ %r.sroa.17.3 = phi i32 [ %r.sroa.17.2, %aa ], [ %q.sroa.19.2, %v ], [ %q.sroa.19.0, %u ]
+ %r.sroa.13.3 = phi i32 [ %r.sroa.11.1, %aa ], [ %q.sroa.15.2, %v ], [ %q.sroa.15.0, %u ]
+ %r.sroa.11.2 = phi i32 [ %r.sroa.11.1, %aa ], [ %q.sroa.11.2, %v ], [ %q.sroa.11.0, %u ]
+ %r.sroa.7.3 = phi i32 [ %r.sroa.7.2, %aa ], [ 0, %v ], [ %q.sroa.7.0, %u ]
+ %r.sroa.3.3 = phi i32 [ %r.sroa.3.2, %aa ], [ 0, %v ], [ %q.sroa.3.0, %u ]
+ %q.sroa.3.4 = phi i32 [ %q.sroa.3.2, %aa ], [ 0, %v ], [ %q.sroa.3.0, %u ]
+ %q.sroa.7.4 = phi i32 [ %q.sroa.7.2, %aa ], [ 0, %v ], [ %q.sroa.7.0, %u ]
+ %q.sroa.11.4 = phi i32 [ %q.sroa.11.2, %aa ], [ 0, %v ], [ %q.sroa.11.0, %u ]
+ %q.sroa.15.4 = phi i32 [ %q.sroa.15.2, %aa ], [ 0, %v ], [ %q.sroa.15.0, %u ]
+ %q.sroa.19.4 = phi i32 [ %q.sroa.19.2, %aa ], [ 0, %v ], [ %q.sroa.19.0, %u ]
+ %q.sroa.23.4 = phi i32 [ %q.sroa.23.2, %aa ], [ 0, %v ], [ %q.sroa.23.0, %u ]
+ %add62 = or i32 %r.sroa.7.3, %r.sroa.3.3
+ %add64 = or i32 %add62, %q.sroa.3.4
+ %add66 = or i32 %add64, %q.sroa.7.4
+ %add68 = or i32 %add66, %q.sroa.11.4
+ %add70 = or i32 %add68, %q.sroa.15.4
+ %add72 = or i32 %add70, %q.sroa.19.4
+ %add74 = or i32 %add72, %q.sroa.23.4
+ ret i32 %add74
+}
More information about the llvm-commits
mailing list