[llvm] [SLP]Fix erasing gathered leftover narrowed reduction leaves (PR #226782)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 27 06:13:44 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/226782
A leftover narrowed reduction leaf, only gathered in the tree, loses all
users once the tree scalars are erased and is swept as a dead operand,
though the reduction epilogue still uses it. Keep such values alive.
Fixes https://github.com/llvm/llvm-project/pull/224919#issuecomment-5855368565
>From a9a9dab5b32ee957b3b3b7ec989618b7d7a7da8c Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 27 Sep 2026 06:13:31 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 22 +++-
...rrowed-reduction-gathered-leftover-leaf.ll | 106 ++++++++++++++++++
2 files changed, 122 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/narrowed-reduction-gathered-leftover-leaf.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f76a754418422..0f1fbea086abc 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3813,9 +3813,9 @@ class slpvectorizer::BoUpSLP {
/// uses.
SmallPtrSet<Value *, 4> ExternalUsesWithNonUsers;
- /// Replacements emitted for the external uses without users, consumed after
- /// the tree vectorization; must not be collected as dead operands of the
- /// erased scalars.
+ /// Externally used values and replacements emitted for the external uses
+ /// without users, consumed after the tree vectorization; must not be
+ /// collected as dead operands of the erased scalars.
SmallPtrSet<Value *, 4> ExternalUseReplacements;
/// Values used only by @llvm.assume calls.
@@ -26608,6 +26608,16 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
SI->setCondition(Constant::getNullValue(SI->getCondition()->getType()));
}
}
+ // Externally used values, which are not operands of the reduction ops (e.g.
+ // gathered narrowed reduction leaves), may lose all users once the tree
+ // scalars are erased; keep them alive for the reduction epilogue. Too many
+ // users - keep conservatively.
+ if (UserIgnoreList)
+ for (Instruction *I : make_isa_range<Instruction>(ExternallyUsedValues))
+ if (I->hasNUsesOrMore(UsesLimit) || none_of(I->users(), [&](User *U) {
+ return UserIgnoreList->contains(U);
+ }))
+ ExternalUseReplacements.insert(I);
// Retain to-be-deleted instructions for some debug-info bookkeeping and alias
// cache correctness.
// NOTE: removeInstructionAndOperands only marks the instruction for deletion
@@ -32688,9 +32698,9 @@ class HorizontalReduction {
RdxVal = It->second;
if (!Visited.insert(RdxVal).second)
continue;
- // Check if the scalar was vectorized as part of the vectorization
- // tree but not the top node.
- if (!VLScalars.contains(RdxVal) && V.isVectorized(RdxVal)) {
+ // Scalars not reduced by the top node may be used by the reduction
+ // epilogue, even if not vectorized as part of the tree.
+ if (!VLScalars.contains(RdxVal)) {
LocalExternallyUsedValues.insert(RdxVal);
continue;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/narrowed-reduction-gathered-leftover-leaf.ll b/llvm/test/Transforms/SLPVectorizer/X86/narrowed-reduction-gathered-leftover-leaf.ll
new file mode 100644
index 0000000000000..1aed39d118ac4
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/narrowed-reduction-gathered-leftover-leaf.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 < %s | FileCheck %s
+
+define i32 @extracted_fields_leaf(ptr %p, i1 %c, i8 %x) {
+; CHECK-LABEL: define i32 @extracted_fields_leaf(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i8 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i8> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP5:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LD:%.*]] = load i64, ptr [[P]], align 4
+; CHECK-NEXT: [[F0:%.*]] = trunc i64 [[LD]] to i8
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> <i8 poison, i8 poison, i8 1, i8 1>, <4 x i32> <i32 0, i32 poison, i32 6, i32 7>
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i8> [[TMP1]], i8 [[X]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i64 [[LD]] to <8 x i8>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
+; CHECK-NEXT: [[TMP5]] = or <4 x i8> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = shl <4 x i32> [[TMP6]], <i32 0, i32 1, i32 0, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = zext i8 [[F0]] to i32
+; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT: ret i32 [[OP_RDX]]
+;
+entry:
+ br label %loop
+
+loop:
+ %p0 = phi i8 [ 0, %entry ], [ %o0, %loop ]
+ %p1 = phi i8 [ 0, %entry ], [ %o1, %loop ]
+ %p2 = phi i8 [ 0, %entry ], [ %o2, %loop ]
+ %p3 = phi i8 [ 0, %entry ], [ %o3, %loop ]
+ %ld = load i64, ptr %p, align 4
+ %f0 = trunc i64 %ld to i8
+ %o0 = or i8 %p0, %f0
+ %sh = lshr i64 %ld, 8
+ %f1 = trunc i64 %sh to i8
+ %o1 = or i8 %x, %f1
+ %o2 = or i8 %f0, 1
+ %o3 = or i8 %f0, 1
+ br i1 %c, label %exit, label %loop
+
+exit:
+ %z1 = zext i8 %p1 to i32
+ %s1 = shl i32 %z1, 1
+ %z0 = zext i8 %o0 to i32
+ %r0 = or i32 %s1, %z0
+ %z2 = zext i8 %p2 to i32
+ %r1 = or i32 %r0, %z2
+ %z3 = zext i8 %p3 to i32
+ %r2 = or i32 %r1, %z3
+ ret i32 %r2
+}
+
+define i32 @extractelement_leaf(ptr %p, i1 %c, i8 %x) {
+; CHECK-LABEL: define i32 @extractelement_leaf(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i8 [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i8> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP4:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LD:%.*]] = load <8 x i8>, ptr [[P]], align 4
+; CHECK-NEXT: [[F0:%.*]] = extractelement <8 x i8> [[LD]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> <i8 poison, i8 poison, i8 1, i8 1>, <4 x i32> <i32 0, i32 poison, i32 6, i32 7>
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i8> [[TMP1]], i8 [[X]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i8> [[LD]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
+; CHECK-NEXT: [[TMP4]] = or <4 x i8> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = shl <4 x i32> [[TMP5]], <i32 0, i32 1, i32 0, i32 0>
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[F0]] to i32
+; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: ret i32 [[OP_RDX]]
+;
+entry:
+ br label %loop
+
+loop:
+ %p0 = phi i8 [ 0, %entry ], [ %o0, %loop ]
+ %p1 = phi i8 [ 0, %entry ], [ %o1, %loop ]
+ %p2 = phi i8 [ 0, %entry ], [ %o2, %loop ]
+ %p3 = phi i8 [ 0, %entry ], [ %o3, %loop ]
+ %ld = load <8 x i8>, ptr %p, align 4
+ %f0 = extractelement <8 x i8> %ld, i64 0
+ %o0 = or i8 %p0, %f0
+ %f1 = extractelement <8 x i8> %ld, i64 1
+ %o1 = or i8 %x, %f1
+ %o2 = or i8 %f0, 1
+ %o3 = or i8 %f0, 1
+ br i1 %c, label %exit, label %loop
+
+exit:
+ %z1 = zext i8 %p1 to i32
+ %s1 = shl i32 %z1, 1
+ %z0 = zext i8 %o0 to i32
+ %r0 = or i32 %s1, %z0
+ %z2 = zext i8 %p2 to i32
+ %r1 = or i32 %r0, %z2
+ %z3 = zext i8 %p3 to i32
+ %r2 = or i32 %r1, %z3
+ ret i32 %r2
+}
More information about the llvm-commits
mailing list