[llvm] [SLP]Fix erasing scalars used by narrowed reduction chain instructions (PR #218174)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 22 17:48:19 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/218174
Narrowed reduction chain instructions may use non-root tree scalars;
clear such uses before erasing the scalars.
Fixes https://github.com/llvm/llvm-project/pull/216062#issuecomment-5381262165
>From b6de2dde3a1c365a81866da7850d70ea39959a72 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 22 Aug 2026 17:48:06 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 23 ++++++-
.../AArch64/narrowed-reduction-chain-use.ll | 60 +++++++++++++++++++
2 files changed, 80 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/narrowed-reduction-chain-use.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6a947cda11751..8d49f8b9fda9d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -748,6 +748,12 @@ class slpvectorizer::BoUpSLP {
/// Construct a vectorizable tree that starts at \p Roots.
void buildTree(ArrayRef<Value *> Roots);
+ /// Sets the narrowed reduction chain instructions, dropped together with
+ /// the reduction.
+ void setNarrowedChainInsts(ArrayRef<Instruction *> Insts) {
+ NarrowedChainInsts.insert(Insts.begin(), Insts.end());
+ }
+
/// Returns true if the last buildTree() observed a may-alias memory
/// dependency between two distinct, range-checkable base objects, i.e. a
/// dependency that could be turned into a runtime alias check.
@@ -958,6 +964,7 @@ class slpvectorizer::BoUpSLP {
ExtraBitWidthNodes.clear();
InstrElementSize.clear();
UserIgnoreList = nullptr;
+ NarrowedChainInsts.clear();
PostponedGathers.clear();
ValueToGatherNodes.clear();
TreeEntryToStridedPtrInfoMap.clear();
@@ -5528,6 +5535,10 @@ class slpvectorizer::BoUpSLP {
/// List of users to ignore during scheduling and that don't need extracting.
const SmallDenseSet<Value *> *UserIgnoreList = nullptr;
+ /// Narrowed reduction chain instructions, dropped together with the
+ /// reduction. Subset of UserIgnoreList.
+ SmallPtrSet<Value *, 4> NarrowedChainInsts;
+
/// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
/// sorted SmallVectors of unsigned.
struct OrdersTypeDenseMapInfo {
@@ -26182,7 +26193,11 @@ Value *BoUpSLP::vectorizeTree(
!(GatheredLoadsEntriesFirst.has_value() &&
IE->Idx >= *GatheredLoadsEntriesFirst && getRootNode().isGather() &&
is_contained(getRootNodeScalars(), I)) &&
- !(!getRootNode().isGather() && getRootNode().isCopyableElement(I)))
+ !(!getRootNode().isGather() && getRootNode().isCopyableElement(I)) &&
+ // Dropped narrowed reduction chain instructions may still use
+ // non-root scalars; such uses must be cleared as well.
+ none_of(I->users(),
+ [&](User *U) { return NarrowedChainInsts.contains(U); }))
continue;
SmallVector<SelectInst *> LogicalOpSelects;
I->replaceUsesWithIf(PoisonValue::get(I->getType()), [&](Use &U) {
@@ -31243,10 +31258,12 @@ class HorizontalReduction {
return RedValI && V.isDeleted(RedValI);
}))
break;
- if (RK == ReductionOrdering::Ordered)
+ if (RK == ReductionOrdering::Ordered) {
V.buildTree(VL);
- else
+ } else {
V.buildTree(VL, IgnoreList);
+ V.setNarrowedChainInsts(NarrowedChainInsts);
+ }
if (V.isTreeTinyAndNotFullyVectorizable(RK ==
ReductionOrdering::Unordered)) {
constexpr unsigned CandidatesLimit = 64;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/narrowed-reduction-chain-use.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/narrowed-reduction-chain-use.ll
new file mode 100644
index 0000000000000..5f1bae8a63bea
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/narrowed-reduction-chain-use.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-linux-gnu | FileCheck %s
+
+
+define ptr @test(ptr %q, ptr %z) {
+; CHECK-LABEL: define ptr @test(
+; CHECK-SAME: ptr [[Q:%.*]], ptr [[Z:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND:.*]]
+; CHECK: [[FOR_COND]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[FOR_COND]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[Z]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[L]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 6, i32 -1>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[TMP4:%.*]] = lshr <4 x i32> [[TMP3]], <i32 2, i32 1, i32 7, i32 3>
+; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i32> [[TMP4]], <i32 -1, i32 255, i32 255, i32 252>
+; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i32> [[TMP5]] to <4 x i64>
+; CHECK-NEXT: [[TMP7:%.*]] = shl <4 x i64> [[TMP6]], <i64 33, i64 0, i64 1, i64 1>
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = and i32 [[L]], 6
+; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP10]], 38
+; CHECK-NEXT: [[OP_RDX:%.*]] = or disjoint i64 [[TMP8]], [[TMP11]]
+; CHECK-NEXT: [[ARRAYIDX16:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[Q]], i64 [[IV]]
+; CHECK-NEXT: store i64 [[OP_RDX]], ptr [[ARRAYIDX16]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: br label %[[FOR_COND]]
+;
+entry:
+ br label %for.cond
+
+for.cond:
+ %iv = phi i64 [ %iv.next, %for.cond ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw [4 x i8], ptr %z, i64 %iv
+ %l = load i32, ptr %arrayidx, align 4
+ %shr = lshr i32 %l, 1
+ %and = and i32 %l, 6
+ %l3 = lshr i32 %l, 3
+ %shl = and i32 %l3, 252
+ %shr9 = lshr i32 %l, 7
+ %shl10 = shl nuw nsw i32 %and, 5
+ %shr11 = lshr i32 %and, 2
+ %or12 = or disjoint i32 %shl10, %shr11
+ %conv2.i = zext nneg i32 %or12 to i64
+ %m5 = and i32 %shr, 255
+ %shr9.masked = and i32 %shr9, 255
+ %o6 = or i32 %shl, %shr9.masked
+ %s7 = shl nuw nsw i32 %o6, 1
+ %or.i.i27 = or i32 %s7, %m5
+ %or.i.i = zext nneg i32 %or.i.i27 to i64
+ %sext.i = shl nuw nsw i64 %conv2.i, 33
+ %or5.i.i = or disjoint i64 %sext.i, %or.i.i
+ %arrayidx16 = getelementptr inbounds nuw [8 x i8], ptr %q, i64 %iv
+ store i64 %or5.i.i, ptr %arrayidx16, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ br label %for.cond
+}
More information about the llvm-commits
mailing list