[llvm] [SLP] Fix reused-scalar reduction counters for copyable root nodes (PR #206102)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 08:03:57 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/206102
The horizontal reduction reuse-counter scale is built in
getRootNodeScalars() order and applied positionally to the emitted
reduction vector. For a root node with copyable elements the scalar
order is reordered while the emitted lanes still follow the reduced
values (candidates) order, so the repeat count was applied to the wrong
lane, producing a wrong reduction result.
Fixes #205614
>From 74f0159073f9610dc6caa23ca4bd6123d055fe3c Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 26 Jun 2026 08:03:39 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 34 +++-
.../X86/reduction-copyable-reused-scalars.ll | 151 ++++++++++++++++++
2 files changed, 177 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reduction-copyable-reused-scalars.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 61e0f7816a89b..d1fb8e2beea50 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2210,6 +2210,12 @@ class slpvectorizer::BoUpSLP {
return VectorizableTree.front()->Scalars;
}
+ /// Returns true if the root node has copyable elements.
+ bool isRootNodeWithCopyableElements() const {
+ assert(!VectorizableTree.empty() && "No graph to get the first node from");
+ return VectorizableTree.front()->hasCopyableElements();
+ }
+
/// Returns the type/is-signed info for the root node in the graph without
/// casting.
std::optional<std::pair<Type *, bool>> getRootNodeTypeWithNoCast() const {
@@ -30023,17 +30029,29 @@ class HorizontalReduction {
// Emit code to correctly handle reused reduced values, if required.
if (OptReusedScalars && !SameScaleFactor) {
- // Build TrackedToOrig aligned with the root node scalars order,
- // which may differ from Candidates order due to tree reordering.
+ // The reuse counters must be aligned with the lane order of the
+ // emitted reduction vector. For a root node without copyable
+ // elements the emitted lane order matches getRootNodeScalars(),
+ // which may differ from the Candidates order due to tree reordering,
+ // so remap the counters through the candidates. For a root node with
+ // copyable elements getRootNodeScalars() may be reordered while the
+ // emitted lanes still follow the reduced values (candidates) order,
+ // so use that order directly to avoid applying a counter to the wrong
+ // lane.
ArrayRef<Value *> RootVL = V.getRootNodeScalars();
ArrayRef<Value *> CandSlice(Candidates.begin() + Pos, ReduxWidth);
SmallVector<Value *> RootTrackedToOrig(RootVL.size());
- for (auto [Idx, V] : enumerate(RootVL)) {
- auto *It = find(CandSlice, V);
- assert(It != CandSlice.end() &&
- "Root scalar not found in candidates");
- RootTrackedToOrig[Idx] =
- TrackedToOrig[Pos + std::distance(CandSlice.begin(), It)];
+ if (V.isRootNodeWithCopyableElements()) {
+ for (unsigned Idx : seq<unsigned>(RootVL.size()))
+ RootTrackedToOrig[Idx] = TrackedToOrig[Pos + Idx];
+ } else {
+ for (auto [Idx, Val] : enumerate(RootVL)) {
+ auto *It = find(CandSlice, Val);
+ assert(It != CandSlice.end() &&
+ "Root scalar not found in candidates");
+ RootTrackedToOrig[Idx] =
+ TrackedToOrig[Pos + std::distance(CandSlice.begin(), It)];
+ }
}
VectorizedRoot = emitReusedOps(VectorizedRoot, Builder, V,
SameValuesCounter, RootTrackedToOrig);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-copyable-reused-scalars.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-copyable-reused-scalars.ll
new file mode 100644
index 0000000000000..897923407240a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-copyable-reused-scalars.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.memmove.p0.p0.i64(ptr writeonly captures(none), ptr readonly captures(none), i64, i1 immarg) #0
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
+declare void @llvm.assume(i1 noundef) #1
+
+define fastcc i32 @func_nno0tw_28(ptr %v1) {
+; CHECK-LABEL: define fastcc i32 @func_nno0tw_28(
+; CHECK-SAME: ptr [[V1:%.*]]) {
+; CHECK-NEXT: [[FOR_BODY_751_PREHEADER:.*:]]
+; CHECK-NEXT: [[ALIASCHECK_50_0_SROA_0:%.*]] = alloca [44 x i32], align 16
+; CHECK-NEXT: [[ARRAYIDX107:%.*]] = getelementptr i8, ptr [[V1]], i64 4
+; CHECK-NEXT: [[ARRAYIDX244:%.*]] = getelementptr i8, ptr [[V1]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX244]], align 4
+; CHECK-NEXT: [[REM2916:%.*]] = srem i32 [[TMP0]], 46337
+; CHECK-NEXT: [[ADD2917:%.*]] = add i32 [[REM2916]], 46337
+; CHECK-NEXT: [[MUL2919:%.*]] = mul i32 [[ADD2917]], 16263
+; CHECK-NEXT: [[ADD2923:%.*]] = add i32 330, [[MUL2919]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX107]], align 4
+; CHECK-NEXT: [[REM2830:%.*]] = srem i32 [[TMP1]], 46337
+; CHECK-NEXT: [[ALIASCHECK_50_0_SROA_0_44_ARRAYIDX3011_SROA_IDX3:%.*]] = getelementptr i8, ptr [[ALIASCHECK_50_0_SROA_0]], i64 44
+; CHECK-NEXT: store i32 -2116438905, ptr [[ALIASCHECK_50_0_SROA_0_44_ARRAYIDX3011_SROA_IDX3]], align 4
+; CHECK-NEXT: [[ALIASCHECK_50_0_SROA_0_48_ARRAYIDX3012_SROA_IDX4:%.*]] = getelementptr i8, ptr [[ALIASCHECK_50_0_SROA_0]], i64 48
+; CHECK-NEXT: store i32 1919032160, ptr [[ALIASCHECK_50_0_SROA_0_48_ARRAYIDX3012_SROA_IDX4]], align 16
+; CHECK-NEXT: [[ALIASCHECK_50_0_SROA_0_44_ARRAYIDX3011_SROA_IDX2:%.*]] = getelementptr i8, ptr [[ALIASCHECK_50_0_SROA_0]], i64 44
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr [[ALIASCHECK_50_0_SROA_0]], ptr [[ALIASCHECK_50_0_SROA_0_44_ARRAYIDX3011_SROA_IDX2]], i64 132, i1 false)
+; CHECK-NEXT: [[MUL2833:%.*]] = mul i32 [[REM2830]], 3724
+; CHECK-NEXT: [[MUL2840:%.*]] = mul i32 [[REM2830]], 16306
+; CHECK-NEXT: [[ADD2842:%.*]] = add i32 [[MUL2833]], [[MUL2840]]
+; CHECK-NEXT: [[ADD2844:%.*]] = add i32 -16457, [[ADD2842]]
+; CHECK-NEXT: [[REASS_SUB:%.*]] = sub i32 0, [[ADD2844]]
+; CHECK-NEXT: [[REM2924:%.*]] = urem i32 [[ADD2923]], 46337
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 0, i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x i32>, ptr [[ALIASCHECK_50_0_SROA_0]], align 16
+; CHECK-NEXT: [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 1)
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 389369760, i32 1339756373, i32 -588265445, i32 45324178, i32 79842678, i32 8113750, i32 64148112, i32 0, i32 poison, i32 poison, i32 poison, i32 poison>, i32 0, i32 12
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[REM2924]], i32 15
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP12]], <16 x i32> [[TMP8]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 12, i32 13, i32 31>
+; CHECK-NEXT: [[TMP14:%.*]] = mul <16 x i32> [[TMP13]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 7, i32 1>
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP14]])
+; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT: [[OP_RDX5:%.*]] = add i32 [[TMP16]], [[REASS_SUB]]
+; CHECK-NEXT: [[OP_RDX6:%.*]] = add i32 [[OP_RDX5]], [[TMP15]]
+; CHECK-NEXT: [[REM_I:%.*]] = srem i32 [[OP_RDX6]], 46337
+; CHECK-NEXT: [[MUL10_I:%.*]] = mul i32 [[REM_I]], 21578
+; CHECK-NEXT: [[REM11_I:%.*]] = urem i32 [[MUL10_I]], 46337
+; CHECK-NEXT: [[MUL15_I:%.*]] = mul i32 2676, [[REM11_I]]
+; CHECK-NEXT: [[REM16_I:%.*]] = urem i32 [[MUL15_I]], 46337
+; CHECK-NEXT: [[ADD23_I:%.*]] = add i32 [[REM16_I]], 14604
+; CHECK-NEXT: [[AND_I:%.*]] = and i32 [[ADD23_I]], 61889
+; CHECK-NEXT: [[ADD27_I:%.*]] = add i32 [[AND_I]], 1748154851
+; CHECK-NEXT: [[OR_I:%.*]] = or i32 [[ADD23_I]], 89846209
+; CHECK-NEXT: [[ADD30_I:%.*]] = add i32 [[ADD27_I]], [[OR_I]]
+; CHECK-NEXT: [[SUB1_I_I:%.*]] = sub i32 1935415880, [[ADD30_I]]
+; CHECK-NEXT: [[REM_I_I:%.*]] = srem i32 -659453650, [[SUB1_I_I]]
+; CHECK-NEXT: [[REASS_SUB_I_I:%.*]] = sub i32 [[REM_I_I]], [[ADD30_I]]
+; CHECK-NEXT: [[CMP_I_I:%.*]] = icmp eq i32 [[REASS_SUB_I_I]], -1913265558
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_I_I]])
+; CHECK-NEXT: ret i32 0
+;
+for_body_751.preheader:
+ %aliasCheck_50_0.sroa.0 = alloca [44 x i32], align 16
+ %arrayidx107 = getelementptr i8, ptr %v1, i64 4
+ %arrayidx244 = getelementptr i8, ptr %v1, i64 8
+ %0 = load i32, ptr %arrayidx244, align 4
+ %rem2916 = srem i32 %0, 46337
+ %add2917 = add i32 %rem2916, 46337
+ %mul2919 = mul i32 %add2917, 16263
+ %add2923 = add i32 330, %mul2919
+ %1 = load i32, ptr %arrayidx107, align 4
+ %rem2830 = srem i32 %1, 46337
+ %aliasCheck_50_0.sroa.0.44.arrayidx3011.sroa_idx3 = getelementptr i8, ptr %aliasCheck_50_0.sroa.0, i64 44
+ store i32 -2116438905, ptr %aliasCheck_50_0.sroa.0.44.arrayidx3011.sroa_idx3, align 4
+ %aliasCheck_50_0.sroa.0.48.arrayidx3012.sroa_idx4 = getelementptr i8, ptr %aliasCheck_50_0.sroa.0, i64 48
+ store i32 1919032160, ptr %aliasCheck_50_0.sroa.0.48.arrayidx3012.sroa_idx4, align 16
+ %aliasCheck_50_0.sroa.0.44.arrayidx3011.sroa_idx2 = getelementptr i8, ptr %aliasCheck_50_0.sroa.0, i64 44
+ call void @llvm.memmove.p0.p0.i64(ptr %aliasCheck_50_0.sroa.0, ptr %aliasCheck_50_0.sroa.0.44.arrayidx3011.sroa_idx2, i64 132, i1 false)
+ %mul2833 = mul i32 %rem2830, 3724
+ %mul2840 = mul i32 %rem2830, 16306
+ %add2842 = add i32 %mul2833, %mul2840
+ %add2844 = add i32 -16457, %add2842
+ %reass.sub = sub i32 0, %add2844
+ %add3182.us.2.4 = add i32 %reass.sub, 8113750
+ %add3178 = add i32 0, 0
+ %rem2924 = urem i32 %add2923, 46337
+ %add2925 = add i32 %rem2924, 64148112
+ %add3182.us.2.3 = add i32 %add3178, %add2925
+ %reass.add1498 = shl i32 %add3178, 0
+ %add3230.us.1.3 = add i32 0, %reass.add1498
+ %add3230.us.2.3 = add i32 %add3182.us.2.3, %add3230.us.1.3
+ %reass.add1499 = shl i32 %add3178, 0
+ %add3230.us.1.4 = add i32 %add3230.us.2.3, %reass.add1499
+ %add3182.us.2.5 = add i32 %add3178, 79842678
+ %add3230.us.2.4 = add i32 %add3182.us.2.4, %add3230.us.1.4
+ %reass.add1500 = shl i32 %add3178, 0
+ %add3230.us.1.5 = add i32 %add3230.us.2.4, %reass.add1500
+ %add3230.us.2.5 = add i32 %add3182.us.2.5, %add3230.us.1.5
+ %reass.add1501 = shl i32 %add3178, 0
+ %aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0. = load i32, ptr %aliasCheck_50_0.sroa.0, align 16
+ %aliasCheck_50_0.sroa.0.4.arrayidx3073.sroa_idx1 = getelementptr i8, ptr %aliasCheck_50_0.sroa.0, i64 4
+ %aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4. = load i32, ptr %aliasCheck_50_0.sroa.0.4.arrayidx3073.sroa_idx1, align 4
+ %add3077 = add i32 %aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0.aliasCheck_50_0.sroa.0.0., %aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.aliasCheck_50_0.sroa.0.4.
+ %add3182.us.2.7 = add i32 %add3178, %add3077
+ %add3182.us.2.6 = add i32 %add3178, 45324178
+ %add3230.us.1.6 = add i32 %add3230.us.2.5, %reass.add1501
+ %add3230.us.2.6 = add i32 %add3182.us.2.6, %add3230.us.1.6
+ %reass.add1502 = shl i32 %add3178, 1
+ %add3230.us.1.7 = add i32 %add3230.us.2.6, %reass.add1502
+ %add3182.us.2.9 = add i32 %add3178, 1339756373
+ %add3182.us.2.8 = add i32 %add3178, -588265445
+ %add3230.us.2.7 = add i32 %add3182.us.2.7, %add3230.us.1.7
+ %reass.add1503 = shl i32 %add3178, 1
+ %add3230.us.1.8 = add i32 %add3230.us.2.7, %reass.add1503
+ %add3230.us.2.8 = add i32 %add3182.us.2.8, %add3230.us.1.8
+ %reass.add1504 = shl i32 %add3178, 1
+ %add3230.us.1.9 = add i32 %add3230.us.2.8, %reass.add1504
+ %add3182.us.2.10 = add i32 %add3178, 389369760
+ %add3230.us.2.9 = add i32 %add3182.us.2.9, %add3230.us.1.9
+ %reass.add1505 = shl i32 %add3178, 1
+ %add3230.us.1.10 = add i32 %add3230.us.2.9, %reass.add1505
+ %add3230.us.2.10 = add i32 %add3182.us.2.10, %add3230.us.1.10
+ %rem.i = srem i32 %add3230.us.2.10, 46337
+ %mul10.i = mul i32 %rem.i, 21578
+ %rem11.i = urem i32 %mul10.i, 46337
+ %mul15.i = mul i32 2676, %rem11.i
+ %rem16.i = urem i32 %mul15.i, 46337
+ %add23.i = add i32 %rem16.i, 14604
+ %and.i = and i32 %add23.i, 61889
+ %add27.i = add i32 %and.i, 1748154851
+ %or.i = or i32 %add23.i, 89846209
+ %add30.i = add i32 %add27.i, %or.i
+ %sub1.i.i = sub i32 1935415880, %add30.i
+ %rem.i.i = srem i32 -659453650, %sub1.i.i
+ %reass.sub.i.i = sub i32 %rem.i.i, %add30.i
+ %cmp.i.i = icmp eq i32 %reass.sub.i.i, -1913265558
+ call void @llvm.assume(i1 %cmp.i.i)
+ ret i32 0
+}
+
+attributes #0 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+attributes #1 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
More information about the llvm-commits
mailing list