[llvm] [SLP][Revec] Fix SLP crash when trying to fold trailing scalars into the reduced value (PR #203477)
Sushant Gokhale via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 20 19:20:52 PDT 2026
https://github.com/sushgokh updated https://github.com/llvm/llvm-project/pull/203477
>From 6c09061b6c13ef2a0f54aab542a67f8bc2559cf6 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Fri, 12 Jun 2026 00:58:05 -0700
Subject: [PATCH 1/5] [SLP][Revec] Address issue #203195
Cost modelling change introduced by commit SHA f15666d is enabling revec of test shown in the above issue. This is introducing a crash as the reduced value(a scalar) is being added to a vector tail value.
Patch tries to address this.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 19 ++++++++++--
.../X86/revec-ordered-reductions.ll | 29 +++++++++++++++++++
2 files changed, 46 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/revec-ordered-reductions.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 94ddd172a9774..ddf9462b0e731 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30433,11 +30433,26 @@ class HorizontalReduction {
// Emit ordered reduction for the vectorized window.
Builder.SetCurrentDebugLocation(
cast<Instruction>(ReductionRoot)->getDebugLoc());
- if (VectorizedTree)
+ if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy);
+ SLPReVec && VecTy && SuccessRoot) {
+ unsigned DestTyNumElements = getNumElements(VecTy);
+ unsigned VF = getNumElements(SuccessRoot->getType()) / DestTyNumElements;
+ for (auto I : seq<unsigned>(VF)) {
+ auto Position = I * DestTyNumElements;
+ Value *SubVec = createExtractVector(Builder, SuccessRoot,
+ DestTyNumElements, Position);
+ if (!VectorizedTree)
+ VectorizedTree = SubVec;
+ else
+ VectorizedTree = createOp(Builder, RdxKind, VectorizedTree, SubVec,
+ "op.rdx", ReductionOps);
+ }
+ } else if (VectorizedTree) {
VectorizedTree =
emitReduction(SuccessRoot, Builder, TTI, DestTy, VectorizedTree);
- else
+ } else {
VectorizedTree = emitReduction(SuccessRoot, Builder, TTI, DestTy);
+ }
// Fold trailing scalars [SuccessStart+SuccessWidth, N).
for (Value *RdxVal :
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-ordered-reductions.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-ordered-reductions.ll
new file mode 100644
index 0000000000000..5beb6275e7ec6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-ordered-reductions.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -p slp-vectorizer -slp-revec -S < %s | FileCheck %s
+
+define <16 x float> @ordered_fadd_chain(ptr %p0, ptr %p1) {
+; CHECK-LABEL: define <16 x float> @ordered_fadd_chain(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) {
+; CHECK-NEXT: [[V0:%.*]] = load <16 x float>, ptr [[P0]], align 64
+; CHECK-NEXT: [[V1:%.*]] = load <16 x float>, ptr [[P1]], align 64
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[V0]], <16 x float> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <64 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, <64 x float> [[TMP1]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x float> [[V1]], <16 x float> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <64 x float> [[TMP2]], <64 x float> [[TMP3]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <64 x float> [[TMP4]], <64 x float> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <64 x float> [[TMP5]], <64 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <64 x float> [[TMP5]], <64 x float> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[OP_RDX:%.*]] = fadd <16 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <64 x float> [[TMP5]], <64 x float> poison, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd <16 x float> [[OP_RDX]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <64 x float> [[TMP5]], <64 x float> poison, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[OP_RDX2:%.*]] = fadd <16 x float> [[OP_RDX1]], [[TMP9]]
+; CHECK-NEXT: ret <16 x float> [[OP_RDX2]]
+;
+ %v0 = load <16 x float>, ptr %p0
+ %v1 = load <16 x float>, ptr %p1
+ %add0 = fadd <16 x float> %v0, %v0
+ %add1 = fadd <16 x float> %add0, %v1
+ %add2 = fadd <16 x float> %add1, splat (float 1.000000e+00)
+ ret <16 x float> %add2
+}
>From 4efd1f384b54e81ef95fea4a04c08f6c8c88bfcc Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 16 Jun 2026 17:24:21 -0700
Subject: [PATCH 2/5] add non-power-of-2 test case
---
.../X86/revec-ordered-reductions.ll | 33 ++++++++++++++++++-
1 file changed, 32 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-ordered-reductions.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-ordered-reductions.ll
index 5beb6275e7ec6..662c5c5fab267 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-ordered-reductions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-ordered-reductions.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -mtriple=x86_64-unknown-linux-gnu -p slp-vectorizer -slp-revec -S < %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -p slp-vectorizer -slp-revec -S < %s | FileCheck %s --check-prefixes=CHECK,POW2
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -p slp-vectorizer -slp-revec -slp-vectorize-non-power-of-2 -S < %s | FileCheck %s --check-prefixes=CHECK,NON-POW2
define <16 x float> @ordered_fadd_chain(ptr %p0, ptr %p1) {
; CHECK-LABEL: define <16 x float> @ordered_fadd_chain(
@@ -27,3 +28,33 @@ define <16 x float> @ordered_fadd_chain(ptr %p0, ptr %p1) {
%add2 = fadd <16 x float> %add1, splat (float 1.000000e+00)
ret <16 x float> %add2
}
+
+define <16 x float> @ordered_fadd_chain_non_power_2(ptr %p0, ptr %p1) {
+; POW2-LABEL: define <16 x float> @ordered_fadd_chain_non_power_2(
+; POW2-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) {
+; POW2-NEXT: [[V0:%.*]] = load <16 x float>, ptr [[P0]], align 64
+; POW2-NEXT: [[V1:%.*]] = load <16 x float>, ptr [[P1]], align 64
+; POW2-NEXT: [[ADD0:%.*]] = fadd <16 x float> [[V0]], [[V0]]
+; POW2-NEXT: [[ADD1:%.*]] = fadd <16 x float> [[ADD0]], splat (float 1.000000e+00)
+; POW2-NEXT: ret <16 x float> [[ADD1]]
+;
+; NON-POW2-LABEL: define <16 x float> @ordered_fadd_chain_non_power_2(
+; NON-POW2-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) {
+; NON-POW2-NEXT: [[V0:%.*]] = load <16 x float>, ptr [[P0]], align 64
+; NON-POW2-NEXT: [[V1:%.*]] = load <16 x float>, ptr [[P1]], align 64
+; NON-POW2-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[V0]], <16 x float> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; NON-POW2-NEXT: [[TMP2:%.*]] = shufflevector <48 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, <48 x float> [[TMP1]], <48 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; NON-POW2-NEXT: [[TMP3:%.*]] = shufflevector <48 x float> [[TMP2]], <48 x float> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; NON-POW2-NEXT: [[TMP4:%.*]] = shufflevector <48 x float> [[TMP3]], <48 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; NON-POW2-NEXT: [[TMP5:%.*]] = shufflevector <48 x float> [[TMP3]], <48 x float> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; NON-POW2-NEXT: [[OP_RDX:%.*]] = fadd <16 x float> [[TMP4]], [[TMP5]]
+; NON-POW2-NEXT: [[TMP6:%.*]] = shufflevector <48 x float> [[TMP3]], <48 x float> poison, <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; NON-POW2-NEXT: [[OP_RDX1:%.*]] = fadd <16 x float> [[OP_RDX]], [[TMP6]]
+; NON-POW2-NEXT: ret <16 x float> [[OP_RDX1]]
+;
+ %v0 = load <16 x float>, ptr %p0
+ %v1 = load <16 x float>, ptr %p1
+ %add0 = fadd <16 x float> %v0, %v0
+ %add1 = fadd <16 x float> %add0, splat (float 1.000000e+00)
+ ret <16 x float> %add1
+}
>From a79a47dc1f5d25e8f44b558acfe5f95c09193313 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 16 Jun 2026 17:36:39 -0700
Subject: [PATCH 3/5] address review comments
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index ddf9462b0e731..e47b48b4f275c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30433,12 +30433,11 @@ class HorizontalReduction {
// Emit ordered reduction for the vectorized window.
Builder.SetCurrentDebugLocation(
cast<Instruction>(ReductionRoot)->getDebugLoc());
- if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy);
- SLPReVec && VecTy && SuccessRoot) {
+ if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy); SLPReVec && VecTy) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(SuccessRoot->getType()) / DestTyNumElements;
for (auto I : seq<unsigned>(VF)) {
- auto Position = I * DestTyNumElements;
+ unsigned Position = I * DestTyNumElements;
Value *SubVec = createExtractVector(Builder, SuccessRoot,
DestTyNumElements, Position);
if (!VectorizedTree)
>From 3e9850d67dd7a3de8ae5faf8c55321b4f645a30c Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 18 Jun 2026 19:10:22 -0700
Subject: [PATCH 4/5] minor refactoring, adding assert
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 30 +++++--------------
1 file changed, 8 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e47b48b4f275c..e6d9d38048840 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30433,25 +30433,8 @@ class HorizontalReduction {
// Emit ordered reduction for the vectorized window.
Builder.SetCurrentDebugLocation(
cast<Instruction>(ReductionRoot)->getDebugLoc());
- if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy); SLPReVec && VecTy) {
- unsigned DestTyNumElements = getNumElements(VecTy);
- unsigned VF = getNumElements(SuccessRoot->getType()) / DestTyNumElements;
- for (auto I : seq<unsigned>(VF)) {
- unsigned Position = I * DestTyNumElements;
- Value *SubVec = createExtractVector(Builder, SuccessRoot,
- DestTyNumElements, Position);
- if (!VectorizedTree)
- VectorizedTree = SubVec;
- else
- VectorizedTree = createOp(Builder, RdxKind, VectorizedTree, SubVec,
- "op.rdx", ReductionOps);
- }
- } else if (VectorizedTree) {
- VectorizedTree =
- emitReduction(SuccessRoot, Builder, TTI, DestTy, VectorizedTree);
- } else {
- VectorizedTree = emitReduction(SuccessRoot, Builder, TTI, DestTy);
- }
+ VectorizedTree = createSingleOp(Builder, *TTI, SuccessRoot, 1, false,
+ DestTy, false, VectorizedTree);
// Fold trailing scalars [SuccessStart+SuccessWidth, N).
for (Value *RdxVal :
@@ -30494,13 +30477,16 @@ class HorizontalReduction {
/// scale \p Scale and signedness \p IsSigned.
Value *createSingleOp(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
Value *Vec, unsigned Scale, bool IsSigned, Type *DestTy,
- bool ReducedInTree) {
+ bool ReducedInTree, Value *Start = nullptr) {
Value *Rdx;
if (ReducedInTree) {
Rdx = Vec;
- } else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy)) {
+ } else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy);
+ VecTy && SLPReVec) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
+ assert(getNumElements(Vec->getType()) % DestTyNumElements == 0 &&
+ "Vec element count must be a multiple of DestTy element count");
// e.g. Consider vector reduce add.
// Initial reduction is
// clang-format off
@@ -30530,7 +30516,7 @@ class HorizontalReduction {
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
}
} else {
- Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
+ Rdx = emitReduction(Vec, Builder, &TTI, DestTy, Start);
}
if (Rdx->getType() != DestTy)
Rdx = Builder.CreateIntCast(Rdx, DestTy, IsSigned);
>From 3ef86825a6e5b36ec8dcd767cbd381d519d9ee1d Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Sat, 20 Jun 2026 19:19:14 -0700
Subject: [PATCH 5/5] address review comments
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 7 ++--
llvm/test/Transforms/SLPVectorizer/revec.ll | 39 ++++++++++++++++++-
2 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e6d9d38048840..62fd7385681ed 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30433,8 +30433,9 @@ class HorizontalReduction {
// Emit ordered reduction for the vectorized window.
Builder.SetCurrentDebugLocation(
cast<Instruction>(ReductionRoot)->getDebugLoc());
- VectorizedTree = createSingleOp(Builder, *TTI, SuccessRoot, 1, false,
- DestTy, false, VectorizedTree);
+ VectorizedTree = createSingleOp(Builder, *TTI, SuccessRoot, /*Scale=*/1,
+ /*IsSigned=*/false, DestTy,
+ /*ReducedInTree=*/false, VectorizedTree);
// Fold trailing scalars [SuccessStart+SuccessWidth, N).
for (Value *RdxVal :
@@ -30505,7 +30506,7 @@ class HorizontalReduction {
// %add0 = add <4 x i32> zeroinitializer, %A
// %add1 = add <4 x i32> %add0, %B
// clang-format on
- Rdx = nullptr;
+ Rdx = Start;
for (auto I : seq<unsigned>(VF)) {
auto Position = I * DestTyNumElements;
Value *SubVec =
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 346669c53824f..1eefceb249717 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 %s | FileCheck %s --check-prefixes=CHECK,POW2
+; RUN: opt -passes=slp-vectorizer -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 -slp-vectorize-non-power-of-2 %s | FileCheck %s --check-prefixes=CHECK,NON-POW2
define void @test1(ptr %a, ptr %b, ptr %c) {
; CHECK-LABEL: @test1(
@@ -498,3 +499,39 @@ entry:
%add3 = add <4 x i32> %add2, %a3
ret <4 x i32> %add3
}
+
+define <4 x i32> @hor_reduction_four_points_non_pow2(ptr %a) {
+; POW2-LABEL: @hor_reduction_four_points_non_pow2(
+; POW2-NEXT: entry:
+; POW2-NEXT: [[GEP1:%.*]] = getelementptr <4 x i32>, ptr [[A:%.*]], i64 1
+; POW2-NEXT: [[GEP2:%.*]] = getelementptr <4 x i32>, ptr [[A]], i64 2
+; POW2-NEXT: [[A0:%.*]] = load <4 x i32>, ptr [[A]], align 16
+; POW2-NEXT: [[A1:%.*]] = load <4 x i32>, ptr [[GEP1]], align 16
+; POW2-NEXT: [[A2:%.*]] = load <4 x i32>, ptr [[GEP2]], align 16
+; POW2-NEXT: [[ADD0:%.*]] = add <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[A0]]
+; POW2-NEXT: [[ADD1:%.*]] = add <4 x i32> [[ADD0]], [[A1]]
+; POW2-NEXT: [[ADD2:%.*]] = add <4 x i32> [[ADD1]], [[A2]]
+; POW2-NEXT: ret <4 x i32> [[ADD2]]
+;
+; NON-POW2-LABEL: @hor_reduction_four_points_non_pow2(
+; NON-POW2-NEXT: entry:
+; NON-POW2-NEXT: [[TMP0:%.*]] = load <12 x i32>, ptr [[A:%.*]], align 16
+; NON-POW2-NEXT: [[TMP1:%.*]] = shufflevector <12 x i32> [[TMP0]], <12 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NON-POW2-NEXT: [[TMP2:%.*]] = shufflevector <12 x i32> [[TMP0]], <12 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NON-POW2-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; NON-POW2-NEXT: [[TMP3:%.*]] = shufflevector <12 x i32> [[TMP0]], <12 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NON-POW2-NEXT: [[RDX_OP1:%.*]] = add <4 x i32> [[RDX_OP]], [[TMP3]]
+; NON-POW2-NEXT: [[OP_RDX:%.*]] = add <4 x i32> [[RDX_OP1]], <i32 0, i32 1, i32 2, i32 3>
+; NON-POW2-NEXT: ret <4 x i32> [[OP_RDX]]
+;
+entry:
+ %gep1 = getelementptr <4 x i32>, ptr %a, i64 1
+ %gep2 = getelementptr <4 x i32>, ptr %a, i64 2
+ %a0 = load <4 x i32>, ptr %a
+ %a1 = load <4 x i32>, ptr %gep1
+ %a2 = load <4 x i32>, ptr %gep2
+ %add0 = add <4 x i32> <i32 0, i32 1, i32 2, i32 3>, %a0
+ %add1 = add <4 x i32> %add0, %a1
+ %add2 = add <4 x i32> %add1, %a2
+ ret <4 x i32> %add2
+}
More information about the llvm-commits
mailing list