[llvm] [SLP] Allow non-power-of-2 VF in tryToVectorizeList (PR #206259)
Harrison Hao via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 27 20:04:19 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/206259
>From 1210e040b99799779fb14574b40bc21a8d76ac25 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Sat, 27 Jun 2026 23:12:07 +0800
Subject: [PATCH 1/2] [SLP] Allow non-power-of-2 VF in tryToVectorizeList
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 7 +-
.../non-power-of-2-buildvector.ll | 92 +++++++++++++++++++
2 files changed, 98 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/non-power-of-2-buildvector.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d1fb8e2beea50..0a58e5599efb6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28576,6 +28576,10 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
unsigned MaxVF = std::max<unsigned>(
getFloorFullVectorNumberOfElements(*TTI, ScalarTy, VL.size()), MinVF);
MaxVF = std::min(R.getMaximumVF(Sz, S.getOpcode()), MaxVF);
+ if (isAllowedNonPowerOf2VF(VL.size()))
+ MaxVF = std::max<unsigned>(
+ MaxVF,
+ std::min<unsigned>(VL.size(), R.getMaximumVF(Sz, S.getOpcode())));
if (MaxVF < 2) {
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0)
@@ -28601,7 +28605,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
for (unsigned I = NextInst; I < MaxInst; ++I) {
unsigned ActualVF = std::min(MaxInst - I, VF);
- if (!hasFullVectorsOrPowerOf2(*TTI, ScalarTy, ActualVF))
+ if (!hasFullVectorsOrPowerOf2(*TTI, ScalarTy, ActualVF) &&
+ !isAllowedNonPowerOf2VF(ActualVF))
continue;
if (MaxVFOnly && ActualVF < MaxVF)
diff --git a/llvm/test/Transforms/SLPVectorizer/non-power-of-2-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/non-power-of-2-buildvector.ll
new file mode 100644
index 0000000000000..e229bef852a38
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/non-power-of-2-buildvector.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -slp-vectorize-non-power-of-2 < %s | FileCheck %s
+
+define <15 x half> @func(ptr addrspace(1) %buffer) {
+; CHECK-LABEL: define <15 x half> @func(
+; CHECK-SAME: ptr addrspace(1) [[BUFFER:%.*]]) {
+; CHECK-NEXT: [[_ENTRY:.*:]]
+; CHECK-NEXT: [[DATA:%.*]] = load <15 x half>, ptr addrspace(1) [[BUFFER]], align 32
+; CHECK-NEXT: [[TMP0:%.*]] = fcmp olt <15 x half> [[DATA]], zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc nnan nsz arcp contract afn <15 x half> [[DATA]], splat (half 1.000210e-02)
+; CHECK-NEXT: [[TMP2:%.*]] = select <15 x i1> [[TMP0]], <15 x half> [[TMP1]], <15 x half> [[DATA]]
+; CHECK-NEXT: ret <15 x half> [[TMP2]]
+;
+.entry:
+ %data = load <15 x half>, ptr addrspace(1) %buffer, align 32
+ %0 = extractelement <15 x half> %data, i64 0
+ %1 = fcmp olt half %0, 0.000000e+00
+ %2 = fmul reassoc nnan nsz arcp contract afn half %0, 1.000210e-02
+ %3 = select i1 %1, half %2, half %0
+ %.0.vec.insert1807 = insertelement <15 x half> %data, half %3, i64 0
+ %4 = extractelement <15 x half> %data, i64 1
+ %5 = fcmp olt half %4, 0.000000e+00
+ %6 = fmul reassoc nnan nsz arcp contract afn half %4, 1.000210e-02
+ %7 = select i1 %5, half %6, half %4
+ %.2.vec.insert1809 = insertelement <15 x half> %.0.vec.insert1807, half %7, i64 1
+ %8 = extractelement <15 x half> %data, i64 2
+ %9 = fcmp olt half %8, 0.000000e+00
+ %10 = fmul reassoc nnan nsz arcp contract afn half %8, 1.000210e-02
+ %11 = select i1 %9, half %10, half %8
+ %.4.vec.insert1811 = insertelement <15 x half> %.2.vec.insert1809, half %11, i64 2
+ %12 = extractelement <15 x half> %data, i64 3
+ %13 = fcmp olt half %12, 0.000000e+00
+ %14 = fmul reassoc nnan nsz arcp contract afn half %12, 1.000210e-02
+ %15 = select i1 %13, half %14, half %12
+ %.6.vec.insert1813 = insertelement <15 x half> %.4.vec.insert1811, half %15, i64 3
+ %16 = extractelement <15 x half> %data, i64 4
+ %17 = fcmp olt half %16, 0.000000e+00
+ %18 = fmul reassoc nnan nsz arcp contract afn half %16, 1.000210e-02
+ %19 = select i1 %17, half %18, half %16
+ %.8.vec.insert1815 = insertelement <15 x half> %.6.vec.insert1813, half %19, i64 4
+ %20 = extractelement <15 x half> %data, i64 5
+ %21 = fcmp olt half %20, 0.000000e+00
+ %22 = fmul reassoc nnan nsz arcp contract afn half %20, 1.000210e-02
+ %23 = select i1 %21, half %22, half %20
+ %.10.vec.insert1817 = insertelement <15 x half> %.8.vec.insert1815, half %23, i64 5
+ %24 = extractelement <15 x half> %data, i64 6
+ %25 = fcmp olt half %24, 0.000000e+00
+ %26 = fmul reassoc nnan nsz arcp contract afn half %24, 1.000210e-02
+ %27 = select i1 %25, half %26, half %24
+ %.12.vec.insert1819 = insertelement <15 x half> %.10.vec.insert1817, half %27, i64 6
+ %28 = extractelement <15 x half> %data, i64 7
+ %29 = fcmp olt half %28, 0.000000e+00
+ %30 = fmul reassoc nnan nsz arcp contract afn half %28, 1.000210e-02
+ %31 = select i1 %29, half %30, half %28
+ %.14.vec.insert1821 = insertelement <15 x half> %.12.vec.insert1819, half %31, i64 7
+ %32 = extractelement <15 x half> %data, i64 8
+ %33 = fcmp olt half %32, 0.000000e+00
+ %34 = fmul reassoc nnan nsz arcp contract afn half %32, 1.000210e-02
+ %35 = select i1 %33, half %34, half %32
+ %.16.vec.insert1823 = insertelement <15 x half> %.14.vec.insert1821, half %35, i64 8
+ %36 = extractelement <15 x half> %data, i64 9
+ %37 = fcmp olt half %36, 0.000000e+00
+ %38 = fmul reassoc nnan nsz arcp contract afn half %36, 1.000210e-02
+ %39 = select i1 %37, half %38, half %36
+ %.18.vec.insert1825 = insertelement <15 x half> %.16.vec.insert1823, half %39, i64 9
+ %40 = extractelement <15 x half> %data, i64 10
+ %41 = fcmp olt half %40, 0.000000e+00
+ %42 = fmul reassoc nnan nsz arcp contract afn half %40, 1.000210e-02
+ %43 = select i1 %41, half %42, half %40
+ %.20.vec.insert1827 = insertelement <15 x half> %.18.vec.insert1825, half %43, i64 10
+ %44 = extractelement <15 x half> %data, i64 11
+ %45 = fcmp olt half %44, 0.000000e+00
+ %46 = fmul reassoc nnan nsz arcp contract afn half %44, 1.000210e-02
+ %47 = select i1 %45, half %46, half %44
+ %.22.vec.insert1829 = insertelement <15 x half> %.20.vec.insert1827, half %47, i64 11
+ %48 = extractelement <15 x half> %data, i64 12
+ %49 = fcmp olt half %48, 0.000000e+00
+ %50 = fmul reassoc nnan nsz arcp contract afn half %48, 1.000210e-02
+ %51 = select i1 %49, half %50, half %48
+ %.24.vec.insert1831 = insertelement <15 x half> %.22.vec.insert1829, half %51, i64 12
+ %52 = extractelement <15 x half> %data, i64 13
+ %53 = fcmp olt half %52, 0.000000e+00
+ %54 = fmul reassoc nnan nsz arcp contract afn half %52, 1.000210e-02
+ %55 = select i1 %53, half %54, half %52
+ %.26.vec.insert1833 = insertelement <15 x half> %.24.vec.insert1831, half %55, i64 13
+ %56 = extractelement <15 x half> %data, i64 14
+ %57 = fcmp olt half %56, 0.000000e+00
+ %58 = fmul reassoc nnan nsz arcp contract afn half %56, 1.000210e-02
+ %59 = select i1 %57, half %58, half %56
+ %.28.vec.insert1835 = insertelement <15 x half> %.26.vec.insert1833, half %59, i64 14
+ ret <15 x half> %.28.vec.insert1835
+}
>From e509f63646acdde8a1b3f5ac347ab23cabcee6f4 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Sun, 28 Jun 2026 11:03:57 +0800
Subject: [PATCH 2/2] Update for comments and fix lit test
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 9 ++-
.../Transforms/SLPVectorizer/X86/odd_store.ll | 65 ++++++++++++-------
2 files changed, 46 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0a58e5599efb6..2cf06817a8920 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28574,12 +28574,11 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
unsigned Sz = R.getVectorElementSize(I0);
unsigned MinVF = R.getMinVF(Sz);
unsigned MaxVF = std::max<unsigned>(
- getFloorFullVectorNumberOfElements(*TTI, ScalarTy, VL.size()), MinVF);
+ isAllowedNonPowerOf2VF(VL.size())
+ ? VL.size()
+ : getFloorFullVectorNumberOfElements(*TTI, ScalarTy, VL.size()),
+ MinVF);
MaxVF = std::min(R.getMaximumVF(Sz, S.getOpcode()), MaxVF);
- if (isAllowedNonPowerOf2VF(VL.size()))
- MaxVF = std::max<unsigned>(
- MaxVF,
- std::min<unsigned>(VL.size(), R.getMaximumVF(Sz, S.getOpcode())));
if (MaxVF < 2) {
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
index cef62c6cae3d5..22d5239c637e9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
@@ -9,29 +9,48 @@
;}
define i32 @foo(ptr noalias nocapture %A, ptr noalias nocapture %B, float %T) {
-; CHECK-LABEL: @foo(
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10
-; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul float [[TMP2]], [[T:%.*]]
-; CHECK-NEXT: [[TMP4:%.*]] = fpext float [[TMP3]] to double
-; CHECK-NEXT: [[TMP5:%.*]] = fadd double [[TMP4]], 4.000000e+00
-; CHECK-NEXT: [[TMP6:%.*]] = fptosi double [[TMP5]] to i8
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[A:%.*]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 11
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = load <2 x float>, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x float> poison, float [[T]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = fmul <2 x float> [[TMP9]], [[TMP11]]
-; CHECK-NEXT: [[TMP17:%.*]] = fpext <2 x float> [[TMP16]] to <2 x double>
-; CHECK-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[TMP17]], <double 5.000000e+00, double 6.000000e+00>
-; CHECK-NEXT: [[TMP15:%.*]] = fptosi <2 x double> [[TMP14]] to <2 x i8>
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i8> [[TMP15]], i32 0
-; CHECK-NEXT: store i8 [[TMP12]], ptr [[TMP13]], align 1
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i8> [[TMP15]], i32 1
-; CHECK-NEXT: store i8 [[TMP19]], ptr [[TMP20]], align 1
-; CHECK-NEXT: ret i32 undef
+; NON-POW2-LABEL: @foo(
+; NON-POW2-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10
+; NON-POW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 1
+; NON-POW2-NEXT: [[TMP3:%.*]] = load <3 x float>, ptr [[TMP1]], align 4
+; NON-POW2-NEXT: [[TMP4:%.*]] = insertelement <3 x float> poison, float [[T:%.*]], i32 0
+; NON-POW2-NEXT: [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> zeroinitializer
+; NON-POW2-NEXT: [[TMP6:%.*]] = fmul <3 x float> [[TMP3]], [[TMP5]]
+; NON-POW2-NEXT: [[TMP7:%.*]] = fpext <3 x float> [[TMP6]] to <3 x double>
+; NON-POW2-NEXT: [[TMP8:%.*]] = fadd <3 x double> [[TMP7]], <double 4.000000e+00, double 5.000000e+00, double 6.000000e+00>
+; NON-POW2-NEXT: [[TMP9:%.*]] = fptosi <3 x double> [[TMP8]] to <3 x i8>
+; NON-POW2-NEXT: [[TMP10:%.*]] = extractelement <3 x i8> [[TMP9]], i32 0
+; NON-POW2-NEXT: store i8 [[TMP10]], ptr [[A]], align 1
+; NON-POW2-NEXT: [[TMP11:%.*]] = extractelement <3 x i8> [[TMP9]], i32 1
+; NON-POW2-NEXT: store i8 [[TMP11]], ptr [[TMP2]], align 1
+; NON-POW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
+; NON-POW2-NEXT: [[TMP13:%.*]] = extractelement <3 x i8> [[TMP9]], i32 2
+; NON-POW2-NEXT: store i8 [[TMP13]], ptr [[TMP12]], align 1
+; NON-POW2-NEXT: ret i32 undef
+;
+; POW2-ONLY-LABEL: @foo(
+; POW2-ONLY-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10
+; POW2-ONLY-NEXT: [[TMP2:%.*]] = load float, ptr [[TMP1]], align 4
+; POW2-ONLY-NEXT: [[TMP3:%.*]] = fmul float [[TMP2]], [[T:%.*]]
+; POW2-ONLY-NEXT: [[TMP4:%.*]] = fpext float [[TMP3]] to double
+; POW2-ONLY-NEXT: [[TMP5:%.*]] = fadd double [[TMP4]], 4.000000e+00
+; POW2-ONLY-NEXT: [[TMP6:%.*]] = fptosi double [[TMP5]] to i8
+; POW2-ONLY-NEXT: store i8 [[TMP6]], ptr [[A:%.*]], align 1
+; POW2-ONLY-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 11
+; POW2-ONLY-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1
+; POW2-ONLY-NEXT: [[TMP9:%.*]] = load <2 x float>, ptr [[TMP7]], align 4
+; POW2-ONLY-NEXT: [[TMP10:%.*]] = insertelement <2 x float> poison, float [[T]], i32 0
+; POW2-ONLY-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT: [[TMP12:%.*]] = fmul <2 x float> [[TMP9]], [[TMP11]]
+; POW2-ONLY-NEXT: [[TMP13:%.*]] = fpext <2 x float> [[TMP12]] to <2 x double>
+; POW2-ONLY-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[TMP13]], <double 5.000000e+00, double 6.000000e+00>
+; POW2-ONLY-NEXT: [[TMP15:%.*]] = fptosi <2 x double> [[TMP14]] to <2 x i8>
+; POW2-ONLY-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i32 0
+; POW2-ONLY-NEXT: store i8 [[TMP16]], ptr [[TMP8]], align 1
+; POW2-ONLY-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
+; POW2-ONLY-NEXT: [[TMP18:%.*]] = extractelement <2 x i8> [[TMP15]], i32 1
+; POW2-ONLY-NEXT: store i8 [[TMP18]], ptr [[TMP17]], align 1
+; POW2-ONLY-NEXT: ret i32 undef
;
%1 = getelementptr inbounds float, ptr %B, i64 10
%2 = load float, ptr %1, align 4
More information about the llvm-commits
mailing list