[llvm] [SLP] Compute a shuffle mask for SK_InsertSubvector (PR #85408)
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 15 08:26:42 PDT 2024
https://github.com/preames updated https://github.com/llvm/llvm-project/pull/85408
>From 26e04558ead1a6ee5290b83e51b3e1cca1641020 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Fri, 15 Mar 2024 07:39:51 -0700
Subject: [PATCH 1/4] [SLP] Compute a shuffle mask for SK_InsertSubvector
This is the third of a series of small patches to compute shuffle masks for the couple of cases where we call getShuffleCost without one. My goal is to add an invariant that all calls to getShuffleCost for fixed length vectors have a mask.
After this change, there is one SK_InsertSubvector case left. I excluded it from this patch just because I thought it worthy of individual attention and review.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b4cce680e2876f..b6868fb3f3ca3f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4328,9 +4328,12 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
llvm_unreachable(
"Expected only consecutive, strided or masked gather loads.");
}
+ SmallVector<int> ShuffleMask(VL.size());
+ for (int i = 0; i < VL.size(); i++)
+ ShuffleMask[i] = i / VF == I ? VL.size() + i % VF : i;
VecLdCost +=
TTI.getShuffleCost(TTI ::SK_InsertSubvector, VecTy,
- std::nullopt, CostKind, I * VF, SubVecTy);
+ ShuffleMask, CostKind, I * VF, SubVecTy);
}
// If masked gather cost is higher - better to vectorize, so
// consider it as a gather node. It will be better estimated
@@ -7454,7 +7457,7 @@ getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
Index + NumSrcElts <= static_cast<int>(Mask.size()))
return TTI.getShuffleCost(
TTI::SK_InsertSubvector,
- FixedVectorType::get(Tp->getElementType(), Mask.size()), std::nullopt,
+ FixedVectorType::get(Tp->getElementType(), Mask.size()), Mask,
TTI::TCK_RecipThroughput, Index, Tp);
}
return TTI.getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp, Args);
@@ -7727,9 +7730,13 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
if (NeedInsertSubvectorAnalysis) {
// Add the cost for the subvectors insert.
- for (int I = VF, E = VL.size(); I < E; I += VF)
+ SmallVector<int> ShuffleMask(VL.size());
+ for (int I = VF, E = VL.size(); I < E; I += VF) {
+ for (int i = 0; i < E; i++)
+ ShuffleMask[i] = i / VF == I ? E + i % VF : i;
GatherCost += TTI.getShuffleCost(TTI::SK_InsertSubvector, VecTy,
- std::nullopt, CostKind, I, LoadTy);
+ ShuffleMask, CostKind, I, LoadTy);
+ }
}
GatherCost -= ScalarsCost;
}
>From aed09af19710b7145ce5ac9b5da46a5cc343cef1 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Fri, 15 Mar 2024 08:15:57 -0700
Subject: [PATCH 2/4] Update llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Co-authored-by: Alexey Bataev <a.bataev at gmx.com>
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b6868fb3f3ca3f..2f3c82824cfb91 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4329,7 +4329,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
"Expected only consecutive, strided or masked gather loads.");
}
SmallVector<int> ShuffleMask(VL.size());
- for (int i = 0; i < VL.size(); i++)
+ for (int Idx : seq<int>(0, VL.size()))
ShuffleMask[i] = i / VF == I ? VL.size() + i % VF : i;
VecLdCost +=
TTI.getShuffleCost(TTI ::SK_InsertSubvector, VecTy,
>From 943a1d4f2bec2088db6e2de4b6d7242a95ed6dce Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Fri, 15 Mar 2024 08:16:16 -0700
Subject: [PATCH 3/4] Update llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Co-authored-by: Alexey Bataev <a.bataev at gmx.com>
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2f3c82824cfb91..2078b5145871b1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7732,7 +7732,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
// Add the cost for the subvectors insert.
SmallVector<int> ShuffleMask(VL.size());
for (int I = VF, E = VL.size(); I < E; I += VF) {
- for (int i = 0; i < E; i++)
+ for (int Idx : seq<int>(0, E))
ShuffleMask[i] = i / VF == I ? E + i % VF : i;
GatherCost += TTI.getShuffleCost(TTI::SK_InsertSubvector, VecTy,
ShuffleMask, CostKind, I, LoadTy);
>From fdb6519fed34446194bec7d23155248302aed6ad Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Fri, 15 Mar 2024 08:26:26 -0700
Subject: [PATCH 4/4] Fix build after applying suggestions
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2078b5145871b1..d850c0ad787702 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4330,7 +4330,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
}
SmallVector<int> ShuffleMask(VL.size());
for (int Idx : seq<int>(0, VL.size()))
- ShuffleMask[i] = i / VF == I ? VL.size() + i % VF : i;
+ ShuffleMask[Idx] = Idx / VF == I ? VL.size() + Idx % VF : Idx;
VecLdCost +=
TTI.getShuffleCost(TTI ::SK_InsertSubvector, VecTy,
ShuffleMask, CostKind, I * VF, SubVecTy);
@@ -7733,7 +7733,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
SmallVector<int> ShuffleMask(VL.size());
for (int I = VF, E = VL.size(); I < E; I += VF) {
for (int Idx : seq<int>(0, E))
- ShuffleMask[i] = i / VF == I ? E + i % VF : i;
+ ShuffleMask[Idx] = Idx / VF == I ? E + Idx % VF : Idx;
GatherCost += TTI.getShuffleCost(TTI::SK_InsertSubvector, VecTy,
ShuffleMask, CostKind, I, LoadTy);
}
More information about the llvm-commits
mailing list