[llvm] [SLP] Extract isIdentityOrder to common routine [probably NFC] (PR #106582)
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 29 10:50:03 PDT 2024
https://github.com/preames updated https://github.com/llvm/llvm-project/pull/106582
>From 387eb6fdd99a57f45f8915eba1462f47e1a91d00 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Thu, 29 Aug 2024 09:40:00 -0700
Subject: [PATCH 1/3] [SLP] Extract isIdentityOrder to common routine [probably
NFC]
This isn't quite just code motion as the four different versions
we had of this routine differed in whether they ignored the
"size" marker used to represent undef. I doubt this matters in
practice, but it is a functional change.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 39 +++++++------------
1 file changed, 13 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 81811e0a4d9295..e2882b3d595351 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1364,6 +1364,19 @@ class BoUpSLP {
/// Perform LICM and CSE on the newly generated gather sequences.
void optimizeGatherSequence();
+ /// Does this non-empty order represent an identity order? Identity
+ /// should be represented as an empty order, so this is used to
+ /// decide if we can canonicalize a computed order. Undef elements
+ /// (represented as size) are ignored.
+ bool IsIdentityOrder(ArrayRef<unsigned> Order) const {
+ assert(!Order.empty());
+ const unsigned Sz = Order.size();
+ for (unsigned Idx : seq<unsigned>(0, Sz))
+ if (Idx != Order[Idx] && Order[Idx] != Sz)
+ return false;
+ return true;
+ }
+
/// Checks if the specified gather tree entry \p TE can be represented as a
/// shuffled vector entry + (possibly) permutation with other gathers. It
/// implements the checks only for possibly ordered scalars (Loads,
@@ -5256,12 +5269,6 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
}
return I1 < I2;
};
- auto IsIdentityOrder = [](const OrdersType &Order) {
- for (unsigned Idx : seq<unsigned>(0, Order.size()))
- if (Idx != Order[Idx])
- return false;
- return true;
- };
DenseMap<unsigned, unsigned> PhiToId;
SmallVector<unsigned> Phis(TE.Scalars.size());
std::iota(Phis.begin(), Phis.end(), 0);
@@ -5565,13 +5572,6 @@ void BoUpSLP::reorderTopToBottom() {
}
if (OrdersUses.empty())
continue;
- auto IsIdentityOrder = [](ArrayRef<unsigned> Order) {
- const unsigned Sz = Order.size();
- for (unsigned Idx : seq<unsigned>(0, Sz))
- if (Idx != Order[Idx] && Order[Idx] != Sz)
- return false;
- return true;
- };
// Choose the most used order.
unsigned IdentityCnt = 0;
unsigned FilledIdentityCnt = 0;
@@ -5891,13 +5891,6 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
OrderedEntries.remove(Op.second);
continue;
}
- auto IsIdentityOrder = [](ArrayRef<unsigned> Order) {
- const unsigned Sz = Order.size();
- for (unsigned Idx : seq<unsigned>(0, Sz))
- if (Idx != Order[Idx] && Order[Idx] != Sz)
- return false;
- return true;
- };
// Choose the most used order.
unsigned IdentityCnt = 0;
unsigned VF = Data.second.front().second->getVectorFactor();
@@ -6186,12 +6179,6 @@ bool BoUpSLP::canFormVector(ArrayRef<StoreInst *> StoresVec,
// Identity order (e.g., {0,1,2,3}) is modeled as an empty OrdersType in
// reorderTopToBottom() and reorderBottomToTop(), so we are following the
// same convention here.
- auto IsIdentityOrder = [](const OrdersType &Order) {
- for (unsigned Idx : seq<unsigned>(0, Order.size()))
- if (Idx != Order[Idx])
- return false;
- return true;
- };
if (IsIdentityOrder(ReorderIndices))
ReorderIndices.clear();
>From 577ce00f0ef59879eb027587aa350a83b91d0535 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Thu, 29 Aug 2024 10:18:00 -0700
Subject: [PATCH 2/3] Address review comments
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 25 +++++++++----------
1 file changed, 12 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e2882b3d595351..174fc6e3b0e2db 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1368,13 +1368,12 @@ class BoUpSLP {
/// should be represented as an empty order, so this is used to
/// decide if we can canonicalize a computed order. Undef elements
/// (represented as size) are ignored.
- bool IsIdentityOrder(ArrayRef<unsigned> Order) const {
- assert(!Order.empty());
+ bool isIdentityOrder(ArrayRef<unsigned> Order) const {
+ assert(!Order.empty() && "expected non-empty mask");
const unsigned Sz = Order.size();
- for (unsigned Idx : seq<unsigned>(0, Sz))
- if (Idx != Order[Idx] && Order[Idx] != Sz)
- return false;
- return true;
+ return all_of(enumerate(Order), [&](const auto &P) {
+ return P.value() == P.index() || P.value() == Sz;
+ });
}
/// Checks if the specified gather tree entry \p TE can be represented as a
@@ -5278,7 +5277,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
stable_sort(Phis, PHICompare);
for (unsigned Id = 0, Sz = Phis.size(); Id < Sz; ++Id)
ResOrder[Id] = PhiToId[Phis[Id]];
- if (IsIdentityOrder(ResOrder))
+ if (isIdentityOrder(ResOrder))
return std::nullopt; // No need to reorder.
return std::move(ResOrder);
}
@@ -5577,7 +5576,7 @@ void BoUpSLP::reorderTopToBottom() {
unsigned FilledIdentityCnt = 0;
OrdersType IdentityOrder(VF, VF);
for (auto &Pair : OrdersUses) {
- if (Pair.first.empty() || IsIdentityOrder(Pair.first)) {
+ if (Pair.first.empty() || isIdentityOrder(Pair.first)) {
if (!Pair.first.empty())
FilledIdentityCnt += Pair.second;
IdentityCnt += Pair.second;
@@ -5593,7 +5592,7 @@ void BoUpSLP::reorderTopToBottom() {
if (Cnt < Pair.second ||
(Cnt == IdentityCnt && IdentityCnt == FilledIdentityCnt &&
Cnt == Pair.second && !BestOrder.empty() &&
- IsIdentityOrder(BestOrder))) {
+ isIdentityOrder(BestOrder))) {
combineOrders(Pair.first, BestOrder);
BestOrder = Pair.first;
Cnt = Pair.second;
@@ -5602,7 +5601,7 @@ void BoUpSLP::reorderTopToBottom() {
}
}
// Set order of the user node.
- if (IsIdentityOrder(BestOrder))
+ if (isIdentityOrder(BestOrder))
continue;
fixupOrderingIndices(BestOrder);
SmallVector<int> Mask;
@@ -5896,7 +5895,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
unsigned VF = Data.second.front().second->getVectorFactor();
OrdersType IdentityOrder(VF, VF);
for (auto &Pair : OrdersUses) {
- if (Pair.first.empty() || IsIdentityOrder(Pair.first)) {
+ if (Pair.first.empty() || isIdentityOrder(Pair.first)) {
IdentityCnt += Pair.second;
combineOrders(IdentityOrder, Pair.first);
}
@@ -5916,7 +5915,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
}
}
// Set order of the user node.
- if (IsIdentityOrder(BestOrder)) {
+ if (isIdentityOrder(BestOrder)) {
for (const std::pair<unsigned, TreeEntry *> &Op : Data.second)
OrderedEntries.remove(Op.second);
continue;
@@ -6179,7 +6178,7 @@ bool BoUpSLP::canFormVector(ArrayRef<StoreInst *> StoresVec,
// Identity order (e.g., {0,1,2,3}) is modeled as an empty OrdersType in
// reorderTopToBottom() and reorderBottomToTop(), so we are following the
// same convention here.
- if (IsIdentityOrder(ReorderIndices))
+ if (isIdentityOrder(ReorderIndices))
ReorderIndices.clear();
return true;
>From 23f41a48d4e63dd7850d230bd7083c0a23274965 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Thu, 29 Aug 2024 10:49:55 -0700
Subject: [PATCH 3/3] Update llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Co-authored-by: Alexey Bataev <a.bataev at gmx.com>
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 174fc6e3b0e2db..e77db3cbd81fe5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1369,7 +1369,7 @@ class BoUpSLP {
/// decide if we can canonicalize a computed order. Undef elements
/// (represented as size) are ignored.
bool isIdentityOrder(ArrayRef<unsigned> Order) const {
- assert(!Order.empty() && "expected non-empty mask");
+ assert(!Order.empty() && "expected non-empty order");
const unsigned Sz = Order.size();
return all_of(enumerate(Order), [&](const auto &P) {
return P.value() == P.index() || P.value() == Sz;
More information about the llvm-commits
mailing list