[llvm] [SLP] Simplify buildTree() and callees (NFC) (PR #135766)
Gaëtan Bossu via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 25 08:33:49 PDT 2025
https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/135766
>From b0a0e077bbe1833cc7e8ac6404edb58a46078a82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 11 Apr 2025 10:50:11 +0000
Subject: [PATCH 1/5] tryToFindDuplicates(): Limit indentation
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 79 ++++++++++---------
1 file changed, 40 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 53da78ee599b7..2ad82165d3015 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9562,49 +9562,50 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
if (NumUniqueScalarValues == VL.size() &&
(VectorizeNonPowerOf2 || IsFullVectors)) {
ReuseShuffleIndices.clear();
- } else {
- // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
- if ((UserTreeIdx.UserTE &&
- UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(TTI)) ||
- !hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
- LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
- "for nodes with padding.\n");
- return false;
- }
- LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
- if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
- (UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
- return isa<UndefValue>(V) || !isConstant(V);
- }))) {
- if (DoNotFail && UniquePositions.size() > 1 &&
- NumUniqueScalarValues > 1 && S.getMainOp()->isSafeToRemove() &&
- all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
- // Find the number of elements, which forms full vectors.
- unsigned PWSz = getFullVectorNumberOfElements(
- TTI, UniqueValues.front()->getType(), UniqueValues.size());
- PWSz = std::min<unsigned>(PWSz, VL.size());
- if (PWSz == VL.size()) {
- ReuseShuffleIndices.clear();
- } else {
- NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
- NonUniqueValueVL.append(
- PWSz - UniqueValues.size(),
- PoisonValue::get(UniqueValues.front()->getType()));
- // Check that extended with poisons operations are still valid for
- // vectorization (div/rem are not allowed).
- if (!getSameOpcode(NonUniqueValueVL, TLI).valid()) {
- LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
- return false;
- }
- VL = NonUniqueValueVL;
+ return true;
+ }
+
+ // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
+ if ((UserTreeIdx.UserTE &&
+ UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(TTI)) ||
+ !hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
+ LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
+ "for nodes with padding.\n");
+ return false;
+ }
+ LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
+ if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
+ (UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
+ return isa<UndefValue>(V) || !isConstant(V);
+ }))) {
+ if (DoNotFail && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
+ S.getMainOp()->isSafeToRemove() &&
+ all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
+ // Find the number of elements, which forms full vectors.
+ unsigned PWSz = getFullVectorNumberOfElements(
+ TTI, UniqueValues.front()->getType(), UniqueValues.size());
+ PWSz = std::min<unsigned>(PWSz, VL.size());
+ if (PWSz == VL.size()) {
+ ReuseShuffleIndices.clear();
+ } else {
+ NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
+ NonUniqueValueVL.append(
+ PWSz - UniqueValues.size(),
+ PoisonValue::get(UniqueValues.front()->getType()));
+ // Check that extended with poisons operations are still valid for
+ // vectorization (div/rem are not allowed).
+ if (!getSameOpcode(NonUniqueValueVL, TLI).valid()) {
+ LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ return false;
}
- return true;
+ VL = NonUniqueValueVL;
}
- LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
- return false;
+ return true;
}
- VL = UniqueValues;
+ LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ return false;
}
+ VL = UniqueValues;
return true;
}
>From ed19c40818c7b267b2df2d500ee54bb9a29acfd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 11 Apr 2025 11:43:51 +0000
Subject: [PATCH 2/5] tryToFindDuplicates(): more descriptive naming
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 24 +++++++++++--------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2ad82165d3015..930b3efe52809 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9539,10 +9539,9 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
const TargetLibraryInfo &TLI,
const InstructionsState &S,
const BoUpSLP::EdgeInfo &UserTreeIdx,
- bool DoNotFail) {
+ bool TryPad) {
// Check that every instruction appears once in this bundle.
SmallVector<Value *> UniqueValues;
- SmallVector<Value *> NonUniqueValueVL;
SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
for (Value *V : VL) {
if (isConstant(V)) {
@@ -9578,7 +9577,7 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
(UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
return isa<UndefValue>(V) || !isConstant(V);
}))) {
- if (DoNotFail && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
+ if (TryPad && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
S.getMainOp()->isSafeToRemove() &&
all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
// Find the number of elements, which forms full vectors.
@@ -9586,19 +9585,24 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
TTI, UniqueValues.front()->getType(), UniqueValues.size());
PWSz = std::min<unsigned>(PWSz, VL.size());
if (PWSz == VL.size()) {
+ // We ended up with the same size after removing duplicates and
+ // upgrading the resulting vector size to a "nice size". Just keep
+ // the initial VL then.
ReuseShuffleIndices.clear();
} else {
- NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
- NonUniqueValueVL.append(
+ // Pad unique values with poison to grow the vector to a "nice" size
+ SmallVector<Value *> PaddedUniqueValues(UniqueValues.begin(),
+ UniqueValues.end());
+ PaddedUniqueValues.append(
PWSz - UniqueValues.size(),
PoisonValue::get(UniqueValues.front()->getType()));
// Check that extended with poisons operations are still valid for
// vectorization (div/rem are not allowed).
- if (!getSameOpcode(NonUniqueValueVL, TLI).valid()) {
+ if (!getSameOpcode(PaddedUniqueValues, TLI).valid()) {
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
return false;
}
- VL = NonUniqueValueVL;
+ VL = std::move(PaddedUniqueValues);
}
return true;
}
@@ -10014,9 +10018,9 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VL, unsigned Depth,
SmallVector<int> ReuseShuffleIndices;
SmallVector<Value *> NonUniqueValueVL(VL.begin(), VL.end());
auto TryToFindDuplicates = [&](const InstructionsState &S,
- bool DoNotFail = false) {
+ bool TryPad = false) {
if (tryToFindDuplicates(NonUniqueValueVL, ReuseShuffleIndices, *TTI, *TLI,
- S, UserTreeIdx, DoNotFail)) {
+ S, UserTreeIdx, TryPad)) {
VL = NonUniqueValueVL;
return true;
}
@@ -10082,7 +10086,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VL, unsigned Depth,
return;
// Check that every instruction appears once in this bundle.
- if (!TryToFindDuplicates(S, /*DoNotFail=*/true))
+ if (!TryToFindDuplicates(S, /*TryPad=*/true))
return;
// Perform specific checks for each particular instruction kind.
>From 6df22ca80daf3f58879c3df943a4c7f79cbfb335 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 11 Apr 2025 12:20:59 +0000
Subject: [PATCH 3/5] tryToFindDuplicates(): more consistent outputs
In particular, ReuseShuffleIndices is always in a consistent state,
which allows to call newTreeEntry with the same parameters regardless of
the result of tryToFindDuplicates().
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 56 ++++++++++---------
1 file changed, 31 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 930b3efe52809..7b3db42973082 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9531,20 +9531,25 @@ getMainAltOpsNoStateVL(ArrayRef<Value *> VL) {
}
/// Checks that every instruction appears once in the list and if not, packs
-/// them, building \p ReuseShuffleIndices mask. The list of unique scalars is
-/// extended by poison values to the whole register size.
+/// them, building \p ReuseShuffleIndices mask and mutating \p VL. The list of
+/// unique scalars is extended by poison values to the whole register size.
+///
+/// \returns false if \p VL could not be uniquified, in which case \p VL is
+/// unchanged and \p ReuseShuffleIndices is empty.
static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
SmallVectorImpl<int> &ReuseShuffleIndices,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
const InstructionsState &S,
const BoUpSLP::EdgeInfo &UserTreeIdx,
- bool TryPad) {
+ bool TryPad = false) {
// Check that every instruction appears once in this bundle.
SmallVector<Value *> UniqueValues;
SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
for (Value *V : VL) {
if (isConstant(V)) {
+ // Constants are always considered distinct, even if the same constant
+ // appears multiple times in VL.
ReuseShuffleIndices.emplace_back(
isa<PoisonValue>(V) ? PoisonMaskElem : UniqueValues.size());
UniqueValues.emplace_back(V);
@@ -9555,6 +9560,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
if (Res.second)
UniqueValues.emplace_back(V);
}
+
+ // Easy case: VL has unique values and a "natural" size
size_t NumUniqueScalarValues = UniqueValues.size();
bool IsFullVectors = hasFullVectorsOrPowerOf2(
TTI, getValueType(UniqueValues.front()), NumUniqueScalarValues);
@@ -9570,8 +9577,10 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
!hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
"for nodes with padding.\n");
+ ReuseShuffleIndices.clear();
return false;
}
+
LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
(UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
@@ -9600,6 +9609,7 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
// vectorization (div/rem are not allowed).
if (!getSameOpcode(PaddedUniqueValues, TLI).valid()) {
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ ReuseShuffleIndices.clear();
return false;
}
VL = std::move(PaddedUniqueValues);
@@ -9607,9 +9617,10 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
return true;
}
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ ReuseShuffleIndices.clear();
return false;
}
- VL = UniqueValues;
+ VL = std::move(UniqueValues);
return true;
}
@@ -10010,24 +10021,13 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
return true;
}
-void BoUpSLP::buildTreeRec(ArrayRef<Value *> VL, unsigned Depth,
+void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
const EdgeInfo &UserTreeIdx,
unsigned InterleaveFactor) {
- assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
+ assert((allConstant(VLRef) || allSameType(VLRef)) && "Invalid types!");
SmallVector<int> ReuseShuffleIndices;
- SmallVector<Value *> NonUniqueValueVL(VL.begin(), VL.end());
- auto TryToFindDuplicates = [&](const InstructionsState &S,
- bool TryPad = false) {
- if (tryToFindDuplicates(NonUniqueValueVL, ReuseShuffleIndices, *TTI, *TLI,
- S, UserTreeIdx, TryPad)) {
- VL = NonUniqueValueVL;
- return true;
- }
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
- return false;
- };
+ SmallVector<Value *> VL(VLRef.begin(), VLRef.end());
InstructionsState S = InstructionsState::invalid();
// Tries to build split node.
@@ -10073,11 +10073,12 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VL, unsigned Depth,
if (MainOp && AltOp && TrySplitNode(InstructionsState(MainOp, AltOp)))
return;
}
- if (!TryToPackDuplicates || TryToFindDuplicates(S)) {
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
- }
+ if (TryToPackDuplicates)
+ tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx);
+
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
+ ReuseShuffleIndices);
return;
}
@@ -10086,8 +10087,13 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VL, unsigned Depth,
return;
// Check that every instruction appears once in this bundle.
- if (!TryToFindDuplicates(S, /*TryPad=*/true))
+ if (!tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx,
+ /*TryPad=*/true)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
+ ReuseShuffleIndices);
return;
+ }
// Perform specific checks for each particular instruction kind.
bool IsScatterVectorizeUserTE =
@@ -10130,7 +10136,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VL, unsigned Depth,
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
BS.ScheduleRegionSize < BS.ScheduleRegionSizeLimit)
- registerNonVectorizableLoads(VL);
+ registerNonVectorizableLoads(ArrayRef(VL));
return;
}
ScheduleBundle Empty;
>From 7179326017305dc128752bb489c938a0dd8c22e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Mon, 14 Apr 2025 12:23:28 +0000
Subject: [PATCH 4/5] buildTree_rec(): Move common code to newGatherTreeEntry()
function
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 25 ++++++++++---------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7b3db42973082..5179c98799da0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4063,6 +4063,15 @@ class BoUpSLP {
}
#endif
+ /// Create a new gather TreeEntry
+ TreeEntry *newGatherTreeEntry(ArrayRef<Value *> VL,
+ const InstructionsState &S,
+ const EdgeInfo &UserTreeIdx,
+ ArrayRef<int> ReuseShuffleIndices = {}) {
+ auto Invalid = ScheduleBundle::invalid();
+ return newTreeEntry(VL, Invalid, S, UserTreeIdx, ReuseShuffleIndices);
+ }
+
/// Create a new VectorizableTree entry.
TreeEntry *newTreeEntry(ArrayRef<Value *> VL, ScheduleBundle &Bundle,
const InstructionsState &S,
@@ -10076,9 +10085,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
if (TryToPackDuplicates)
tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx);
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
@@ -10089,9 +10096,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Check that every instruction appears once in this bundle.
if (!tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx,
/*TryPad=*/true)) {
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
@@ -10104,9 +10109,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
TreeEntry::EntryState State = getScalarsVectorizationState(
S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
if (State == TreeEntry::NeedToGather) {
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
@@ -10130,9 +10133,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Last chance to try to vectorize alternate node.
if (S.isAltShuffle() && ReuseShuffleIndices.empty() && TrySplitNode(S))
return;
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
BS.ScheduleRegionSize < BS.ScheduleRegionSizeLimit)
>From dcd12cfec5153379e6a7f91810f0b2e5126b8a8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Mon, 14 Apr 2025 11:21:22 +0000
Subject: [PATCH 5/5] buildTree_rec(): Single point of definition for legality
characteristics
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 91 +++++++++++--------
1 file changed, 54 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5179c98799da0..986b7d114564f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4260,13 +4260,34 @@ class BoUpSLP {
bool areAltOperandsProfitable(const InstructionsState &S,
ArrayRef<Value *> VL) const;
+ /// Contains all the outputs of legality analysis for a list of values to
+ /// vectorize.
+ class ScalarsVectorizationLegality {
+ InstructionsState S;
+ bool IsLegal;
+ bool TryToFindDuplicates;
+ bool TrySplitVectorize;
+
+ public:
+ ScalarsVectorizationLegality(InstructionsState S, bool IsLegal,
+ bool TryToFindDuplicates = true,
+ bool TrySplitVectorize = false)
+ : S(S), IsLegal(IsLegal), TryToFindDuplicates(TryToFindDuplicates),
+ TrySplitVectorize(TrySplitVectorize) {
+ assert((!IsLegal || (S.valid() && TryToFindDuplicates)) &&
+ "Inconsistent state");
+ }
+ const InstructionsState &getInstructionsState() const { return S; };
+ bool isLegal() const { return IsLegal; }
+ bool tryToFindDuplicates() const { return TryToFindDuplicates; }
+ bool trySplitVectorize() const { return TrySplitVectorize; }
+ };
+
/// Checks if the specified list of the instructions/values can be vectorized
/// in general.
- bool isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
- const EdgeInfo &UserTreeIdx,
- InstructionsState &S,
- bool &TryToFindDuplicates,
- bool &TrySplitVectorize) const;
+ ScalarsVectorizationLegality
+ getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
+ const EdgeInfo &UserTreeIdx) const;
/// Checks if the specified list of the instructions/values can be vectorized
/// and fills required data before actual scheduling of the instructions.
@@ -9764,16 +9785,12 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
return true;
}
-bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
- const EdgeInfo &UserTreeIdx,
- InstructionsState &S,
- bool &TryToFindDuplicates,
- bool &TrySplitVectorize) const {
+BoUpSLP::ScalarsVectorizationLegality
+BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
+ const EdgeInfo &UserTreeIdx) const {
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
- S = getSameOpcode(VL, *TLI);
- TryToFindDuplicates = true;
- TrySplitVectorize = false;
+ InstructionsState S = getSameOpcode(VL, *TLI);
// Don't go into catchswitch blocks, which can happen with PHIs.
// Such blocks can only have PHIs and the catchswitch. There is no
@@ -9781,8 +9798,8 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (S && isa<CatchSwitchInst>(S.getMainOp()->getParent()->getTerminator())) {
LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
// Check if this is a duplicate of another entry.
@@ -9792,14 +9809,14 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (E->isSame(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
<< ".\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
if (all_of(VL, [&](Value *V) {
return isa<PoisonValue>(V) || Values.contains(V);
})) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
@@ -9816,7 +9833,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
cast<Instruction>(I)->getOpcode() == S.getOpcode();
})))) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't handle scalable vectors
@@ -9824,15 +9841,15 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
isa<ScalableVectorType>(
cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't handle vectors.
if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
// If all of the operands are identical or constant we have a simple solution.
@@ -9922,11 +9939,12 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (!S) {
LLVM_DEBUG(dbgs() << "SLP: Try split and if failed, gathering due to "
"C,S,B,O, small shuffle. \n");
- TrySplitVectorize = true;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/true,
+ /*TrySplitVectorize=*/true);
}
LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't vectorize ephemeral values.
@@ -9936,8 +9954,8 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
<< ") is ephemeral.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
}
}
@@ -9986,7 +10004,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (PreferScalarize) {
LLVM_DEBUG(dbgs() << "SLP: The instructions are in tree and alternate "
"node is not profitable.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
@@ -9995,7 +10013,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
for (Value *V : VL) {
if (UserIgnoreList->contains(V)) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
@@ -10025,9 +10043,9 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
// Do not vectorize EH and non-returning blocks, not profitable in most
// cases.
LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
- return true;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/true);
}
void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
@@ -10038,7 +10056,6 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
SmallVector<int> ReuseShuffleIndices;
SmallVector<Value *> VL(VLRef.begin(), VLRef.end());
- InstructionsState S = InstructionsState::invalid();
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState) {
SmallVector<Value *> Op1, Op2;
@@ -10072,17 +10089,17 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
return true;
};
- bool TryToPackDuplicates;
- bool TrySplitVectorize;
- if (!isLegalToVectorizeScalars(VL, Depth, UserTreeIdx, S, TryToPackDuplicates,
- TrySplitVectorize)) {
- if (TrySplitVectorize) {
+ ScalarsVectorizationLegality SVL =
+ getScalarsVectorizationLegality(VL, Depth, UserTreeIdx);
+ const InstructionsState &S = SVL.getInstructionsState();
+ if (!SVL.isLegal()) {
+ if (SVL.trySplitVectorize()) {
auto [MainOp, AltOp] = getMainAltOpsNoStateVL(VL);
// Last chance to try to vectorize alternate node.
if (MainOp && AltOp && TrySplitNode(InstructionsState(MainOp, AltOp)))
return;
}
- if (TryToPackDuplicates)
+ if (SVL.tryToFindDuplicates())
tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx);
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
More information about the llvm-commits
mailing list