[llvm] [SLP] Simplify buildTree() and callees (NFC) (PR #135766)
Gaëtan Bossu via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 15 02:25:25 PDT 2025
https://github.com/gbossu created https://github.com/llvm/llvm-project/pull/135766
This NFC aims to simplify the control-flow and interfaces used in `buildTree(). The point is to make it easier to understand where decisions for legality and scalar de-duplication are made.
In particular:
- Limit indentation and rename some variables in tryToFindDuplicates()
- SImplify tryToFindDuplicates() so it always gives consistent outputs for `VL` and `ReuseShuffleIndices`. This makes it possible to use the same code for building gather `TreeEntry` everywhere, and allows to remove the `TryToFindDuplicates` lambda.
- Have a single point of definition for legality decisions, instead of passing multiple variables with a large scope by reference. This makes it clear where all those decisions are made.
>From 647f6e5d004bc7f11fb22d59049d5aeaf0234b44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 11 Apr 2025 10:50:11 +0000
Subject: [PATCH 1/5] tryToFindDuplicates(): Limit indentation
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 79 ++++++++++---------
1 file changed, 40 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a0837ab214219..7e59178425a98 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9518,49 +9518,50 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
if (NumUniqueScalarValues == VL.size() &&
(VectorizeNonPowerOf2 || IsFullVectors)) {
ReuseShuffleIndices.clear();
- } else {
- // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
- if ((UserTreeIdx.UserTE &&
- UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(TTI)) ||
- !hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
- LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
- "for nodes with padding.\n");
- return false;
- }
- LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
- if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
- (UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
- return isa<UndefValue>(V) || !isConstant(V);
- }))) {
- if (DoNotFail && UniquePositions.size() > 1 &&
- NumUniqueScalarValues > 1 && S.getMainOp()->isSafeToRemove() &&
- all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
- // Find the number of elements, which forms full vectors.
- unsigned PWSz = getFullVectorNumberOfElements(
- TTI, UniqueValues.front()->getType(), UniqueValues.size());
- PWSz = std::min<unsigned>(PWSz, VL.size());
- if (PWSz == VL.size()) {
- ReuseShuffleIndices.clear();
- } else {
- NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
- NonUniqueValueVL.append(
- PWSz - UniqueValues.size(),
- PoisonValue::get(UniqueValues.front()->getType()));
- // Check that extended with poisons operations are still valid for
- // vectorization (div/rem are not allowed).
- if (!getSameOpcode(NonUniqueValueVL, TLI).valid()) {
- LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
- return false;
- }
- VL = NonUniqueValueVL;
+ return true;
+ }
+
+ // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
+ if ((UserTreeIdx.UserTE &&
+ UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(TTI)) ||
+ !hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
+ LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
+ "for nodes with padding.\n");
+ return false;
+ }
+ LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
+ if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
+ (UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
+ return isa<UndefValue>(V) || !isConstant(V);
+ }))) {
+ if (DoNotFail && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
+ S.getMainOp()->isSafeToRemove() &&
+ all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
+ // Find the number of elements, which forms full vectors.
+ unsigned PWSz = getFullVectorNumberOfElements(
+ TTI, UniqueValues.front()->getType(), UniqueValues.size());
+ PWSz = std::min<unsigned>(PWSz, VL.size());
+ if (PWSz == VL.size()) {
+ ReuseShuffleIndices.clear();
+ } else {
+ NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
+ NonUniqueValueVL.append(
+ PWSz - UniqueValues.size(),
+ PoisonValue::get(UniqueValues.front()->getType()));
+ // Check that extended with poisons operations are still valid for
+ // vectorization (div/rem are not allowed).
+ if (!getSameOpcode(NonUniqueValueVL, TLI).valid()) {
+ LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ return false;
}
- return true;
+ VL = NonUniqueValueVL;
}
- LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
- return false;
+ return true;
}
- VL = UniqueValues;
+ LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ return false;
}
+ VL = UniqueValues;
return true;
}
>From 429e618f6458fc9b652eff6a531b86cd7f8665c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 11 Apr 2025 11:43:51 +0000
Subject: [PATCH 2/5] tryToFindDuplicates(): more descriptive naming
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 24 +++++++++++--------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7e59178425a98..3704e674671ad 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9495,10 +9495,9 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
const TargetLibraryInfo &TLI,
const InstructionsState &S,
const BoUpSLP::EdgeInfo &UserTreeIdx,
- bool DoNotFail) {
+ bool TryPad) {
// Check that every instruction appears once in this bundle.
SmallVector<Value *> UniqueValues;
- SmallVector<Value *> NonUniqueValueVL;
SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
for (Value *V : VL) {
if (isConstant(V)) {
@@ -9534,7 +9533,7 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
(UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
return isa<UndefValue>(V) || !isConstant(V);
}))) {
- if (DoNotFail && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
+ if (TryPad && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
S.getMainOp()->isSafeToRemove() &&
all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
// Find the number of elements, which forms full vectors.
@@ -9542,19 +9541,24 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
TTI, UniqueValues.front()->getType(), UniqueValues.size());
PWSz = std::min<unsigned>(PWSz, VL.size());
if (PWSz == VL.size()) {
+ // We ended up with the same size after removing duplicates and
+ // upgrading the resulting vector size to a "nice size". Just keep
+ // the initial VL then.
ReuseShuffleIndices.clear();
} else {
- NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
- NonUniqueValueVL.append(
+ // Pad unique values with poison to grow the vector to a "nice" size
+ SmallVector<Value *> PaddedUniqueValues(UniqueValues.begin(),
+ UniqueValues.end());
+ PaddedUniqueValues.append(
PWSz - UniqueValues.size(),
PoisonValue::get(UniqueValues.front()->getType()));
// Check that extended with poisons operations are still valid for
// vectorization (div/rem are not allowed).
- if (!getSameOpcode(NonUniqueValueVL, TLI).valid()) {
+ if (!getSameOpcode(PaddedUniqueValues, TLI).valid()) {
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
return false;
}
- VL = NonUniqueValueVL;
+ VL = std::move(PaddedUniqueValues);
}
return true;
}
@@ -9952,9 +9956,9 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
SmallVector<int> ReuseShuffleIndices;
SmallVector<Value *> NonUniqueValueVL(VL.begin(), VL.end());
auto TryToFindDuplicates = [&](const InstructionsState &S,
- bool DoNotFail = false) {
+ bool TryPad = false) {
if (tryToFindDuplicates(NonUniqueValueVL, ReuseShuffleIndices, *TTI, *TLI,
- S, UserTreeIdx, DoNotFail)) {
+ S, UserTreeIdx, TryPad)) {
VL = NonUniqueValueVL;
return true;
}
@@ -10020,7 +10024,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
return;
// Check that every instruction appears once in this bundle.
- if (!TryToFindDuplicates(S, /*DoNotFail=*/true))
+ if (!TryToFindDuplicates(S, /*TryPad=*/true))
return;
// Perform specific checks for each particular instruction kind.
>From c352ef27a986a6db3b5daf134e180300a4ca794b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 11 Apr 2025 12:20:59 +0000
Subject: [PATCH 3/5] tryToFindDuplicates(): more consistent outputs
In particular, ReuseShuffleIndices is always in a consistent state,
which allows to call newTreeEntry with the same parameters regardless of
the result of tryToFindDuplicates().
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 56 ++++++++++---------
1 file changed, 31 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3704e674671ad..d92b30bfc24e5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9487,20 +9487,25 @@ getMainAltOpsNoStateVL(ArrayRef<Value *> VL) {
}
/// Checks that every instruction appears once in the list and if not, packs
-/// them, building \p ReuseShuffleIndices mask. The list of unique scalars is
-/// extended by poison values to the whole register size.
+/// them, building \p ReuseShuffleIndices mask and mutating \p VL. The list of
+/// unique scalars is extended by poison values to the whole register size.
+///
+/// \returns false if \p VL could not be uniquified, in which case \p VL is
+/// unchanged and \p ReuseShuffleIndices is empty.
static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
SmallVectorImpl<int> &ReuseShuffleIndices,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
const InstructionsState &S,
const BoUpSLP::EdgeInfo &UserTreeIdx,
- bool TryPad) {
+ bool TryPad = false) {
// Check that every instruction appears once in this bundle.
SmallVector<Value *> UniqueValues;
SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
for (Value *V : VL) {
if (isConstant(V)) {
+ // Constants are always considered distinct, even if the same constant
+ // appears multiple times in VL.
ReuseShuffleIndices.emplace_back(
isa<PoisonValue>(V) ? PoisonMaskElem : UniqueValues.size());
UniqueValues.emplace_back(V);
@@ -9511,6 +9516,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
if (Res.second)
UniqueValues.emplace_back(V);
}
+
+ // Easy case: VL has unique values and a "natural" size
size_t NumUniqueScalarValues = UniqueValues.size();
bool IsFullVectors = hasFullVectorsOrPowerOf2(
TTI, getValueType(UniqueValues.front()), NumUniqueScalarValues);
@@ -9526,8 +9533,10 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
!hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
"for nodes with padding.\n");
+ ReuseShuffleIndices.clear();
return false;
}
+
LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
(UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
@@ -9556,6 +9565,7 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
// vectorization (div/rem are not allowed).
if (!getSameOpcode(PaddedUniqueValues, TLI).valid()) {
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ ReuseShuffleIndices.clear();
return false;
}
VL = std::move(PaddedUniqueValues);
@@ -9563,9 +9573,10 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
return true;
}
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ ReuseShuffleIndices.clear();
return false;
}
- VL = UniqueValues;
+ VL = std::move(UniqueValues);
return true;
}
@@ -9948,24 +9959,13 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
return true;
}
-void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
+void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
const EdgeInfo &UserTreeIdx,
unsigned InterleaveFactor) {
- assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
+ assert((allConstant(VLRef) || allSameType(VLRef)) && "Invalid types!");
SmallVector<int> ReuseShuffleIndices;
- SmallVector<Value *> NonUniqueValueVL(VL.begin(), VL.end());
- auto TryToFindDuplicates = [&](const InstructionsState &S,
- bool TryPad = false) {
- if (tryToFindDuplicates(NonUniqueValueVL, ReuseShuffleIndices, *TTI, *TLI,
- S, UserTreeIdx, TryPad)) {
- VL = NonUniqueValueVL;
- return true;
- }
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
- return false;
- };
+ SmallVector<Value *> VL(VLRef.begin(), VLRef.end());
InstructionsState S = InstructionsState::invalid();
// Tries to build split node.
@@ -10011,11 +10011,12 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
if (MainOp && AltOp && TrySplitNode(InstructionsState(MainOp, AltOp)))
return;
}
- if (!TryToPackDuplicates || TryToFindDuplicates(S)) {
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
- }
+ if (TryToPackDuplicates)
+ tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx);
+
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
+ ReuseShuffleIndices);
return;
}
@@ -10024,8 +10025,13 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
return;
// Check that every instruction appears once in this bundle.
- if (!TryToFindDuplicates(S, /*TryPad=*/true))
+ if (!tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx,
+ /*TryPad=*/true)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
+ ReuseShuffleIndices);
return;
+ }
// Perform specific checks for each particular instruction kind.
bool IsScatterVectorizeUserTE =
@@ -10068,7 +10074,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
BS.ScheduleRegionSize < BS.ScheduleRegionSizeLimit)
- registerNonVectorizableLoads(VL);
+ registerNonVectorizableLoads(ArrayRef(VL));
return;
}
ScheduleBundle Empty;
>From 2de50c647e41babb3aa5bcbbbe17d8451ca61fd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Mon, 14 Apr 2025 12:23:28 +0000
Subject: [PATCH 4/5] buildTree_rec(): Move common code to
CreateGatherTreeEntry() lambda
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 22 +++++++++----------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d92b30bfc24e5..86882b94ed19f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9967,6 +9967,12 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
SmallVector<int> ReuseShuffleIndices;
SmallVector<Value *> VL(VLRef.begin(), VLRef.end());
+ auto CreateGatherTreeEntry = [&](const InstructionsState &S) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
+ ReuseShuffleIndices);
+ };
+
InstructionsState S = InstructionsState::invalid();
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState) {
@@ -10014,9 +10020,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
if (TryToPackDuplicates)
tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx);
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ CreateGatherTreeEntry(S);
return;
}
@@ -10027,9 +10031,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
// Check that every instruction appears once in this bundle.
if (!tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx,
/*TryPad=*/true)) {
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ CreateGatherTreeEntry(S);
return;
}
@@ -10042,9 +10044,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
TreeEntry::EntryState State = getScalarsVectorizationState(
S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
if (State == TreeEntry::NeedToGather) {
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ CreateGatherTreeEntry(S);
return;
}
@@ -10068,9 +10068,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
// Last chance to try to vectorize alternate node.
if (S.isAltShuffle() && ReuseShuffleIndices.empty() && TrySplitNode(S))
return;
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ CreateGatherTreeEntry(S);
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
BS.ScheduleRegionSize < BS.ScheduleRegionSizeLimit)
>From 612924ad8baddbc4a0f7afd9bd9e83bcc8777c9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Mon, 14 Apr 2025 11:21:22 +0000
Subject: [PATCH 5/5] buildTree_rec(): Single point of definition for legality
characteristics
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 91 +++++++++++--------
1 file changed, 54 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 86882b94ed19f..c738c47318ad7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4248,13 +4248,34 @@ class BoUpSLP {
bool areAltOperandsProfitable(const InstructionsState &S,
ArrayRef<Value *> VL) const;
+ /// Contains all the outputs of legality analysis for a list of values to
+ /// vectorize.
+ class ScalarsVectorizationLegality {
+ InstructionsState S;
+ bool IsLegal;
+ bool TryToFindDuplicates;
+ bool TrySplitVectorize;
+
+ public:
+ ScalarsVectorizationLegality(InstructionsState S, bool IsLegal,
+ bool TryToFindDuplicates = true,
+ bool TrySplitVectorize = false)
+ : S(S), IsLegal(IsLegal), TryToFindDuplicates(TryToFindDuplicates),
+ TrySplitVectorize(TrySplitVectorize) {
+ assert((!IsLegal || (S.valid() && TryToFindDuplicates)) &&
+ "Inconsistent state");
+ }
+ const InstructionsState &getInstructionsState() const { return S; };
+ bool isLegal() const { return IsLegal; }
+ bool tryToFindDuplicates() const { return TryToFindDuplicates; }
+ bool trySplitVectorize() const { return TrySplitVectorize; }
+ };
+
/// Checks if the specified list of the instructions/values can be vectorized
/// in general.
- bool isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
- const EdgeInfo &UserTreeIdx,
- InstructionsState &S,
- bool &TryToFindDuplicates,
- bool &TrySplitVectorize) const;
+ ScalarsVectorizationLegality
+ getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
+ const EdgeInfo &UserTreeIdx) const;
/// Checks if the specified list of the instructions/values can be vectorized
/// and fills required data before actual scheduling of the instructions.
@@ -9693,16 +9714,12 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
return true;
}
-bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
- const EdgeInfo &UserTreeIdx,
- InstructionsState &S,
- bool &TryToFindDuplicates,
- bool &TrySplitVectorize) const {
+BoUpSLP::ScalarsVectorizationLegality
+BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
+ const EdgeInfo &UserTreeIdx) const {
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
- S = getSameOpcode(VL, *TLI);
- TryToFindDuplicates = true;
- TrySplitVectorize = false;
+ InstructionsState S = getSameOpcode(VL, *TLI);
// Don't go into catchswitch blocks, which can happen with PHIs.
// Such blocks can only have PHIs and the catchswitch. There is no
@@ -9710,8 +9727,8 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (S && isa<CatchSwitchInst>(S.getMainOp()->getParent()->getTerminator())) {
LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
// Check if this is a duplicate of another entry.
@@ -9721,14 +9738,14 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (E->isSame(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
<< ".\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
if (all_of(VL, [&](Value *V) {
return isa<PoisonValue>(V) || Values.contains(V);
})) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
@@ -9745,7 +9762,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
cast<Instruction>(I)->getOpcode() == S.getOpcode();
})))) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't handle scalable vectors
@@ -9753,15 +9770,15 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
isa<ScalableVectorType>(
cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't handle vectors.
if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
// If all of the operands are identical or constant we have a simple solution.
@@ -9851,11 +9868,12 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (!S) {
LLVM_DEBUG(dbgs() << "SLP: Try split and if failed, gathering due to "
"C,S,B,O, small shuffle. \n");
- TrySplitVectorize = true;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/true,
+ /*TrySplitVectorize=*/true);
}
LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't vectorize ephemeral values.
@@ -9865,8 +9883,8 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
<< ") is ephemeral.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
}
}
@@ -9915,7 +9933,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (PreferScalarize) {
LLVM_DEBUG(dbgs() << "SLP: The instructions are in tree and alternate "
"node is not profitable.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
@@ -9924,7 +9942,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
for (Value *V : VL) {
if (UserIgnoreList->contains(V)) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
@@ -9954,9 +9972,9 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
// Do not vectorize EH and non-returning blocks, not profitable in most
// cases.
LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
- return true;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/true);
}
void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
@@ -9973,7 +9991,6 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
ReuseShuffleIndices);
};
- InstructionsState S = InstructionsState::invalid();
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState) {
SmallVector<Value *> Op1, Op2;
@@ -10007,17 +10024,17 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
return true;
};
- bool TryToPackDuplicates;
- bool TrySplitVectorize;
- if (!isLegalToVectorizeScalars(VL, Depth, UserTreeIdx, S, TryToPackDuplicates,
- TrySplitVectorize)) {
- if (TrySplitVectorize) {
+ ScalarsVectorizationLegality SVL =
+ getScalarsVectorizationLegality(VL, Depth, UserTreeIdx);
+ const InstructionsState &S = SVL.getInstructionsState();
+ if (!SVL.isLegal()) {
+ if (SVL.trySplitVectorize()) {
auto [MainOp, AltOp] = getMainAltOpsNoStateVL(VL);
// Last chance to try to vectorize alternate node.
if (MainOp && AltOp && TrySplitNode(InstructionsState(MainOp, AltOp)))
return;
}
- if (TryToPackDuplicates)
+ if (SVL.tryToFindDuplicates())
tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx);
CreateGatherTreeEntry(S);
More information about the llvm-commits
mailing list