[llvm] [SLP] Simplify buildTree() and callees (NFC) (PR #135766)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 15 02:26:00 PDT 2025
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/135766 at github.com>
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Gaëtan Bossu (gbossu)
<details>
<summary>Changes</summary>
This NFC aims to simplify the control-flow and interfaces used in `buildTree(). The point is to make it easier to understand where decisions for legality and scalar de-duplication are made.
In particular:
- Limit indentation and rename some variables in tryToFindDuplicates()
- SImplify tryToFindDuplicates() so it always gives consistent outputs for `VL` and `ReuseShuffleIndices`. This makes it possible to use the same code for building gather `TreeEntry` everywhere, and allows to remove the `TryToFindDuplicates` lambda.
- Have a single point of definition for legality decisions, instead of passing multiple variables with a large scope by reference. This makes it clear where all those decisions are made.
---
Full diff: https://github.com/llvm/llvm-project/pull/135766.diff
1 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+129-103)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a0837ab214219..c738c47318ad7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4248,13 +4248,34 @@ class BoUpSLP {
bool areAltOperandsProfitable(const InstructionsState &S,
ArrayRef<Value *> VL) const;
+ /// Contains all the outputs of legality analysis for a list of values to
+ /// vectorize.
+ class ScalarsVectorizationLegality {
+ InstructionsState S;
+ bool IsLegal;
+ bool TryToFindDuplicates;
+ bool TrySplitVectorize;
+
+ public:
+ ScalarsVectorizationLegality(InstructionsState S, bool IsLegal,
+ bool TryToFindDuplicates = true,
+ bool TrySplitVectorize = false)
+ : S(S), IsLegal(IsLegal), TryToFindDuplicates(TryToFindDuplicates),
+ TrySplitVectorize(TrySplitVectorize) {
+ assert((!IsLegal || (S.valid() && TryToFindDuplicates)) &&
+ "Inconsistent state");
+ }
+ const InstructionsState &getInstructionsState() const { return S; };
+ bool isLegal() const { return IsLegal; }
+ bool tryToFindDuplicates() const { return TryToFindDuplicates; }
+ bool trySplitVectorize() const { return TrySplitVectorize; }
+ };
+
/// Checks if the specified list of the instructions/values can be vectorized
/// in general.
- bool isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
- const EdgeInfo &UserTreeIdx,
- InstructionsState &S,
- bool &TryToFindDuplicates,
- bool &TrySplitVectorize) const;
+ ScalarsVectorizationLegality
+ getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
+ const EdgeInfo &UserTreeIdx) const;
/// Checks if the specified list of the instructions/values can be vectorized
/// and fills required data before actual scheduling of the instructions.
@@ -9487,21 +9508,25 @@ getMainAltOpsNoStateVL(ArrayRef<Value *> VL) {
}
/// Checks that every instruction appears once in the list and if not, packs
-/// them, building \p ReuseShuffleIndices mask. The list of unique scalars is
-/// extended by poison values to the whole register size.
+/// them, building \p ReuseShuffleIndices mask and mutating \p VL. The list of
+/// unique scalars is extended by poison values to the whole register size.
+///
+/// \returns false if \p VL could not be uniquified, in which case \p VL is
+/// unchanged and \p ReuseShuffleIndices is empty.
static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
SmallVectorImpl<int> &ReuseShuffleIndices,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
const InstructionsState &S,
const BoUpSLP::EdgeInfo &UserTreeIdx,
- bool DoNotFail) {
+ bool TryPad = false) {
// Check that every instruction appears once in this bundle.
SmallVector<Value *> UniqueValues;
- SmallVector<Value *> NonUniqueValueVL;
SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
for (Value *V : VL) {
if (isConstant(V)) {
+ // Constants are always considered distinct, even if the same constant
+ // appears multiple times in VL.
ReuseShuffleIndices.emplace_back(
isa<PoisonValue>(V) ? PoisonMaskElem : UniqueValues.size());
UniqueValues.emplace_back(V);
@@ -9512,55 +9537,67 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
if (Res.second)
UniqueValues.emplace_back(V);
}
+
+ // Easy case: VL has unique values and a "natural" size
size_t NumUniqueScalarValues = UniqueValues.size();
bool IsFullVectors = hasFullVectorsOrPowerOf2(
TTI, getValueType(UniqueValues.front()), NumUniqueScalarValues);
if (NumUniqueScalarValues == VL.size() &&
(VectorizeNonPowerOf2 || IsFullVectors)) {
ReuseShuffleIndices.clear();
- } else {
- // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
- if ((UserTreeIdx.UserTE &&
- UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(TTI)) ||
- !hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
- LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
- "for nodes with padding.\n");
- return false;
- }
- LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
- if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
- (UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
- return isa<UndefValue>(V) || !isConstant(V);
- }))) {
- if (DoNotFail && UniquePositions.size() > 1 &&
- NumUniqueScalarValues > 1 && S.getMainOp()->isSafeToRemove() &&
- all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
- // Find the number of elements, which forms full vectors.
- unsigned PWSz = getFullVectorNumberOfElements(
- TTI, UniqueValues.front()->getType(), UniqueValues.size());
- PWSz = std::min<unsigned>(PWSz, VL.size());
- if (PWSz == VL.size()) {
+ return true;
+ }
+
+ // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
+ if ((UserTreeIdx.UserTE &&
+ UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(TTI)) ||
+ !hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
+ LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
+ "for nodes with padding.\n");
+ ReuseShuffleIndices.clear();
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
+ if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
+ (UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
+ return isa<UndefValue>(V) || !isConstant(V);
+ }))) {
+ if (TryPad && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
+ S.getMainOp()->isSafeToRemove() &&
+ all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
+ // Find the number of elements, which forms full vectors.
+ unsigned PWSz = getFullVectorNumberOfElements(
+ TTI, UniqueValues.front()->getType(), UniqueValues.size());
+ PWSz = std::min<unsigned>(PWSz, VL.size());
+ if (PWSz == VL.size()) {
+ // We ended up with the same size after removing duplicates and
+ // upgrading the resulting vector size to a "nice size". Just keep
+ // the initial VL then.
+ ReuseShuffleIndices.clear();
+ } else {
+ // Pad unique values with poison to grow the vector to a "nice" size
+ SmallVector<Value *> PaddedUniqueValues(UniqueValues.begin(),
+ UniqueValues.end());
+ PaddedUniqueValues.append(
+ PWSz - UniqueValues.size(),
+ PoisonValue::get(UniqueValues.front()->getType()));
+ // Check that extended with poisons operations are still valid for
+ // vectorization (div/rem are not allowed).
+ if (!getSameOpcode(PaddedUniqueValues, TLI).valid()) {
+ LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
ReuseShuffleIndices.clear();
- } else {
- NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
- NonUniqueValueVL.append(
- PWSz - UniqueValues.size(),
- PoisonValue::get(UniqueValues.front()->getType()));
- // Check that extended with poisons operations are still valid for
- // vectorization (div/rem are not allowed).
- if (!getSameOpcode(NonUniqueValueVL, TLI).valid()) {
- LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
- return false;
- }
- VL = NonUniqueValueVL;
+ return false;
}
- return true;
+ VL = std::move(PaddedUniqueValues);
}
- LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
- return false;
+ return true;
}
- VL = UniqueValues;
+ LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
+ ReuseShuffleIndices.clear();
+ return false;
}
+ VL = std::move(UniqueValues);
return true;
}
@@ -9677,16 +9714,12 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
return true;
}
-bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
- const EdgeInfo &UserTreeIdx,
- InstructionsState &S,
- bool &TryToFindDuplicates,
- bool &TrySplitVectorize) const {
+BoUpSLP::ScalarsVectorizationLegality
+BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
+ const EdgeInfo &UserTreeIdx) const {
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
- S = getSameOpcode(VL, *TLI);
- TryToFindDuplicates = true;
- TrySplitVectorize = false;
+ InstructionsState S = getSameOpcode(VL, *TLI);
// Don't go into catchswitch blocks, which can happen with PHIs.
// Such blocks can only have PHIs and the catchswitch. There is no
@@ -9694,8 +9727,8 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (S && isa<CatchSwitchInst>(S.getMainOp()->getParent()->getTerminator())) {
LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
// Check if this is a duplicate of another entry.
@@ -9705,14 +9738,14 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (E->isSame(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
<< ".\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
if (all_of(VL, [&](Value *V) {
return isa<PoisonValue>(V) || Values.contains(V);
})) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
@@ -9729,7 +9762,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
cast<Instruction>(I)->getOpcode() == S.getOpcode();
})))) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't handle scalable vectors
@@ -9737,15 +9770,15 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
isa<ScalableVectorType>(
cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't handle vectors.
if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
// If all of the operands are identical or constant we have a simple solution.
@@ -9835,11 +9868,12 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (!S) {
LLVM_DEBUG(dbgs() << "SLP: Try split and if failed, gathering due to "
"C,S,B,O, small shuffle. \n");
- TrySplitVectorize = true;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/true,
+ /*TrySplitVectorize=*/true);
}
LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't vectorize ephemeral values.
@@ -9849,8 +9883,8 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
<< ") is ephemeral.\n");
// Do not try to pack to avoid extra instructions here.
- TryToFindDuplicates = false;
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
+ /*TryToFindDuplicates=*/false);
}
}
}
@@ -9899,7 +9933,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (PreferScalarize) {
LLVM_DEBUG(dbgs() << "SLP: The instructions are in tree and alternate "
"node is not profitable.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
@@ -9908,7 +9942,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
for (Value *V : VL) {
if (UserIgnoreList->contains(V)) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
@@ -9938,31 +9972,25 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
// Do not vectorize EH and non-returning blocks, not profitable in most
// cases.
LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
- return false;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
- return true;
+ return ScalarsVectorizationLegality(S, /*IsLegal=*/true);
}
-void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
+void BoUpSLP::buildTree_rec(ArrayRef<Value *> VLRef, unsigned Depth,
const EdgeInfo &UserTreeIdx,
unsigned InterleaveFactor) {
- assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
+ assert((allConstant(VLRef) || allSameType(VLRef)) && "Invalid types!");
SmallVector<int> ReuseShuffleIndices;
- SmallVector<Value *> NonUniqueValueVL(VL.begin(), VL.end());
- auto TryToFindDuplicates = [&](const InstructionsState &S,
- bool DoNotFail = false) {
- if (tryToFindDuplicates(NonUniqueValueVL, ReuseShuffleIndices, *TTI, *TLI,
- S, UserTreeIdx, DoNotFail)) {
- VL = NonUniqueValueVL;
- return true;
- }
+ SmallVector<Value *> VL(VLRef.begin(), VLRef.end());
+
+ auto CreateGatherTreeEntry = [&](const InstructionsState &S) {
auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
- return false;
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
+ ReuseShuffleIndices);
};
- InstructionsState S = InstructionsState::invalid();
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState) {
SmallVector<Value *> Op1, Op2;
@@ -9996,21 +10024,20 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
return true;
};
- bool TryToPackDuplicates;
- bool TrySplitVectorize;
- if (!isLegalToVectorizeScalars(VL, Depth, UserTreeIdx, S, TryToPackDuplicates,
- TrySplitVectorize)) {
- if (TrySplitVectorize) {
+ ScalarsVectorizationLegality SVL =
+ getScalarsVectorizationLegality(VL, Depth, UserTreeIdx);
+ const InstructionsState &S = SVL.getInstructionsState();
+ if (!SVL.isLegal()) {
+ if (SVL.trySplitVectorize()) {
auto [MainOp, AltOp] = getMainAltOpsNoStateVL(VL);
// Last chance to try to vectorize alternate node.
if (MainOp && AltOp && TrySplitNode(InstructionsState(MainOp, AltOp)))
return;
}
- if (!TryToPackDuplicates || TryToFindDuplicates(S)) {
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
- }
+ if (SVL.tryToFindDuplicates())
+ tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx);
+
+ CreateGatherTreeEntry(S);
return;
}
@@ -10019,8 +10046,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
return;
// Check that every instruction appears once in this bundle.
- if (!TryToFindDuplicates(S, /*DoNotFail=*/true))
+ if (!tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx,
+ /*TryPad=*/true)) {
+ CreateGatherTreeEntry(S);
return;
+ }
// Perform specific checks for each particular instruction kind.
bool IsScatterVectorizeUserTE =
@@ -10031,9 +10061,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
TreeEntry::EntryState State = getScalarsVectorizationState(
S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
if (State == TreeEntry::NeedToGather) {
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ CreateGatherTreeEntry(S);
return;
}
@@ -10057,13 +10085,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
// Last chance to try to vectorize alternate node.
if (S.isAltShuffle() && ReuseShuffleIndices.empty() && TrySplitNode(S))
return;
- auto Invalid = ScheduleBundle::invalid();
- newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices);
+ CreateGatherTreeEntry(S);
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
BS.ScheduleRegionSize < BS.ScheduleRegionSizeLimit)
- registerNonVectorizableLoads(VL);
+ registerNonVectorizableLoads(ArrayRef(VL));
return;
}
ScheduleBundle Empty;
``````````
</details>
https://github.com/llvm/llvm-project/pull/135766
More information about the llvm-commits
mailing list