[llvm] [SLP][NFC] Try to cleanup and better document some isGatherShuffledEntry code. (PR #69384)
Valery Dmitriev via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 17 14:08:54 PDT 2023
https://github.com/valerydmit created https://github.com/llvm/llvm-project/pull/69384
Outline some often used common code to dedicated variables in order
to make code compact. Rename variables to more accurately reflect
their purpose. Apply const qualifier where appropriate.
Fix and add bit more explanation comment for the existing code.
>From c03dab157bf8b176519f5225c86632977f078cab Mon Sep 17 00:00:00 2001
From: Valery N Dmitriev <valery.n.dmitriev at intel.com>
Date: Tue, 17 Oct 2023 13:18:59 -0700
Subject: [PATCH] [SLP][NFC] Try to cleanup and better document some
isGatherShuffledEntry code.
Outline some often used common code to dedicated variables in order
to make code compact. Rename variables to more accuretely reflect
their purpose. Apply const qualifier where appropriate.
Fix and add some more explanation comment for the existing code.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 100 ++++++++----------
1 file changed, 45 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 32ddd82d9adbd82..d09bf3872f04f06 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9036,41 +9036,45 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef<Value *> VL,
"Expected only single user of the gather node.");
// TODO: currently checking only for Scalars in the tree entry, need to count
// reused elements too for better cost estimation.
- Instruction &UserInst =
- getLastInstructionInBundle(TE->UserTreeIndices.front().UserTE);
- BasicBlock *ParentBB = nullptr;
+ const EdgeInfo &TEUseEI = TE->UserTreeIndices.front();
+ const Instruction *TEInsertPt = &getLastInstructionInBundle(TEUseEI.UserTE);
+ const BasicBlock *TEInsertBlock = nullptr;
// Main node of PHI entries keeps the correct order of operands/incoming
// blocks.
- if (auto *PHI =
- dyn_cast<PHINode>(TE->UserTreeIndices.front().UserTE->getMainOp())) {
- ParentBB = PHI->getIncomingBlock(TE->UserTreeIndices.front().EdgeIdx);
+ if (auto *PHI = dyn_cast<PHINode>(TEUseEI.UserTE->getMainOp())) {
+ TEInsertBlock = PHI->getIncomingBlock(TEUseEI.EdgeIdx);
} else {
- ParentBB = UserInst.getParent();
+ TEInsertBlock = TEInsertPt->getParent();
}
- auto *NodeUI = DT->getNode(ParentBB);
+ auto *NodeUI = DT->getNode(TEInsertBlock);
assert(NodeUI && "Should only process reachable instructions");
SmallPtrSet<Value *, 4> GatheredScalars(VL.begin(), VL.end());
- auto CheckOrdering = [&](Instruction *LastEI) {
- // Check if the user node of the TE comes after user node of EntryPtr,
- // otherwise EntryPtr depends on TE.
- // Gather nodes usually are not scheduled and inserted before their first
- // user node. So, instead of checking dependency between the gather nodes
- // themselves, we check the dependency between their user nodes.
- // If one user node comes before the second one, we cannot use the second
- // gather node as the source vector for the first gather node, because in
- // the list of instructions it will be emitted later.
- auto *EntryParent = LastEI->getParent();
- auto *NodeEUI = DT->getNode(EntryParent);
+ auto CheckOrdering = [&](const Instruction *InsertPt) {
+ // Argument InsertPt is an instruction where vector code for some other
+ // tree entry (one that shares one or more scalars with TE) is going to be
+ // generated. This lambda returns true if insertion point of vector code
+ // for the TE dominates that point (otherwise dependency is the other way
+ // around). The other node is not limited to be of a gather kind. Gather
+ // nodes are not scheduled and their vector code is inserted before their
+ // first user. If user is PHI, that is supposed to be at the end of a
+ // predecessor block. Otherwise it is the last instruction among scalars of
+ // the user node. So, instead of checking dependency between instructions
+ // themselves, we check dependency between their insertion points for vector
+ // code (since each scalar instruction ends up as a lane of a vector
+ // instruction).
+ const BasicBlock *InsertBlock = InsertPt->getParent();
+ auto *NodeEUI = DT->getNode(InsertBlock);
if (!NodeEUI)
return false;
assert((NodeUI == NodeEUI) ==
(NodeUI->getDFSNumIn() == NodeEUI->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
// Check the order of the gather nodes users.
- if (UserInst.getParent() != EntryParent &&
+ if (TEInsertPt->getParent() != InsertBlock &&
(DT->dominates(NodeUI, NodeEUI) || !DT->dominates(NodeEUI, NodeUI)))
return false;
- if (UserInst.getParent() == EntryParent && UserInst.comesBefore(LastEI))
+ if (TEInsertPt->getParent() == InsertBlock &&
+ TEInsertPt->comesBefore(InsertPt))
return false;
return true;
};
@@ -9095,49 +9099,35 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef<Value *> VL,
[&](Value *V) { return GatheredScalars.contains(V); }) &&
"Must contain at least single gathered value.");
assert(TEPtr->UserTreeIndices.size() == 1 &&
- "Expected only single user of the gather node.");
- PHINode *EntryPHI =
- dyn_cast<PHINode>(TEPtr->UserTreeIndices.front().UserTE->getMainOp());
- Instruction *EntryUserInst =
- EntryPHI ? nullptr
- : &getLastInstructionInBundle(
- TEPtr->UserTreeIndices.front().UserTE);
- if (&UserInst == EntryUserInst) {
- assert(!EntryPHI && "Unexpected phi node entry.");
- // If 2 gathers are operands of the same entry, compare operands
- // indices, use the earlier one as the base.
- if (TE->UserTreeIndices.front().UserTE ==
- TEPtr->UserTreeIndices.front().UserTE &&
- TE->UserTreeIndices.front().EdgeIdx <
- TEPtr->UserTreeIndices.front().EdgeIdx)
+ "Expected only single user of a gather node.");
+ const EdgeInfo &UseEI = TEPtr->UserTreeIndices.front();
+
+ PHINode *UserPHI = dyn_cast<PHINode>(UseEI.UserTE->getMainOp());
+ const Instruction *InsertPt =
+ UserPHI ? UserPHI->getIncomingBlock(UseEI.EdgeIdx)->getTerminator()
+ : &getLastInstructionInBundle(UseEI.UserTE);
+ if (!UserPHI && TEInsertPt == InsertPt) {
+ // If 2 gathers are operands of the same non-PHI entry,
+ // compare operands indices, use the earlier one as the base.
+ if (TEUseEI.UserTE == UseEI.UserTE && TEUseEI.EdgeIdx < UseEI.EdgeIdx)
continue;
// If the user instruction is used for some reason in different
// vectorized nodes - make it depend on index.
- if (TE->UserTreeIndices.front().UserTE !=
- TEPtr->UserTreeIndices.front().UserTE &&
- TE->Idx < TEPtr->Idx)
+ if (TEUseEI.UserTE != UseEI.UserTE && TE->Idx < TEPtr->Idx)
continue;
}
- // Check if the user node of the TE comes after user node of EntryPtr,
- // otherwise EntryPtr depends on TE.
- auto *EntryI =
- EntryPHI
- ? EntryPHI
- ->getIncomingBlock(TEPtr->UserTreeIndices.front().EdgeIdx)
- ->getTerminator()
- : EntryUserInst;
- if ((ParentBB != EntryI->getParent() ||
- TE->UserTreeIndices.front().EdgeIdx <
- TEPtr->UserTreeIndices.front().EdgeIdx ||
- TE->UserTreeIndices.front().UserTE !=
- TEPtr->UserTreeIndices.front().UserTE) &&
- !CheckOrdering(EntryI))
+
+ // Check if the user node of the TE comes after user node of TEPtr,
+ // otherwise TEPtr depends on TE.
+ if ((TEInsertBlock != InsertPt->getParent() ||
+ TEUseEI.EdgeIdx < UseEI.EdgeIdx || TEUseEI.UserTE != UseEI.UserTE) &&
+ !CheckOrdering(InsertPt))
continue;
VToTEs.insert(TEPtr);
}
if (const TreeEntry *VTE = getTreeEntry(V)) {
- Instruction &EntryUserInst = getLastInstructionInBundle(VTE);
- if (&EntryUserInst == &UserInst || !CheckOrdering(&EntryUserInst))
+ Instruction &LastBundleInst = getLastInstructionInBundle(VTE);
+ if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst))
continue;
VToTEs.insert(VTE);
}
More information about the llvm-commits
mailing list