[llvm] ddb2188 - [SLP][NFC] Cleanup: Separate vectorization of Inserts and CmpInsts.
Vasileios Porpodas via llvm-commits
llvm-commits at lists.llvm.org
Mon May 15 10:12:59 PDT 2023
Author: Vasileios Porpodas
Date: 2023-05-15T10:12:34-07:00
New Revision: ddb2188afc86c7819e87ec538545b45803984342
URL: https://github.com/llvm/llvm-project/commit/ddb2188afc86c7819e87ec538545b45803984342
DIFF: https://github.com/llvm/llvm-project/commit/ddb2188afc86c7819e87ec538545b45803984342.diff
LOG: [SLP][NFC] Cleanup: Separate vectorization of Inserts and CmpInsts.
This deprecates `vectorizeSimpleInstructions()` and replaces it with separate
functions that vectorize CmpInsts and Inserts.
Differential Revision: https://reviews.llvm.org/D149993
Added:
Modified:
llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 007dbffd38647..87fdb8b26382a 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -144,14 +144,14 @@ struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
slpvectorizer::BoUpSLP &R);
/// Tries to vectorize \p CmpInts. \Returns true on success.
- bool vectorizeCmpInsts(ArrayRef<CmpInst *> CmpInsts, BasicBlock *BB,
+ template <typename ItT>
+ bool vectorizeCmpInsts(iterator_range<ItT> CmpInsts, BasicBlock *BB,
slpvectorizer::BoUpSLP &R);
- /// Tries to vectorize constructs started from CmpInst, InsertValueInst or
+ /// Tries to vectorize constructs started from InsertValueInst or
/// InsertElementInst instructions.
- bool vectorizeSimpleInstructions(InstSetVector &Instructions, BasicBlock *BB,
- slpvectorizer::BoUpSLP &R,
- bool AtTerminator);
+ bool vectorizeInserts(InstSetVector &Instructions, BasicBlock *BB,
+ slpvectorizer::BoUpSLP &R);
/// Scan the basic block and look for patterns that are likely to start
/// a vectorization chain.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 17326700a9d0f..eb63290c474be 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14399,11 +14399,12 @@ static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI,
return IsCompatibility;
}
-bool SLPVectorizerPass::vectorizeCmpInsts(ArrayRef<CmpInst *> CmpInsts,
+template <typename ItT>
+bool SLPVectorizerPass::vectorizeCmpInsts(iterator_range<ItT> CmpInsts,
BasicBlock *BB, BoUpSLP &R) {
bool Changed = false;
// Try to find reductions first.
- for (Instruction *I : CmpInsts) {
+ for (CmpInst *I : CmpInsts) {
if (R.isDeleted(I))
continue;
for (Value *Op : I->operands())
@@ -14411,7 +14412,7 @@ bool SLPVectorizerPass::vectorizeCmpInsts(ArrayRef<CmpInst *> CmpInsts,
Changed |= vectorizeRootInstruction(nullptr, RootOp, BB, R, TTI);
}
// Try to vectorize operands as vector bundles.
- for (Instruction *I : CmpInsts) {
+ for (CmpInst *I : CmpInsts) {
if (R.isDeleted(I))
continue;
Changed |= tryToVectorize(I, R);
@@ -14450,25 +14451,19 @@ bool SLPVectorizerPass::vectorizeCmpInsts(ArrayRef<CmpInst *> CmpInsts,
return Changed;
}
-bool SLPVectorizerPass::vectorizeSimpleInstructions(InstSetVector &Instructions,
- BasicBlock *BB, BoUpSLP &R,
- bool AtTerminator) {
+bool SLPVectorizerPass::vectorizeInserts(InstSetVector &Instructions,
+ BasicBlock *BB, BoUpSLP &R) {
assert(all_of(Instructions,
[](auto *I) {
- return isa<CmpInst, InsertElementInst, InsertValueInst>(I);
+ return isa<InsertElementInst, InsertValueInst>(I);
}) &&
- "This function only accepts Cmp and Insert instructions");
+ "This function only accepts Insert instructions");
bool OpsChanged = false;
- SmallVector<CmpInst *, 4> PostponedCmps;
SmallVector<WeakTrackingVH> PostponedInsts;
// pass1 - try to vectorize reductions only
for (auto *I : reverse(Instructions)) {
if (R.isDeleted(I))
continue;
- if (isa<CmpInst>(I)) {
- PostponedCmps.push_back(cast<CmpInst>(I));
- continue;
- }
OpsChanged |= vectorizeHorReduction(nullptr, I, BB, R, TTI, PostponedInsts);
}
// pass2 - try to match and vectorize a buildvector sequence.
@@ -14484,15 +14479,7 @@ bool SLPVectorizerPass::vectorizeSimpleInstructions(InstSetVector &Instructions,
// Now try to vectorize postponed instructions.
OpsChanged |= tryToVectorize(PostponedInsts, R);
- if (AtTerminator) {
- OpsChanged |= vectorizeCmpInsts(PostponedCmps, BB, R);
- Instructions.clear();
- } else {
- Instructions.clear();
- // Insert in reverse order since the PostponedCmps vector was filled in
- // reverse order.
- Instructions.insert(PostponedCmps.rbegin(), PostponedCmps.rend());
- }
+ Instructions.clear();
return OpsChanged;
}
@@ -14640,8 +14627,27 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
VisitedInstrs.clear();
- InstSetVector PostProcessInstructions;
+ InstSetVector PostProcessInserts;
+ SmallSetVector<CmpInst *, 8> PostProcessCmps;
SmallDenseSet<Instruction *, 4> KeyNodes;
+ // Vectorizes Inserts in `PostProcessInserts` and if `VecctorizeCmps` is true
+ // also vectorizes `PostProcessCmps`.
+ auto VectorizeInsertsAndCmps = [&](bool VectorizeCmps) {
+ bool Changed = vectorizeInserts(PostProcessInserts, BB, R);
+ if (VectorizeCmps) {
+ Changed |= vectorizeCmpInsts(reverse(PostProcessCmps), BB, R);
+ PostProcessCmps.clear();
+ }
+ PostProcessInserts.clear();
+ return Changed;
+ };
+ // Returns true if `I` is in `PostProcessInserts` or `PostProcessCmps`.
+ auto IsInPostProcessInstrs = [&](Instruction *I) {
+ if (auto *Cmp = dyn_cast<CmpInst>(I))
+ return PostProcessCmps.contains(Cmp);
+ return isa<InsertElementInst, InsertValueInst>(I) &&
+ PostProcessInserts.contains(I);
+ };
for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
// Skip instructions with scalable type. The num of elements is unknown at
// compile-time for scalable type.
@@ -14654,8 +14660,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
// We may go through BB multiple times so skip the one we have checked.
if (!VisitedInstrs.insert(&*it).second) {
if (it->use_empty() && KeyNodes.contains(&*it) &&
- vectorizeSimpleInstructions(PostProcessInstructions, BB, R,
- it->isTerminator())) {
+ VectorizeInsertsAndCmps(/*VectorizeCmps=*/it->isTerminator())) {
// We would like to start over since some instructions are deleted
// and the iterator may become invalid value.
Changed = true;
@@ -14695,7 +14700,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
// Postponed instructions should not be vectorized here, delay their
// vectorization.
if (auto *PI = dyn_cast<Instruction>(P->getIncomingValue(I));
- PI && !PostProcessInstructions.contains(PI))
+ PI && !IsInPostProcessInstrs(PI))
Changed |= vectorizeRootInstruction(nullptr, PI,
P->getIncomingBlock(I), R, TTI);
}
@@ -14727,7 +14732,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
// Postponed instructions should not be vectorized here, delay their
// vectorization.
if (auto *VI = dyn_cast<Instruction>(V);
- VI && !PostProcessInstructions.contains(VI))
+ VI && !IsInPostProcessInstrs(VI))
// Try to match and vectorize a horizontal reduction.
OpsChanged |= vectorizeRootInstruction(nullptr, VI, BB, R, TTI);
}
@@ -14735,8 +14740,8 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
// Start vectorization of post-process list of instructions from the
// top-tree instructions to try to vectorize as many instructions as
// possible.
- OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R,
- it->isTerminator());
+ OpsChanged |=
+ VectorizeInsertsAndCmps(/*VectorizeCmps=*/it->isTerminator());
if (OpsChanged) {
// We would like to start over since some instructions are deleted
// and the iterator may become invalid value.
@@ -14747,8 +14752,10 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
}
}
- if (isa<CmpInst, InsertElementInst, InsertValueInst>(it))
- PostProcessInstructions.insert(&*it);
+ if (isa<InsertElementInst, InsertValueInst>(it))
+ PostProcessInserts.insert(&*it);
+ else if (isa<CmpInst>(it))
+ PostProcessCmps.insert(cast<CmpInst>(&*it));
}
return Changed;
More information about the llvm-commits
mailing list