[llvm] [SandboxVec] Move seed collection into its own separate pass (PR #127132)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 14 08:59:58 PST 2025
https://github.com/vporpo updated https://github.com/llvm/llvm-project/pull/127132
>From 74618beafda537dc0def42f90934b0509720f61b Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vporpodas at google.com>
Date: Wed, 29 Jan 2025 09:53:39 -0800
Subject: [PATCH 1/2] [SandboxVec] Move seed collection into its own separate
pass
This patch moves the seed collection logic from the BottomUpVec pass into
a new Sandbox IR Function pass. The new "seed-collection" pass collects the
seeds, builds a region and runs the region pass pipeline.
---
llvm/include/llvm/SandboxIR/Region.h | 4 +
.../SandboxVectorizer/Passes/BottomUpVec.h | 14 +--
.../SandboxVectorizer/Passes/SeedCollection.h | 36 +++++++
llvm/lib/SandboxIR/Region.cpp | 25 +++--
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 1 +
.../SandboxVectorizer/Passes/BottomUpVec.cpp | 97 +++---------------
.../SandboxVectorizer/Passes/PassRegistry.def | 3 +-
.../Passes/SeedCollection.cpp | 98 +++++++++++++++++++
.../SandboxVectorizer/SandboxVectorizer.cpp | 6 +-
.../SandboxVectorizerPassBuilder.cpp | 1 +
.../SandboxVectorizer/bottomup_basic.ll | 2 +-
.../SandboxVectorizer/bottomup_seed_slice.ll | 2 +-
.../bottomup_seed_slice_pow2.ll | 4 +-
.../Transforms/SandboxVectorizer/cross_bbs.ll | 2 +-
.../default_pass_pipeline.ll | 3 +-
.../test/Transforms/SandboxVectorizer/pack.ll | 2 +-
.../SandboxVectorizer/repeated_instrs.ll | 2 +-
.../Transforms/SandboxVectorizer/scheduler.ll | 2 +-
.../SandboxVectorizer/special_opcodes.ll | 2 +-
.../SandboxVectorizer/user_pass_pipeline.ll | 8 +-
20 files changed, 194 insertions(+), 120 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h
create mode 100644 llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
diff --git a/llvm/include/llvm/SandboxIR/Region.h b/llvm/include/llvm/SandboxIR/Region.h
index 66eda34ea2edb..ecc6c2049ef8d 100644
--- a/llvm/include/llvm/SandboxIR/Region.h
+++ b/llvm/include/llvm/SandboxIR/Region.h
@@ -120,6 +120,10 @@ class Region {
/// Set \p I as the \p Idx'th element in the auxiliary vector.
/// NOTE: This is for internal use, it does not set the metadata.
void setAux(unsigned Idx, Instruction *I);
+ /// Helper for dropping Aux metadata for \p I.
+ void dropAuxMetadata(Instruction *I);
+ /// Remove instruction \p I from Aux and drop metadata.
+ void removeFromAux(Instruction *I);
public:
Region(Context &Ctx, TargetTransformInfo &TTI);
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index 147a86de4e34e..4712ea7eae29e 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -16,14 +16,13 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/SandboxIR/Constant.h"
#include "llvm/SandboxIR/Pass.h"
-#include "llvm/SandboxIR/PassManager.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrMaps.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
namespace llvm::sandboxir {
-class BottomUpVec final : public FunctionPass {
+class BottomUpVec final : public RegionPass {
bool Change = false;
std::unique_ptr<LegalityAnalysis> Legality;
/// The original instructions that are potentially dead after vectorization.
@@ -55,16 +54,9 @@ class BottomUpVec final : public FunctionPass {
/// Entry point for vectorization starting from \p Seeds.
bool tryVectorize(ArrayRef<Value *> Seeds);
- /// The PM containing the pipeline of region passes.
- RegionPassManager RPM;
-
public:
- BottomUpVec(StringRef Pipeline);
- bool runOnFunction(Function &F, const Analyses &A) final;
- void printPipeline(raw_ostream &OS) const final {
- OS << getName() << "\n";
- RPM.printPipeline(OS);
- }
+ BottomUpVec() : RegionPass("bottom-up-vec") {}
+ bool runOnRegion(Region &Rgn, const Analyses &A) final;
};
} // namespace llvm::sandboxir
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h
new file mode 100644
index 0000000000000..4469c6f98e47d
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h
@@ -0,0 +1,36 @@
+//===- SeedCollection.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The seed-collection pass of the bottom-up vectorizer.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDCOLLECTION_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDCOLLECTION_H
+
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/PassManager.h"
+
+namespace llvm::sandboxir {
+
+class SeedCollection final : public FunctionPass {
+
+ /// The PM containing the pipeline of region passes.
+ RegionPassManager RPM;
+
+public:
+ SeedCollection(StringRef Pipeline);
+ bool runOnFunction(Function &F, const Analyses &A) final;
+ void printPipeline(raw_ostream &OS) const final {
+ OS << getName() << "\n";
+ RPM.printPipeline(OS);
+ }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_SEEDCOLLECTION_H
diff --git a/llvm/lib/SandboxIR/Region.cpp b/llvm/lib/SandboxIR/Region.cpp
index ede738c35f032..086993e6dc872 100644
--- a/llvm/lib/SandboxIR/Region.cpp
+++ b/llvm/lib/SandboxIR/Region.cpp
@@ -40,8 +40,10 @@ Region::Region(Context &Ctx, TargetTransformInfo &TTI)
CreateInstCB = Ctx.registerCreateInstrCallback(
[this](Instruction *NewInst) { add(NewInst); });
- EraseInstCB = Ctx.registerEraseInstrCallback(
- [this](Instruction *ErasedInst) { remove(ErasedInst); });
+ EraseInstCB = Ctx.registerEraseInstrCallback([this](Instruction *ErasedInst) {
+ remove(ErasedInst);
+ removeFromAux(ErasedInst);
+ });
}
Region::~Region() {
@@ -84,11 +86,22 @@ void Region::setAux(unsigned Idx, Instruction *I) {
Aux[Idx] = I;
}
+void Region::dropAuxMetadata(Instruction *I) {
+ auto *LLVMI = cast<llvm::Instruction>(I->Val);
+ LLVMI->setMetadata(AuxMDKind, nullptr);
+}
+
+void Region::removeFromAux(Instruction *I) {
+ auto It = find(Aux, I);
+ if (It == Aux.end())
+ return;
+ dropAuxMetadata(I);
+ Aux.erase(It);
+}
+
void Region::clearAux() {
- for (unsigned Idx : seq<unsigned>(0, Aux.size())) {
- auto *LLVMI = cast<llvm::Instruction>(Aux[Idx]->Val);
- LLVMI->setMetadata(AuxMDKind, nullptr);
- }
+ for (unsigned Idx : seq<unsigned>(0, Aux.size()))
+ dropAuxMetadata(Aux[Idx]);
Aux.clear();
}
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 872e055294d55..676d23e1ebdf0 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMVectorize
SandboxVectorizer/Legality.cpp
SandboxVectorizer/Passes/BottomUpVec.cpp
SandboxVectorizer/Passes/RegionsFromMetadata.cpp
+ SandboxVectorizer/Passes/SeedCollection.cpp
SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
SandboxVectorizer/SandboxVectorizer.cpp
SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index 0ccef5aecd28b..d57732090dcd6 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -14,20 +14,10 @@
#include "llvm/SandboxIR/Module.h"
#include "llvm/SandboxIR/Region.h"
#include "llvm/SandboxIR/Utils.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
namespace llvm {
-static cl::opt<unsigned>
- OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(0), cl::Hidden,
- cl::desc("Override the vector register size in bits, "
- "which is otherwise found by querying TTI."));
-static cl::opt<bool>
- AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden,
- cl::desc("Allow non-power-of-2 vectorization."));
-
#ifndef NDEBUG
static cl::opt<bool>
AlwaysVerify("sbvec-always-verify", cl::init(false), cl::Hidden,
@@ -37,10 +27,6 @@ static cl::opt<bool>
namespace sandboxir {
-BottomUpVec::BottomUpVec(StringRef Pipeline)
- : FunctionPass("bottom-up-vec"),
- RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {}
-
static SmallVector<Value *, 4> getOperand(ArrayRef<Value *> Bndl,
unsigned OpIdx) {
SmallVector<Value *, 4> Operands;
@@ -413,6 +399,7 @@ Value *BottomUpVec::vectorizeRec(ArrayRef<Value *> Bndl,
}
bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl) {
+ Change = false;
DeadInstrCandidates.clear();
Legality->clear();
vectorizeRec(Bndl, {}, /*Depth=*/0);
@@ -420,83 +407,21 @@ bool BottomUpVec::tryVectorize(ArrayRef<Value *> Bndl) {
return Change;
}
-bool BottomUpVec::runOnFunction(Function &F, const Analyses &A) {
+bool BottomUpVec::runOnRegion(Region &Rgn, const Analyses &A) {
+ const auto &SeedSlice = Rgn.getAux();
+ assert(SeedSlice.size() >= 2 && "Bad slice!");
+ Function &F = *SeedSlice[0]->getParent()->getParent();
IMaps = std::make_unique<InstrMaps>(F.getContext());
Legality = std::make_unique<LegalityAnalysis>(
A.getAA(), A.getScalarEvolution(), F.getParent()->getDataLayout(),
F.getContext(), *IMaps);
- Change = false;
- const auto &DL = F.getParent()->getDataLayout();
- unsigned VecRegBits =
- OverrideVecRegBits != 0
- ? OverrideVecRegBits
- : A.getTTI()
- .getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
- .getFixedValue();
-
- // TODO: Start from innermost BBs first
- for (auto &BB : F) {
- SeedCollector SC(&BB, A.getScalarEvolution());
- for (SeedBundle &Seeds : SC.getStoreSeeds()) {
- unsigned ElmBits =
- Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType(
- Seeds[Seeds.getFirstUnusedElementIdx()])),
- DL);
-
- auto DivideBy2 = [](unsigned Num) {
- auto Floor = VecUtils::getFloorPowerOf2(Num);
- if (Floor == Num)
- return Floor / 2;
- return Floor;
- };
- // Try to create the largest vector supported by the target. If it fails
- // reduce the vector size by half.
- for (unsigned SliceElms = std::min(VecRegBits / ElmBits,
- Seeds.getNumUnusedBits() / ElmBits);
- SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) {
- if (Seeds.allUsed())
- break;
- // Keep trying offsets after FirstUnusedElementIdx, until we vectorize
- // the slice. This could be quite expensive, so we enforce a limit.
- for (unsigned Offset = Seeds.getFirstUnusedElementIdx(),
- OE = Seeds.size();
- Offset + 1 < OE; Offset += 1) {
- // Seeds are getting used as we vectorize, so skip them.
- if (Seeds.isUsed(Offset))
- continue;
- if (Seeds.allUsed())
- break;
- auto SeedSlice =
- Seeds.getSlice(Offset, SliceElms * ElmBits, !AllowNonPow2);
- if (SeedSlice.empty())
- continue;
-
- assert(SeedSlice.size() >= 2 && "Should have been rejected!");
-
- // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
- SmallVector<Value *> SeedSliceVals(SeedSlice.begin(),
- SeedSlice.end());
- // Create an empty region. Instructions get added to the region
- // automatically by the callbacks.
- auto &Ctx = F.getContext();
- Region Rgn(Ctx, A.getTTI());
- // Save the state of the IR before we make any changes. The
- // transaction gets accepted/reverted by the tr-accept-or-revert pass.
- Ctx.save();
- // Try to vectorize starting from the seed slice. The returned value
- // is true if we found vectorizable code and generated some vector
- // code for it. It does not mean that the code is profitable.
- bool VecSuccess = tryVectorize(SeedSliceVals);
- if (VecSuccess)
- // WARNING: All passes should return false, except those that
- // accept/revert the state.
- Change |= RPM.runOnRegion(Rgn, A);
- }
- }
- }
- }
- return Change;
+ // TODO: Refactor to remove the unnecessary copy to SeedSliceVals.
+ SmallVector<Value *> SeedSliceVals(SeedSlice.begin(), SeedSlice.end());
+ // Try to vectorize starting from the seed slice. The returned value
+ // is true if we found vectorizable code and generated some vector
+ // code for it. It does not mean that the code is profitable.
+ return tryVectorize(SeedSliceVals);
}
} // namespace sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index f3aa12729860f..722c6f5db4192 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -21,6 +21,7 @@ REGION_PASS("null", ::llvm::sandboxir::NullPass)
REGION_PASS("print-instruction-count", ::llvm::sandboxir::PrintInstructionCount)
REGION_PASS("tr-accept", ::llvm::sandboxir::TransactionAlwaysAccept)
REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
+REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
#undef REGION_PASS
@@ -28,7 +29,7 @@ REGION_PASS("tr-accept-or-revert", ::llvm::sandboxir::TransactionAcceptOrRevert)
#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS_NAME)
#endif
-FUNCTION_PASS_WITH_PARAMS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
+FUNCTION_PASS_WITH_PARAMS("seed-collection", ::llvm::sandboxir::SeedCollection)
FUNCTION_PASS_WITH_PARAMS("regions-from-metadata", ::llvm::sandboxir::RegionsFromMetadata)
#undef FUNCTION_PASS_WITH_PARAMS
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
new file mode 100644
index 0000000000000..0001c9bb7c7e4
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
@@ -0,0 +1,98 @@
+//===- SeedCollection.cpp - Seed collection pass --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/SandboxIR/Module.h"
+#include "llvm/SandboxIR/Region.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h"
+
+namespace llvm {
+
+static cl::opt<unsigned>
+ OverrideVecRegBits("sbvec-vec-reg-bits", cl::init(0), cl::Hidden,
+ cl::desc("Override the vector register size in bits, "
+ "which is otherwise found by querying TTI."));
+static cl::opt<bool>
+ AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden,
+ cl::desc("Allow non-power-of-2 vectorization."));
+
+namespace sandboxir {
+SeedCollection::SeedCollection(StringRef Pipeline)
+ : FunctionPass("seed-collection"),
+ RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {}
+
+bool SeedCollection::runOnFunction(Function &F, const Analyses &A) {
+ bool Change = false;
+ const auto &DL = F.getParent()->getDataLayout();
+ unsigned VecRegBits =
+ OverrideVecRegBits != 0
+ ? OverrideVecRegBits
+ : A.getTTI()
+ .getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+ .getFixedValue();
+
+ // TODO: Start from innermost BBs first
+ for (auto &BB : F) {
+ SeedCollector SC(&BB, A.getScalarEvolution());
+ for (SeedBundle &Seeds : SC.getStoreSeeds()) {
+ unsigned ElmBits =
+ Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType(
+ Seeds[Seeds.getFirstUnusedElementIdx()])),
+ DL);
+
+ auto DivideBy2 = [](unsigned Num) {
+ auto Floor = VecUtils::getFloorPowerOf2(Num);
+ if (Floor == Num)
+ return Floor / 2;
+ return Floor;
+ };
+ // Try to create the largest vector supported by the target. If it fails
+ // reduce the vector size by half.
+ for (unsigned SliceElms = std::min(VecRegBits / ElmBits,
+ Seeds.getNumUnusedBits() / ElmBits);
+ SliceElms >= 2u; SliceElms = DivideBy2(SliceElms)) {
+ if (Seeds.allUsed())
+ break;
+ // Keep trying offsets after FirstUnusedElementIdx, until we vectorize
+ // the slice. This could be quite expensive, so we enforce a limit.
+ for (unsigned Offset = Seeds.getFirstUnusedElementIdx(),
+ OE = Seeds.size();
+ Offset + 1 < OE; Offset += 1) {
+ // Seeds are getting used as we vectorize, so skip them.
+ if (Seeds.isUsed(Offset))
+ continue;
+ if (Seeds.allUsed())
+ break;
+
+ auto SeedSlice =
+ Seeds.getSlice(Offset, SliceElms * ElmBits, !AllowNonPow2);
+ if (SeedSlice.empty())
+ continue;
+
+ assert(SeedSlice.size() >= 2 && "Should have been rejected!");
+
+ // Create a region containing the seed slice.
+ auto &Ctx = F.getContext();
+ Region Rgn(Ctx, A.getTTI());
+ // TODO: Replace save() with a save pass in the pass pipeline.
+ Ctx.save();
+ Rgn.setAux(SeedSlice);
+ // Run the region pass pipeline.
+ Change |= RPM.runOnRegion(Rgn, A);
+ Rgn.clearAux();
+ }
+ }
+ }
+ }
+ return Change;
+}
+} // namespace sandboxir
+} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
index b233d35212f94..4f17aa213bbb0 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
@@ -32,9 +32,11 @@ static cl::opt<std::string> UserDefinedPassPipeline(
SandboxVectorizerPass::SandboxVectorizerPass() : FPM("fpm") {
if (UserDefinedPassPipeline == DefaultPipelineMagicStr) {
// TODO: Add passes to the default pipeline. It currently contains:
- // - the bottom-up-vectorizer pass
+ // - Seed collection, which creates seed regions and runs the pipeline
+ // - Bottom-up Vectorizer pass that starts from a seed
+ // - Accept or revert IR state pass
FPM.setPassPipeline(
- "bottom-up-vec<tr-accept-or-revert>",
+ "seed-collection<bottom-up-vec,tr-accept-or-revert>",
sandboxir::SandboxVectorizerPassBuilder::createFunctionPass);
} else {
// Create the user-defined pipeline.
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index 0c1ab55e91a5c..e552f0570dd9d 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -4,6 +4,7 @@
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAlwaysAccept.h"
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
index 6b18d4069e0ae..ad23126b681fd 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_basic.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @store_load(ptr %ptr) {
; CHECK-LABEL: define void @store_load(
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
index 202b5a6fbd6c9..f4fcc5742f8a7 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
declare void @foo()
diff --git a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
index 1b189831569f5..4218ca830dccf 100644
--- a/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/bottomup_seed_slice_pow2.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s --check-prefix=POW2
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s --check-prefix=NON-POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=false -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=POW2
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2=true -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s --check-prefix=NON-POW2
define void @pow2(ptr %ptr, float %val) {
; POW2-LABEL: define void @pow2(
diff --git a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
index ff1604173c317..75d7cc8d9cefe 100644
--- a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @cross_bbs(ptr %ptr) {
; CHECK-LABEL: define void @cross_bbs(
diff --git a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
index 10de4338caf23..86000da42c799 100644
--- a/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/default_pass_pipeline.ll
@@ -5,8 +5,9 @@
; This checks the default pass pipeline for the sandbox vectorizer.
define void @pipeline() {
; CHECK: fpm
-; CHECK: bottom-up-vec
+; CHECK: seed-collection
; CHECK: rpm
+; CHECK: bottom-up-vec
; CHECK: tr-accept-or-revert
; CHECK-EMPTY:
ret void
diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll
index da41036e3a58b..c1e22d3dc2e73 100644
--- a/llvm/test/Transforms/SandboxVectorizer/pack.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define void @pack_constants(ptr %ptr) {
; CHECK-LABEL: define void @pack_constants(
diff --git a/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll b/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll
index add762ac2d894..3e4ef6787a563 100644
--- a/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/repeated_instrs.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
define i32 @repeated_splat(ptr %ptr, i32 %v) #0 {
; CHECK-LABEL: define i32 @repeated_splat(
diff --git a/llvm/test/Transforms/SandboxVectorizer/scheduler.ll b/llvm/test/Transforms/SandboxVectorizer/scheduler.ll
index acbec80db6b06..fd4847da920d1 100644
--- a/llvm/test/Transforms/SandboxVectorizer/scheduler.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/scheduler.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
; This used to crash because the newly added pack instructions would not update
; the DAG and scheduler, leading to def-after-use.
diff --git a/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll b/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll
index e8fe8b4fa88e3..1aca7cf2a8bd8 100644
--- a/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/special_opcodes.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="bottom-up-vec<tr-accept>" %s -S | FileCheck %s
+; RUN: opt -passes=sandbox-vectorizer -sbvec-vec-reg-bits=1024 -sbvec-allow-non-pow2 -sbvec-passes="seed-collection<bottom-up-vec,tr-accept>" %s -S | FileCheck %s
; This file includes tests for opcodes that need special checks.
diff --git a/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll b/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll
index b11b55ed96019..2c57a8e7347d2 100644
--- a/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll
+++ b/llvm/test/Transforms/SandboxVectorizer/user_pass_pipeline.ll
@@ -1,9 +1,9 @@
; RUN: opt -passes=sandbox-vectorizer -sbvec-print-pass-pipeline \
-; RUN: -disable-output -sbvec-passes="bottom-up-vec<null,null>" %s \
+; RUN: -disable-output -sbvec-passes="seed-collection<null,null>" %s \
; RUN: | FileCheck %s
;
; RUN: opt -passes=sandbox-vectorizer -sbvec-print-pass-pipeline \
-; RUN: -disable-output -sbvec-passes="bottom-up-vec<>,regions-from-metadata<>" %s \
+; RUN: -disable-output -sbvec-passes="seed-collection<>,regions-from-metadata<>" %s \
; RUN: | FileCheck --check-prefix CHECK-MULTIPLE-FUNCTION-PASSES %s
; !!!WARNING!!! This won't get updated by update_test_checks.py !
@@ -14,14 +14,14 @@ define void @pipeline() {
}
; CHECK: fpm
-; CHECK: bottom-up-vec
+; CHECK: seed-collection
; CHECK: rpm
; CHECK: null
; CHECK: null
; CHECK-EMPTY:
; CHECK-MULTIPLE-FUNCTION-PASSES: fpm
-; CHECK-MULTIPLE-FUNCTION-PASSES: bottom-up-vec
+; CHECK-MULTIPLE-FUNCTION-PASSES: seed-collection
; CHECK-MULTIPLE-FUNCTION-PASSES: rpm
; CHECK-MULTIPLE-FUNCTION-PASSES: regions-from-metadata
; CHECK-MULTIPLE-FUNCTION-PASSES: rpm
>From 452ab21b50010835afca1beaf6482140cee09957 Mon Sep 17 00:00:00 2001
From: Vasileios Porpodas <vporpodas at google.com>
Date: Fri, 14 Feb 2025 08:59:20 -0800
Subject: [PATCH 2/2] fixup! [SandboxVec] Move seed collection into its own
separate pass
Adds test that checks that the region Aux vector and the corresponding metadata get updated when an instruction gets deleted
---
llvm/unittests/SandboxIR/RegionTest.cpp | 43 +++++++++++++++++++++++++
1 file changed, 43 insertions(+)
diff --git a/llvm/unittests/SandboxIR/RegionTest.cpp b/llvm/unittests/SandboxIR/RegionTest.cpp
index 201378980fe3b..8263f310632e1 100644
--- a/llvm/unittests/SandboxIR/RegionTest.cpp
+++ b/llvm/unittests/SandboxIR/RegionTest.cpp
@@ -388,6 +388,49 @@ define void @foo(i8 %v) {
#endif // NDEBUG
}
+// Check that Aux automatically drops instructions that get deleted.
+TEST_F(RegionTest, AuxDeleteInstr) {
+ parseIR(C, R"IR(
+define void @foo(i8 %v) {
+ %Add0 = add i8 %v, 0, !sandboxvec !0, !sandboxaux !1
+ %Add1 = add i8 %v, 1, !sandboxvec !0, !sandboxaux !2
+ %Add2 = add i8 %v, 2, !sandboxvec !0, !sandboxaux !3
+ %Add3 = add i8 %v, 2, !sandboxvec !0, !sandboxaux !4
+ ret void
+}
+
+!0 = distinct !{!"sandboxregion"}
+!1 = !{i32 0}
+!2 = !{i32 1}
+!3 = !{i32 2}
+!4 = !{i32 3}
+)IR");
+ llvm::Function *LLVMF = &*M->getFunction("foo");
+ sandboxir::Context Ctx(C);
+ auto *F = Ctx.createFunction(LLVMF);
+ auto *BB = &*F->begin();
+ auto It = BB->begin();
+ auto *Add0 = &*It++;
+ auto *Add1 = &*It++;
+ auto *Add2 = &*It++;
+ auto *Add3 = &*It++;
+ SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
+ sandboxir::Region::createRegionsFromMD(*F, *TTI);
+ auto &R = *Regions[0];
+ EXPECT_THAT(R.getAux(), testing::ElementsAre(Add0, Add1, Add2, Add3));
+ // Now delete Add1 and check that Aux contains nullptr instead of Add1.
+ Add2->eraseFromParent();
+ EXPECT_THAT(R.getAux(), testing::ElementsAre(Add0, Add1, Add3));
+ {
+ // Check that metadata have also been updated.
+ // But first drop Add3 to create a legal Aux vector with no gaps.
+ Add3->eraseFromParent();
+ SmallVector<std::unique_ptr<sandboxir::Region>> Regions =
+ sandboxir::Region::createRegionsFromMD(*F, *TTI);
+ EXPECT_THAT(Regions[0]->getAux(), testing::ElementsAre(Add0, Add1));
+ }
+}
+
TEST_F(RegionTest, AuxRoundTrip) {
parseIR(C, R"IR(
define i8 @foo(i8 %v0, i8 %v1) {
More information about the llvm-commits
mailing list