[llvm] r278907 - [LTO] Introduce an Output class to wrap the output stream creation (NFC)
Mehdi Amini via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 16 23:23:10 PDT 2016
Author: mehdi_amini
Date: Wed Aug 17 01:23:09 2016
New Revision: 278907
URL: http://llvm.org/viewvc/llvm-project?rev=278907&view=rev
Log:
[LTO] Introduce an Output class to wrap the output stream creation (NFC)
Summary:
While NFC for now, this will allow more flexibility on the client side
to hold state necessary to back up the stream.
Also when adding caching, this class will grow in complexity.
Note I blindly modified the gold-plugin as I can't compile it.
Reviewers: tejohnson
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D23542
Modified:
llvm/trunk/include/llvm/LTO/Config.h
llvm/trunk/include/llvm/LTO/LTO.h
llvm/trunk/include/llvm/LTO/LTOBackend.h
llvm/trunk/lib/LTO/LTO.cpp
llvm/trunk/lib/LTO/LTOBackend.cpp
llvm/trunk/tools/gold/gold-plugin.cpp
llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp
Modified: llvm/trunk/include/llvm/LTO/Config.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/Config.h?rev=278907&r1=278906&r2=278907&view=diff
==============================================================================
--- llvm/trunk/include/llvm/LTO/Config.h (original)
+++ llvm/trunk/include/llvm/LTO/Config.h Wed Aug 17 01:23:09 2016
@@ -30,6 +30,15 @@ class raw_pwrite_stream;
namespace lto {
+/// Abstract class representing a single Task output to be implemented by the
+/// client of the LTO API.
+class NativeObjectOutput {
+public:
+ // Return an allocated stream for the output, or null in case of failure.
+ virtual std::unique_ptr<raw_pwrite_stream> getStream() = 0;
+ virtual ~NativeObjectOutput() = default;
+};
+
/// LTO configuration. A linker can configure LTO by setting fields in this data
/// structure and passing it to the lto::LTO constructor.
struct Config {
@@ -186,13 +195,12 @@ struct Config {
bool UseInputModulePath = false);
};
-/// This type defines a stream callback. A stream callback is used to add a
-/// native object that is generated on the fly. The callee must set up and
-/// return a output stream to write the native object to.
+/// This type defines the callback to add a native object that is generated on
+/// the fly.
///
-/// Stream callbacks must be thread safe.
-typedef std::function<std::unique_ptr<raw_pwrite_stream>(unsigned Task)>
- AddStreamFn;
+/// Output callbacks must be thread safe.
+typedef std::function<std::unique_ptr<NativeObjectOutput>(unsigned Task)>
+ AddOutputFn;
/// A derived class of LLVMContext that initializes itself according to a given
/// Config object. The purpose of this class is to tie ownership of the
Modified: llvm/trunk/include/llvm/LTO/LTO.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/LTO.h?rev=278907&r1=278906&r2=278907&view=diff
==============================================================================
--- llvm/trunk/include/llvm/LTO/LTO.h (original)
+++ llvm/trunk/include/llvm/LTO/LTO.h Wed Aug 17 01:23:09 2016
@@ -236,7 +236,7 @@ public:
typedef std::function<std::unique_ptr<ThinBackendProc>(
Config &C, ModuleSummaryIndex &CombinedIndex,
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream)>
+ AddOutputFn AddOutput)>
ThinBackend;
/// This ThinBackend runs the individual backend jobs in-process.
@@ -269,7 +269,7 @@ ThinBackend createWriteIndexesThinBacken
/// and pass it and an array of symbol resolutions to the add() function.
/// - Call the getMaxTasks() function to get an upper bound on the number of
/// native object files that LTO may add to the link.
-/// - Call the run() function. This function will use the supplied AddStream
+/// - Call the run() function. This function will use the supplied AddOutput
/// function to add up to getMaxTasks() native object files to the link.
class LTO {
friend InputFile;
@@ -293,9 +293,9 @@ public:
/// full description of tasks see LTOBackend.h.
unsigned getMaxTasks() const;
- /// Runs the LTO pipeline. This function calls the supplied AddStream function
+ /// Runs the LTO pipeline. This function calls the supplied AddOutput function
/// to add native object files to the link.
- Error run(AddStreamFn AddStream);
+ Error run(AddOutputFn AddOutput);
private:
Config Conf;
@@ -371,8 +371,8 @@ private:
Error addThinLTO(std::unique_ptr<InputFile> Input,
ArrayRef<SymbolResolution> Res);
- Error runRegularLTO(AddStreamFn AddStream);
- Error runThinLTO(AddStreamFn AddStream);
+ Error runRegularLTO(AddOutputFn AddOutput);
+ Error runThinLTO(AddOutputFn AddOutput);
mutable bool CalledGetMaxTasks = false;
};
Modified: llvm/trunk/include/llvm/LTO/LTOBackend.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/LTOBackend.h?rev=278907&r1=278906&r2=278907&view=diff
==============================================================================
--- llvm/trunk/include/llvm/LTO/LTOBackend.h (original)
+++ llvm/trunk/include/llvm/LTO/LTOBackend.h Wed Aug 17 01:23:09 2016
@@ -34,12 +34,12 @@ class Target;
namespace lto {
/// Runs a regular LTO backend.
-Error backend(Config &C, AddStreamFn AddStream,
+Error backend(Config &C, AddOutputFn AddStream,
unsigned ParallelCodeGenParallelismLevel,
std::unique_ptr<Module> M);
/// Runs a ThinLTO backend.
-Error thinBackend(Config &C, unsigned Task, AddStreamFn AddStream, Module &M,
+Error thinBackend(Config &C, unsigned Task, AddOutputFn AddStream, Module &M,
ModuleSummaryIndex &CombinedIndex,
const FunctionImporter::ImportMapTy &ImportList,
const GVSummaryMapTy &DefinedGlobals,
Modified: llvm/trunk/lib/LTO/LTO.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/LTO.cpp?rev=278907&r1=278906&r2=278907&view=diff
==============================================================================
--- llvm/trunk/lib/LTO/LTO.cpp (original)
+++ llvm/trunk/lib/LTO/LTO.cpp Wed Aug 17 01:23:09 2016
@@ -350,19 +350,19 @@ unsigned LTO::getMaxTasks() const {
return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size();
}
-Error LTO::run(AddStreamFn AddStream) {
+Error LTO::run(AddOutputFn AddOutput) {
// Invoke regular LTO if there was a regular LTO module to start with,
// or if there are any hooks that the linker may have used to add
// its own resolved symbols to the combined module.
if (RegularLTO.HasModule || Conf.PreOptModuleHook ||
Conf.PostInternalizeModuleHook || Conf.PostOptModuleHook ||
Conf.PreCodeGenModuleHook)
- if (auto E = runRegularLTO(AddStream))
+ if (auto E = runRegularLTO(AddOutput))
return E;
- return runThinLTO(AddStream);
+ return runThinLTO(AddOutput);
}
-Error LTO::runRegularLTO(AddStreamFn AddStream) {
+Error LTO::runRegularLTO(AddOutputFn AddOutput) {
if (Conf.PreOptModuleHook &&
!Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
return Error();
@@ -388,7 +388,7 @@ Error LTO::runRegularLTO(AddStreamFn Add
!Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
return Error();
- return backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
+ return backend(Conf, AddOutput, RegularLTO.ParallelCodeGenParallelismLevel,
std::move(RegularLTO.CombinedModule));
}
@@ -397,14 +397,14 @@ class lto::ThinBackendProc {
protected:
Config &Conf;
ModuleSummaryIndex &CombinedIndex;
- AddStreamFn AddStream;
+ AddOutputFn AddOutput;
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries;
public:
ThinBackendProc(Config &Conf, ModuleSummaryIndex &CombinedIndex,
- AddStreamFn AddStream,
+ AddOutputFn AddOutput,
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries)
- : Conf(Conf), CombinedIndex(CombinedIndex), AddStream(AddStream),
+ : Conf(Conf), CombinedIndex(CombinedIndex), AddOutput(AddOutput),
ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries) {}
virtual ~ThinBackendProc() {}
@@ -424,13 +424,13 @@ public:
InProcessThinBackend(Config &Conf, ModuleSummaryIndex &CombinedIndex,
unsigned ThinLTOParallelismLevel,
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream)
- : ThinBackendProc(Conf, CombinedIndex, AddStream,
+ AddOutputFn AddOutput)
+ : ThinBackendProc(Conf, CombinedIndex, AddOutput,
ModuleToDefinedGVSummaries),
BackendThreadPool(ThinLTOParallelismLevel) {}
Error
- runThinLTOBackendThread(AddStreamFn AddStream, unsigned Task,
+ runThinLTOBackendThread(AddOutputFn AddOutput, unsigned Task,
MemoryBufferRef MBRef,
ModuleSummaryIndex &CombinedIndex,
const FunctionImporter::ImportMapTy &ImportList,
@@ -442,7 +442,7 @@ public:
parseBitcodeFile(MBRef, BackendContext);
assert(MOrErr && "Unable to load module in thread?");
- return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
+ return thinBackend(Conf, Task, AddOutput, **MOrErr, CombinedIndex,
ImportList, DefinedGlobals, ModuleMap);
}
@@ -456,7 +456,7 @@ public:
GVSummaryMapTy &DefinedGlobals,
MapVector<StringRef, MemoryBufferRef> &ModuleMap) {
Error E =
- runThinLTOBackendThread(AddStream, Task, MBRef, CombinedIndex,
+ runThinLTOBackendThread(AddOutput, Task, MBRef, CombinedIndex,
ImportList, DefinedGlobals, ModuleMap);
if (E) {
std::unique_lock<std::mutex> L(ErrMu);
@@ -483,10 +483,10 @@ public:
ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) {
return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream) {
+ AddOutputFn AddOutput) {
return llvm::make_unique<InProcessThinBackend>(
Conf, CombinedIndex, ParallelismLevel, ModuleToDefinedGVSummaries,
- AddStream);
+ AddOutput);
};
}
@@ -500,10 +500,10 @@ class WriteIndexesThinBackend : public T
public:
WriteIndexesThinBackend(Config &Conf, ModuleSummaryIndex &CombinedIndex,
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, std::string OldPrefix,
+ AddOutputFn AddOutput, std::string OldPrefix,
std::string NewPrefix, bool ShouldEmitImportsFiles,
std::string LinkedObjectsFileName)
- : ThinBackendProc(Conf, CombinedIndex, AddStream,
+ : ThinBackendProc(Conf, CombinedIndex, AddOutput,
ModuleToDefinedGVSummaries),
OldPrefix(OldPrefix), NewPrefix(NewPrefix),
ShouldEmitImportsFiles(ShouldEmitImportsFiles),
@@ -572,14 +572,14 @@ ThinBackend lto::createWriteIndexesThinB
std::string LinkedObjectsFile) {
return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream) {
+ AddOutputFn AddOutput) {
return llvm::make_unique<WriteIndexesThinBackend>(
- Conf, CombinedIndex, ModuleToDefinedGVSummaries, AddStream, OldPrefix,
+ Conf, CombinedIndex, ModuleToDefinedGVSummaries, AddOutput, OldPrefix,
NewPrefix, ShouldEmitImportsFiles, LinkedObjectsFile);
};
}
-Error LTO::runThinLTO(AddStreamFn AddStream) {
+Error LTO::runThinLTO(AddOutputFn AddOutput) {
if (ThinLTO.ModuleMap.empty())
return Error();
@@ -622,7 +622,7 @@ Error LTO::runThinLTO(AddStreamFn AddStr
[](StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes) {});
std::unique_ptr<ThinBackendProc> BackendProc = ThinLTO.Backend(
- Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, AddStream);
+ Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, AddOutput);
// Partition numbers for ThinLTO jobs start at 1 (see comments for
// GlobalResolution in LTO.h). Task numbers, however, start at
Modified: llvm/trunk/lib/LTO/LTOBackend.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/LTOBackend.cpp?rev=278907&r1=278906&r2=278907&view=diff
==============================================================================
--- llvm/trunk/lib/LTO/LTOBackend.cpp (original)
+++ llvm/trunk/lib/LTO/LTOBackend.cpp Wed Aug 17 01:23:09 2016
@@ -19,6 +19,7 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/LTO/LTO.h"
#include "llvm/MC/SubtargetFeature.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
@@ -144,12 +145,13 @@ bool opt(Config &C, TargetMachine *TM, u
return true;
}
-void codegen(Config &C, TargetMachine *TM, AddStreamFn AddStream, unsigned Task,
+void codegen(Config &C, TargetMachine *TM, AddOutputFn AddOutput, unsigned Task,
Module &M) {
if (C.PreCodeGenModuleHook && !C.PreCodeGenModuleHook(Task, M))
return;
- std::unique_ptr<raw_pwrite_stream> OS = AddStream(Task);
+ auto Output = AddOutput(Task);
+ std::unique_ptr<raw_pwrite_stream> OS = Output->getStream();
legacy::PassManager CodeGenPasses;
if (TM->addPassesToEmitFile(CodeGenPasses, *OS,
TargetMachine::CGFT_ObjectFile))
@@ -157,7 +159,7 @@ void codegen(Config &C, TargetMachine *T
CodeGenPasses.run(M);
}
-void splitCodeGen(Config &C, TargetMachine *TM, AddStreamFn AddStream,
+void splitCodeGen(Config &C, TargetMachine *TM, AddOutputFn AddOutput,
unsigned ParallelCodeGenParallelismLevel,
std::unique_ptr<Module> M) {
ThreadPool CodegenThreadPool(ParallelCodeGenParallelismLevel);
@@ -190,7 +192,7 @@ void splitCodeGen(Config &C, TargetMachi
std::unique_ptr<TargetMachine> TM =
createTargetMachine(C, MPartInCtx->getTargetTriple(), T);
- codegen(C, TM.get(), AddStream, ThreadId, *MPartInCtx);
+ codegen(C, TM.get(), AddOutput, ThreadId, *MPartInCtx);
},
// Pass BC using std::move to ensure that it get moved rather than
// copied into the thread's context.
@@ -214,7 +216,7 @@ Expected<const Target *> initAndLookupTa
}
-Error lto::backend(Config &C, AddStreamFn AddStream,
+Error lto::backend(Config &C, AddOutputFn AddOutput,
unsigned ParallelCodeGenParallelismLevel,
std::unique_ptr<Module> M) {
Expected<const Target *> TOrErr = initAndLookupTarget(C, *M);
@@ -228,14 +230,14 @@ Error lto::backend(Config &C, AddStreamF
return Error();
if (ParallelCodeGenParallelismLevel == 1)
- codegen(C, TM.get(), AddStream, 0, *M);
+ codegen(C, TM.get(), AddOutput, 0, *M);
else
- splitCodeGen(C, TM.get(), AddStream, ParallelCodeGenParallelismLevel,
+ splitCodeGen(C, TM.get(), AddOutput, ParallelCodeGenParallelismLevel,
std::move(M));
return Error();
}
-Error lto::thinBackend(Config &Conf, unsigned Task, AddStreamFn AddStream,
+Error lto::thinBackend(Config &Conf, unsigned Task, AddOutputFn AddOutput,
Module &Mod, ModuleSummaryIndex &CombinedIndex,
const FunctionImporter::ImportMapTy &ImportList,
const GVSummaryMapTy &DefinedGlobals,
@@ -281,6 +283,6 @@ Error lto::thinBackend(Config &Conf, uns
if (!opt(Conf, TM.get(), Task, Mod, /*IsThinLto=*/true))
return Error();
- codegen(Conf, TM.get(), AddStream, Task, Mod);
+ codegen(Conf, TM.get(), AddOutput, Task, Mod);
return Error();
}
Modified: llvm/trunk/tools/gold/gold-plugin.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/gold/gold-plugin.cpp?rev=278907&r1=278906&r2=278907&view=diff
==============================================================================
--- llvm/trunk/tools/gold/gold-plugin.cpp (original)
+++ llvm/trunk/tools/gold/gold-plugin.cpp Wed Aug 17 01:23:09 2016
@@ -647,16 +647,14 @@ static void recordFile(std::string Filen
Cleanup.push_back(Filename.c_str());
}
-/// Open a file and return the new file descriptor given a base input
-/// file name, a flag indicating whether a temp file should be generated,
-/// and an optional task id. The new filename generated is
-/// returned in \p NewFilename.
-static int openOutputFile(SmallString<128> InFilename, bool TempOutFile,
- SmallString<128> &NewFilename, int TaskID = -1) {
- int FD;
+/// Return the desired output filename given a base input name, a flag
+/// indicating whether a temp file should be generated, and an optional task id.
+/// The new filename generated is returned in \p NewFilename.
+static void getOutputFileName(SmallString<128> InFilename, bool TempOutFile,
+ SmallString<128> &NewFilename, int TaskID = -1) {
if (TempOutFile) {
std::error_code EC =
- sys::fs::createTemporaryFile("lto-llvm", "o", FD, NewFilename);
+ sys::fs::createTemporaryFile("lto-llvm", "o", NewFilename);
if (EC)
message(LDPL_FATAL, "Could not create temporary file: %s",
EC.message().c_str());
@@ -664,12 +662,7 @@ static int openOutputFile(SmallString<12
NewFilename = InFilename;
if (TaskID >= 0)
NewFilename += utostr(TaskID);
- std::error_code EC =
- sys::fs::openFileForWrite(NewFilename, FD, sys::fs::F_None);
- if (EC)
- message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str());
}
- return FD;
}
/// Add all required common symbols to M, which is expected to be the first
@@ -723,6 +716,24 @@ static void getThinLTOOldAndNewPrefix(st
NewPrefix = Split.second.str();
}
+namespace {
+// Define the LTOOutput handling
+class LTOOutput : public lto::NativeObjectOutput {
+ StringRef Path;
+
+public:
+ LTOOutput(StringRef Path) : Path(Path) {}
+ // Open the filename \p Path and allocate a stream.
+ std::unique_ptr<raw_pwrite_stream> getStream() override {
+ int FD;
+ std::error_code EC = sys::fs::openFileForWrite(Path, FD, sys::fs::F_None);
+ if (EC)
+ message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str());
+ return llvm::make_unique<llvm::raw_fd_ostream>(FD, true);
+ }
+};
+}
+
static std::unique_ptr<LTO> createLTO() {
Config Conf;
ThinBackend Backend;
@@ -814,7 +825,7 @@ static ld_plugin_status allSymbolsReadHo
}
SmallString<128> Filename;
- // Note that openOutputFile will append a unique ID for each task
+ // Note that getOutputFileName will append a unique ID for each task
if (!options::obj_path.empty())
Filename = options::obj_path;
else if (options::TheOutputType == options::OT_SAVE_TEMPS)
@@ -825,15 +836,15 @@ static ld_plugin_status allSymbolsReadHo
std::vector<uintptr_t> IsTemporary(MaxTasks);
std::vector<SmallString<128>> Filenames(MaxTasks);
- auto AddStream = [&](size_t Task) {
- int FD = openOutputFile(Filename, /*TempOutFile=*/!SaveTemps,
- Filenames[Task], MaxTasks > 1 ? Task : -1);
+ auto AddOutput = [&](size_t Task) {
+ auto &OutputName = Filenames[Task];
+ getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps, OutputName,
+ MaxTasks > 1 ? Task : -1);
IsTemporary[Task] = !SaveTemps;
-
- return llvm::make_unique<llvm::raw_fd_ostream>(FD, true);
+ return llvm::make_unique<LTOOutput>(OutputName);
};
- check(Lto->run(AddStream));
+ check(Lto->run(AddOutput));
if (options::TheOutputType == options::OT_DISABLE ||
options::TheOutputType == options::OT_BC_ONLY)
Modified: llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp?rev=278907&r1=278906&r2=278907&view=diff
==============================================================================
--- llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp (original)
+++ llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp Wed Aug 17 01:23:09 2016
@@ -74,6 +74,22 @@ template <typename T> static T check(Err
return T();
}
+namespace {
+// Define the LTOOutput handling
+class LTOOutput : public lto::NativeObjectOutput {
+ StringRef Path;
+
+public:
+ LTOOutput(StringRef Path) : Path(Path) {}
+ std::unique_ptr<raw_pwrite_stream> getStream() override {
+ std::error_code EC;
+ auto S = llvm::make_unique<raw_fd_ostream>(Path, EC, sys::fs::F_None);
+ check(EC, Path);
+ return std::move(S);
+ }
+};
+}
+
int main(int argc, char **argv) {
InitializeAllTargets();
InitializeAllTargetMCs();
@@ -156,13 +172,10 @@ int main(int argc, char **argv) {
if (HasErrors)
return 1;
- auto AddStream = [&](size_t Task) {
+ auto AddOutput = [&](size_t Task) {
std::string Path = OutputFilename + "." + utostr(Task);
- std::error_code EC;
- auto S = llvm::make_unique<raw_fd_ostream>(Path, EC, sys::fs::F_None);
- check(EC, Path);
- return S;
+ return llvm::make_unique<LTOOutput>(Path);
};
- check(Lto.run(AddStream), "LTO::run failed");
+ check(Lto.run(AddOutput), "LTO::run failed");
}
More information about the llvm-commits
mailing list