[llvm] r188188 - Misc enhancements to LTO:
Shuxin Yang
shuxin.llvm at gmail.com
Mon Aug 12 11:29:43 PDT 2013
Author: shuxin_yang
Date: Mon Aug 12 13:29:43 2013
New Revision: 188188
URL: http://llvm.org/viewvc/llvm-project?rev=188188&view=rev
Log:
Misc enhancements to LTO:
1. Add some helper classes for partitions. They are designed in a
way such that the top-level LTO driver will not see much difference
with or without partitioning.
2. Introduce work-dir. Now all intermediate files generated during
LTO phases will be saved under work-dir. User can specify the workdir
via -lto-workdir=/path/to/dir. By default the work-dir will be
erased before linker exit. To keep the workdir, do -lto-keep, or -lto-keep=1.
TODO: Erase the workdir, if the linker exit prematurely.
We are currently not able to remove directory on signal. The support
routines simply ignore directory.
3. Add one new API lto_codegen_get_files_need_remove().
Linker and LTO plugin will communicate via this API about which files
(including directories) need to removed before linker exit.
Added:
llvm/trunk/tools/lto/LTOPartition.cpp
llvm/trunk/tools/lto/LTOPartition.h
llvm/trunk/tools/lto/LTOPostIPODriver.cpp
llvm/trunk/tools/lto/LTOPostIPODriver.h
Modified:
llvm/trunk/include/llvm-c/lto.h
llvm/trunk/tools/gold/gold-plugin.cpp
llvm/trunk/tools/lto/CMakeLists.txt
llvm/trunk/tools/lto/LTOCodeGenerator.cpp
llvm/trunk/tools/lto/LTOCodeGenerator.h
llvm/trunk/tools/lto/lto.cpp
llvm/trunk/tools/lto/lto.exports
Modified: llvm/trunk/include/llvm-c/lto.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm-c/lto.h?rev=188188&r1=188187&r2=188188&view=diff
==============================================================================
--- llvm/trunk/include/llvm-c/lto.h (original)
+++ llvm/trunk/include/llvm-c/lto.h Mon Aug 12 13:29:43 2013
@@ -284,6 +284,18 @@ lto_codegen_compile(lto_code_gen_t cg, s
extern bool
lto_codegen_compile_to_file(lto_code_gen_t cg, const char** name);
+/**
+ * Get intermediate files that need to be removed before linker exit. Upon
+ * return, the paths of the files need to be removed is written to "paths". The
+ * paths are separated by a single '\0', and the last path is ended by double
+ * '\0's. A file could be a directory; in this case, the entire directory needs
+ * to be removed recursively.
+ *
+ * It is only necessary to call this function after \p lto_codegen_compile was
+ * successfully called.
+ */
+extern void
+lto_codegen_get_files_need_remove(lto_code_gen_t cg, const char **paths);
/**
* Sets options to help debug codegen bugs.
Modified: llvm/trunk/tools/gold/gold-plugin.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/gold/gold-plugin.cpp?rev=188188&r1=188187&r2=188188&view=diff
==============================================================================
--- llvm/trunk/tools/gold/gold-plugin.cpp (original)
+++ llvm/trunk/tools/gold/gold-plugin.cpp Mon Aug 12 13:29:43 2013
@@ -74,7 +74,6 @@ namespace options {
static bool generate_api_file = false;
static generate_bc generate_bc_file = BC_NO;
static std::string bc_path;
- static std::string obj_path;
static std::string extra_library_path;
static std::string triple;
static std::string mcpu;
@@ -99,8 +98,6 @@ namespace options {
extra_library_path = opt.substr(strlen("extra_library_path="));
} else if (opt.startswith("mtriple=")) {
triple = opt.substr(strlen("mtriple="));
- } else if (opt.startswith("obj-path=")) {
- obj_path = opt.substr(strlen("obj-path="));
} else if (opt == "emit-llvm") {
generate_bc_file = BC_ONLY;
} else if (opt == "also-emit-llvm") {
@@ -425,6 +422,14 @@ static ld_plugin_status all_symbols_read
(*message)(LDPL_ERROR, "Could not produce a combined object file\n");
}
+ // Get files that need to be removed in cleanup_hook.
+ const char *ToRm;
+ lto_codegen_get_files_need_remove(code_gen, &ToRm);
+ while (*ToRm) {
+ Cleanup.push_back(std::string(ToRm));
+ ToRm += strlen(ToRm) + 1;
+ }
+
lto_codegen_dispose(code_gen);
for (std::list<claimed_file>::iterator I = Modules.begin(),
E = Modules.end(); I != E; ++I) {
@@ -446,17 +451,28 @@ static ld_plugin_status all_symbols_read
return LDPS_ERR;
}
- if (options::obj_path.empty())
- Cleanup.push_back(objPath);
-
return LDPS_OK;
}
static ld_plugin_status cleanup_hook(void) {
for (int i = 0, e = Cleanup.size(); i != e; ++i) {
- error_code EC = sys::fs::remove(Cleanup[i]);
+ const char *FN = Cleanup[i].c_str();
+ sys::fs::file_status Stat;
+ error_code EC = sys::fs::status(Twine(FN), Stat);
+ if (EC) {
+ (*message)(LDPL_ERROR, "Failed to stat '%s': %s", FN,
+ EC.message().c_str());
+ continue;
+ }
+
+ uint32_t Dummy;
+ if (sys::fs::is_directory(FN))
+ EC = sys::fs::remove_all(Twine(FN), Dummy);
+ else
+ EC = sys::fs::remove(Twine(FN));
+
if (EC)
- (*message)(LDPL_ERROR, "Failed to delete '%s': %s", Cleanup[i].c_str(),
+ (*message)(LDPL_ERROR, "Failed to remove '%s': %s", FN,
EC.message().c_str());
}
Modified: llvm/trunk/tools/lto/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/CMakeLists.txt?rev=188188&r1=188187&r2=188188&view=diff
==============================================================================
--- llvm/trunk/tools/lto/CMakeLists.txt (original)
+++ llvm/trunk/tools/lto/CMakeLists.txt Mon Aug 12 13:29:43 2013
@@ -9,6 +9,8 @@ set(SOURCES
LTODisassembler.cpp
lto.cpp
LTOModule.cpp
+ LTOPartition.cpp
+ LTOPostIPODriver.cpp
)
set(LLVM_COMMON_DEPENDS intrinsics_gen)
Modified: llvm/trunk/tools/lto/LTOCodeGenerator.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/LTOCodeGenerator.cpp?rev=188188&r1=188187&r2=188188&view=diff
==============================================================================
--- llvm/trunk/tools/lto/LTOCodeGenerator.cpp (original)
+++ llvm/trunk/tools/lto/LTOCodeGenerator.cpp Mon Aug 12 13:29:43 2013
@@ -14,6 +14,8 @@
#include "LTOCodeGenerator.h"
#include "LTOModule.h"
+#include "LTOPartition.h"
+#include "LTOPostIPODriver.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Analysis/Passes.h"
#include "llvm/Analysis/Verifier.h"
@@ -35,11 +37,13 @@
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Program.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/Support/system_error.h"
+#include "llvm/Support/SourceMgr.h"
#include "llvm/Target/Mangler.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
@@ -47,8 +51,16 @@
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include "llvm/Transforms/ObjCARC.h"
+
using namespace llvm;
+using namespace lto;
+// /////////////////////////////////////////////////////////////////////////////
+//
+// Internal options. To avoid collision, most options start with "lto-".
+//
+// /////////////////////////////////////////////////////////////////////////////
+//
static cl::opt<bool>
DisableOpt("disable-opt", cl::init(false),
cl::desc("Do not run any optimization passes"));
@@ -61,6 +73,28 @@ static cl::opt<bool>
DisableGVNLoadPRE("disable-gvn-loadpre", cl::init(false),
cl::desc("Do not run the GVN load PRE pass"));
+// To break merged module into partitions, and compile them independently.
+static cl::opt<bool>
+EnablePartition("lto-partition", cl::init(false),
+ cl::desc("Partition program and compile each piece in parallel"));
+
+// Specify the work-directory for the LTO compilation. All intermeidate
+// files will be created immediately under this dir. If it is not
+// specified, compiler will create an unique directory under current-dir.
+//
+static cl::opt<std::string>
+TmpWorkDir("lto-workdir", cl::init(""), cl::desc("Specify working directory"));
+
+static cl::opt<bool>
+KeepWorkDir("lto-keep", cl::init(false), cl::desc("Keep working directory"));
+
+
+// /////////////////////////////////////////////////////////////////////////////
+//
+// Implementation of LTOCodeGenerator
+//
+// /////////////////////////////////////////////////////////////////////////////
+//
const char* LTOCodeGenerator::getVersionString() {
#ifdef LLVM_VERSION_INFO
return PACKAGE_NAME " version " PACKAGE_VERSION ", " LLVM_VERSION_INFO;
@@ -74,7 +108,8 @@ LTOCodeGenerator::LTOCodeGenerator()
_linker(new Module("ld-temp.o", _context)), _target(NULL),
_emitDwarfDebugInfo(false), _scopeRestrictionsDone(false),
_codeModel(LTO_CODEGEN_PIC_MODEL_DYNAMIC),
- _nativeObjectFile(NULL) {
+ _nativeObjectFile(NULL), PartitionMgr(FileMgr),
+ OptionsParsed(false) {
InitializeAllTargets();
InitializeAllTargetMCs();
InitializeAllAsmPrinters();
@@ -187,35 +222,41 @@ bool LTOCodeGenerator::writeMergedModule
return true;
}
-bool LTOCodeGenerator::compile_to_file(const char** name, std::string& errMsg) {
- // make unique temp .o file to put generated object file
- SmallString<128> Filename;
- int FD;
- error_code EC = sys::fs::createTemporaryFile("lto-llvm", "o", FD, Filename);
- if (EC) {
- errMsg = EC.message();
+// This function is to ensure cl::ParseCommandLineOptions() is called no more
+// than once. It would otherwise complain and exit the compilation prematurely.
+//
+void LTOCodeGenerator::parseOptions() {
+ if (OptionsParsed)
+ return;
+
+ if (!_codegenOptions.empty())
+ cl::ParseCommandLineOptions(_codegenOptions.size(),
+ const_cast<char **>(&_codegenOptions[0]));
+
+ OptionsParsed = true;
+}
+
+// Do some prepartion right before compilation starts.
+bool LTOCodeGenerator::prepareBeforeCompile(std::string &ErrMsg) {
+ parseOptions();
+
+ if (!determineTarget(ErrMsg))
return false;
- }
- // generate object file
- tool_output_file objFile(Filename.c_str(), FD);
+ FileMgr.setWorkDir(TmpWorkDir.c_str());
+ FileMgr.setKeepWorkDir(KeepWorkDir);
+ return FileMgr.createWorkDir(ErrMsg);
+}
- bool genResult = generateObjectFile(objFile.os(), errMsg);
- objFile.os().close();
- if (objFile.os().has_error()) {
- objFile.os().clear_error();
- sys::fs::remove(Twine(Filename));
+bool LTOCodeGenerator::compile_to_file(const char** Name, std::string& ErrMsg) {
+ if (!prepareBeforeCompile(ErrMsg))
return false;
- }
- objFile.keep();
- if (!genResult) {
- sys::fs::remove(Twine(Filename));
+ performIPO(EnablePartition, ErrMsg);
+ if (!performPostIPO(ErrMsg))
return false;
- }
- _nativeObjectPath = Filename.c_str();
- *name = _nativeObjectPath.c_str();
+ *Name = PartitionMgr.getSinglePartition()->getObjFilePath().c_str();
return true;
}
@@ -229,21 +270,41 @@ const void* LTOCodeGenerator::compile(si
// read .o file into memory buffer
OwningPtr<MemoryBuffer> BuffPtr;
+ const char *BufStart = 0;
+
if (error_code ec = MemoryBuffer::getFile(name, BuffPtr, -1, false)) {
errMsg = ec.message();
- sys::fs::remove(_nativeObjectPath);
- return NULL;
+ _nativeObjectFile = 0;
+ } else {
+ if ((_nativeObjectFile = BuffPtr.take())) {
+ *length = _nativeObjectFile->getBufferSize();
+ BufStart = _nativeObjectFile->getBufferStart();
+ }
}
- _nativeObjectFile = BuffPtr.take();
- // remove temp files
- sys::fs::remove(_nativeObjectPath);
+ // Now that the resulting single object file is handed to linker via memory
+ // buffer, it is safe to remove all intermediate files now.
+ //
+ FileMgr.removeAllUnneededFiles();
- // return buffer, unless error
- if (_nativeObjectFile == NULL)
- return NULL;
- *length = _nativeObjectFile->getBufferSize();
- return _nativeObjectFile->getBufferStart();
+ return BufStart;
+}
+
+const char *LTOCodeGenerator::getFilesNeedToRemove() {
+ IPOFileMgr::FileNameVect ToRm;
+ FileMgr.getFilesNeedToRemove(ToRm);
+
+ ConcatStrings.clear();
+ for (IPOFileMgr::FileNameVect::iterator I = ToRm.begin(), E = ToRm.end();
+ I != E; I++) {
+ StringRef S(*I);
+ ConcatStrings.append(S.begin(), S.end());
+ ConcatStrings.push_back('\0');
+ }
+ ConcatStrings.push_back('\0');
+ ConcatStrings.push_back('\0');
+
+ return ConcatStrings.data();
}
bool LTOCodeGenerator::determineTarget(std::string &errMsg) {
@@ -251,9 +312,7 @@ bool LTOCodeGenerator::determineTarget(s
return true;
// if options were requested, set them
- if (!_codegenOptions.empty())
- cl::ParseCommandLineOptions(_codegenOptions.size(),
- const_cast<char **>(&_codegenOptions[0]));
+ parseOptions();
std::string TripleStr = _linker.getModule()->getTargetTriple();
if (TripleStr.empty())
@@ -384,6 +443,70 @@ void LTOCodeGenerator::applyScopeRestric
_scopeRestrictionsDone = true;
}
+void LTOCodeGenerator::performIPO(bool ToPartition, std::string &errMsg) {
+ // Mark which symbols can not be internalized
+ applyScopeRestrictions();
+
+ // Instantiate the pass manager to organize the passes.
+ PassManager Passes;
+
+ // Start off with a verification pass.
+ Passes.add(createVerifierPass());
+
+ // Add an appropriate DataLayout instance for this module...
+ Passes.add(new DataLayout(*_target->getDataLayout()));
+ _target->addAnalysisPasses(Passes);
+
+ // Enabling internalize here would use its AllButMain variant. It
+ // keeps only main if it exists and does nothing for libraries. Instead
+ // we create the pass ourselves with the symbol list provided by the linker.
+ if (!DisableOpt)
+ PassManagerBuilder().populateLTOPassManager(Passes,
+ /*Internalize=*/false,
+ !DisableInline,
+ DisableGVNLoadPRE);
+ // Make sure everything is still good.
+ Passes.add(createVerifierPass());
+
+ Module* M = _linker.getModule();
+ if (ToPartition)
+ assert(false && "TBD");
+ else {
+ Passes.run(*M);
+
+ // Create a partition for the merged module.
+ PartitionMgr.createIPOPart(M);
+ }
+}
+
+// Perform Post-IPO compilation. If the partition is enabled, there may
+// be multiple partitions, and therefore there may be multiple objects.
+// In this case, "MergeObjs" indicates to merge all object together (via ld -r)
+// and return the path to the merged object via "MergObjPath".
+//
+bool LTOCodeGenerator::performPostIPO(std::string &ErrMsg,
+ bool MergeObjs,
+ const char **MergObjPath) {
+ // Determine the variant of post-ipo driver
+ PostIPODriver::VariantTy DrvTy;
+ if (!EnablePartition) {
+ assert(!MergeObjs && !MergObjPath && "Invalid parameter");
+ DrvTy = PostIPODriver::PIDV_SERIAL;
+ } else {
+ DrvTy = PostIPODriver::PIDV_Invalid;
+ assert(false && "TBD");
+ }
+
+ PostIPODriver D(DrvTy, _target, PartitionMgr, FileMgr, MergeObjs);
+ if (D.Compile(ErrMsg)) {
+ if (MergeObjs)
+ *MergObjPath = D.getSingleObjFile()->getPath().c_str();
+ return true;
+ }
+
+ return false;
+}
+
/// Optimize merged modules using various IPO passes
bool LTOCodeGenerator::generateObjectFile(raw_ostream &out,
std::string &errMsg) {
Modified: llvm/trunk/tools/lto/LTOCodeGenerator.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/LTOCodeGenerator.h?rev=188188&r1=188187&r2=188188&view=diff
==============================================================================
--- llvm/trunk/tools/lto/LTOCodeGenerator.h (original)
+++ llvm/trunk/tools/lto/LTOCodeGenerator.h Mon Aug 12 13:29:43 2013
@@ -41,6 +41,7 @@
#include "llvm/Linker.h"
#include <string>
#include <vector>
+#include "LTOPartition.h"
namespace llvm {
class LLVMContext;
@@ -102,16 +103,34 @@ struct LTOCodeGenerator {
//
const void *compile(size_t *length, std::string &errMsg);
+ // Return the paths of the intermediate files that linker needs to delete
+ // before it exits. The paths are delimited by a single '\0', and the last
+ // path is ended by double '\0's. The file could be a directory. In that
+ // case, the entire directory should be erased recusively. This function
+ // must be called after the compilexxx() is successfuly called, because
+ // only after that moment, compiler is aware which files need to be removed.
+ // If calling compilexxx() is not successful, it is up to compiler to clean
+ // up all the intermediate files generated during the compilation process.
+ //
+ const char *getFilesNeedToRemove();
+
private:
void initializeLTOPasses();
+ bool determineTarget(std::string &errMsg);
+ void parseOptions();
+ bool prepareBeforeCompile(std::string &ErrMsg);
+ void performIPO(bool PerformPartition, std::string &ErrMsg);
+ bool performPostIPO(std::string &ErrMsg, bool MergeObjs = false,
+ const char **MergObjPath = 0);
bool generateObjectFile(llvm::raw_ostream &out, std::string &errMsg);
+
void applyScopeRestrictions();
void applyRestriction(llvm::GlobalValue &GV,
std::vector<const char*> &mustPreserveList,
llvm::SmallPtrSet<llvm::GlobalValue*, 8> &asmUsed,
llvm::Mangler &mangler);
- bool determineTarget(std::string &errMsg);
+
typedef llvm::StringMap<uint8_t> StringSet;
@@ -127,6 +146,24 @@ private:
std::vector<char*> _codegenOptions;
std::string _mCpu;
std::string _nativeObjectPath;
+
+ // To manage the partitions. If partition is not enabled, the whole merged
+ // module is considered as a single degenerated partition, and the "manager"
+ // is still active.
+ lto::IPOPartMgr PartitionMgr;
+
+ // To manage the intermediate files during the compilations.
+ lto::IPOFileMgr FileMgr;
+
+ // Sometimes we need to return a vector of strings in a "C" way (to work with
+ // the C-APIs). We encode such C-thinking string vector by concatenating all
+ // strings tegother with a single '\0' as the delimitor, the last string ended
+ // by double '\0's.
+ SmallVector<char, 4> ConcatStrings;
+
+ // Make sure command line is parsed only once. It would otherwise complain
+ // and quite prematurely.
+ bool OptionsParsed;
};
#endif // LTO_CODE_GENERATOR_H
Added: llvm/trunk/tools/lto/LTOPartition.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/LTOPartition.cpp?rev=188188&view=auto
==============================================================================
--- llvm/trunk/tools/lto/LTOPartition.cpp (added)
+++ llvm/trunk/tools/lto/LTOPartition.cpp Mon Aug 12 13:29:43 2013
@@ -0,0 +1,205 @@
+//===-- LTOPartition.cpp - Parition Merged Module --------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "LTOPartition.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+
+using namespace llvm;
+using namespace lto;
+
+// /////////////////////////////////////////////////////////////////////////////
+//
+// Implementation of IPOPartition and IPOPartMgr
+//
+// /////////////////////////////////////////////////////////////////////////////
+//
+IPOPartition::IPOPartition(Module *M, const char *NameWoExt, IPOFileMgr &FM) :
+ Mod(0), Ctx(0), IRFile(0), ObjFile(0), FileNameWoExt(NameWoExt), FileMgr(FM) {
+}
+
+IPOFile &IPOPartition::getIRFile() const {
+ if (IRFile)
+ return *IRFile;
+ else {
+ std::string FN(FileNameWoExt + ".bc");
+ return *(IRFile = FileMgr.createIRFile(FN.c_str()));
+ }
+}
+
+IPOFile &IPOPartition::getObjFile() const {
+ if (ObjFile)
+ return *ObjFile;
+ else {
+ std::string FN(FileNameWoExt + ".o");
+ return *(ObjFile = FileMgr.createObjFile(FN.c_str()));
+ }
+}
+
+bool IPOPartition::saveBitCode() {
+ if (!Mod) {
+ // The bit-code have already saved in disk.
+ return true;
+ }
+
+ IPOFile &F = getIRFile();
+ if (F.errOccur())
+ return false;
+
+ raw_fd_ostream OF(F.getPath().c_str(), F.getLastErrStr(),
+ sys::fs::F_Binary);
+ WriteBitcodeToFile(Mod, OF);
+ OF.close();
+
+ Mod = 0;
+ delete Ctx;
+ Ctx = 0;
+
+ return !F.errOccur();
+}
+
+bool IPOPartition::loadBitCode() {
+ if (Mod)
+ return true;
+
+ IPOFile &F = getIRFile();
+ if (F.errOccur())
+ return false;
+
+ Ctx = new LLVMContext;
+
+ error_code &EC = F.getLastErrCode();
+ std::string &ErrMsg = F.getLastErrStr();
+
+ OwningPtr<MemoryBuffer> Buf;
+ if (error_code ec = MemoryBuffer::getFile(F.getPath(), Buf, -1, false)) {
+ EC = ec;
+ ErrMsg += ec.message();
+ return false;
+ }
+
+ Mod = ParseBitcodeFile(Buf.get(), *Ctx, &ErrMsg);
+
+ return Mod != 0;
+}
+
+IPOPartition *IPOPartMgr::createIPOPart(Module *M) {
+ std::string PartName;
+ raw_string_ostream OS(PartName);
+ OS << "part" << NextPartId++;
+
+ IPOPartition *P = new IPOPartition(M, OS.str().c_str(), FileMgr);
+ P->Mod = M;
+ IPOParts.push_back(P);
+ return P;
+}
+
+// ///////////////////////////////////////////////////////////////////////////
+//
+// Implementation of IPOFile and IPOFileMgr
+//
+// ///////////////////////////////////////////////////////////////////////////
+//
+IPOFile::IPOFile(const char *DirName, const char *BaseName, bool KeepFile)
+ : Fname(BaseName), Keep(KeepFile) {
+ // Concatenate dirname and basename
+ StringRef D(DirName);
+ SmallVector<char, 64> Path(D.begin(), D.end());
+ sys::path::append(Path, Twine(BaseName));
+ Fpath = StringRef(Path.data(), Path.size());
+}
+
+IPOFileMgr::IPOFileMgr() {
+ IRFiles.reserve(20);
+ ObjFiles.reserve(20);
+ OtherFiles.reserve(8);
+ KeepWorkDir = false;
+ WorkDirCreated = false;
+}
+
+bool IPOFileMgr::createWorkDir(std::string &ErrorInfo) {
+ if (WorkDirCreated)
+ return true;
+
+ error_code EC;
+ if (WorkDir.empty()) {
+ // If the workdir is not specified, then create workdir under current
+ // directory.
+ //
+ SmallString<128> D;
+ if (sys::fs::current_path(D) != error_code::success()) {
+ ErrorInfo += "fail to get current directory";
+ return false;
+ }
+ sys::path::append(D, "llvmipo");
+ sys::fs::make_absolute(D);
+
+ SmallVector<char, 64> ResPath;
+ EC = sys::fs::createUniqueDirectory(Twine(StringRef(D.data(), D.size())),
+ ResPath);
+ WorkDir = StringRef(ResPath.data(), ResPath.size());
+ } else {
+ bool Exist;
+ EC = sys::fs::create_directory(Twine(WorkDir), Exist);
+ }
+
+ if (EC == error_code::success()) {
+ WorkDirCreated = true;
+ return true;
+ }
+
+ return false;
+}
+
+IPOFile *IPOFileMgr::createIRFile(const char *Name) {
+ IPOFile *F = CreateFile(Name);
+ IRFiles.push_back(F);
+ return F;
+}
+
+IPOFile *IPOFileMgr::createObjFile(const char *Name) {
+ IPOFile *F = CreateFile(Name);
+ ObjFiles.push_back(F);
+ return F;
+}
+
+IPOFile *IPOFileMgr::createMakefile(const char *Name) {
+ IPOFile *F = CreateFile(Name);
+ OtherFiles.push_back(F);
+ return F;
+}
+
+void IPOFileMgr::removeAllUnneededFiles() {
+ FileNameVect ToRm;
+ getFilesNeedToRemove(ToRm);
+
+ for (SmallVector<const char *, 4>::iterator I = ToRm.begin(), E = ToRm.end();
+ I != E; I++) {
+ const char *FN = *I;
+ sys::fs::file_status Stat;
+ if (sys::fs::status(Twine(FN), Stat) != error_code::success())
+ continue;
+
+ uint32_t Dummy;
+ if (sys::fs::is_directory(FN))
+ sys::fs::remove_all(Twine(FN), Dummy);
+ else
+ sys::fs::remove(Twine(FN));
+ }
+}
Added: llvm/trunk/tools/lto/LTOPartition.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/LTOPartition.h?rev=188188&view=auto
==============================================================================
--- llvm/trunk/tools/lto/LTOPartition.h (added)
+++ llvm/trunk/tools/lto/LTOPartition.h Mon Aug 12 13:29:43 2013
@@ -0,0 +1,187 @@
+//===-------- LTOPartition.h - Partition related classes and functions ---===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declare the partition related classes and functions. A partition
+// is a portion of the merged module. In case partition is disabled, the entire
+// merged module is considered as a degenerated partition.
+//
+// The classes declared in this file are:
+// o. IPOPartition : to depicit a partition
+// o. IPOFile: It is a "container" collecting miscellaneous information about
+// an intermeidate file, including file name, path, last-err-message etc.
+// o. IPOPartMgr, IPOFileMgr: as the name suggests, it's the manager of
+// IPOPartitions and IPOFiles, respectively.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LTO_PARTITION_H
+#define LTO_PARTITION_H
+
+#include "llvm/Pass.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/Support/system_error.h"
+
+using namespace llvm;
+
+namespace lto {
+ /// \brief To collect miscellaneous information about an intermdiate file.
+ ///
+ /// These informration include file name, path, last error message etc.
+ ///
+ class IPOFile {
+ public:
+ const std::string &getName() { return Fname; }
+ const std::string &getPath() { return Fpath; }
+
+ error_code &getLastErrCode() { return LastErr; }
+ std::string &getLastErrStr() { return LastErrStr; }
+
+ bool errOccur() const {
+ return LastErr != error_code::success() || !LastErrStr.empty();
+ }
+
+ // To keep this file after compilation finish.
+ void setKeep() { Keep = true; }
+ bool isKept() const { return Keep; }
+
+ private:
+ friend class IPOFileMgr;
+ IPOFile(const char* DirName, const char *BaseName, bool Keep=false);
+ ~IPOFile();
+
+ private:
+ std::string Fname;
+ std::string Fpath;
+ error_code LastErr;
+ std::string LastErrStr;
+ bool Keep;
+ };
+
+ /// \brief To manage IPOFiles, create and remove work-directory.
+ ///
+ class IPOFileMgr {
+ public:
+ typedef SmallVector<const char *, 4> FileNameVect;
+
+ IPOFileMgr();
+
+ // NOTE: Do not delete intermeidate in the destructor as we never know
+ // if these files out-last the class or not. It is safe to let linker's
+ // clean-up hook to take care these files.
+ ~IPOFileMgr() {};
+
+ void setWorkDir(const char* WD) {
+ assert(!WorkDirCreated /* Too late to change mind */ &&
+ WorkDir.empty() /* don't change back and forth */ &&
+ "Cannot change work dir");
+ WorkDir = WD;
+ }
+ void setKeepWorkDir(bool Keep) { KeepWorkDir = Keep; }
+ bool IsToKeepWorkDir() const { return KeepWorkDir; }
+ const std::string &getWorkDir() { return WorkDir; }
+
+ bool createWorkDir(std::string &ErrorInfo);
+
+ IPOFile *createIRFile(const char *Name);
+ IPOFile *createObjFile(const char *Name);
+ IPOFile *createMakefile(const char *Name);
+
+ typedef std::vector<IPOFile *> FileVect;
+ FileVect &getIRFiles() { return IRFiles; }
+ FileVect &getObjFiles() { return ObjFiles; }
+
+ // Get all files/dirs that need to removed after the LTO complete.
+ void getFilesNeedToRemove(FileNameVect &ToRm) {
+ ToRm.clear();
+ if (!IsToKeepWorkDir() && WorkDirCreated)
+ ToRm.push_back(WorkDir.c_str());
+ }
+
+ // Remove all files/dirs returned from getFilesNeedToRemove().
+ void removeAllUnneededFiles();
+
+ private:
+ IPOFile *CreateFile(const char *Name) {
+ return new IPOFile(WorkDir.c_str(), Name);
+ }
+
+ private:
+ FileVect IRFiles;
+ FileVect ObjFiles;
+ FileVect OtherFiles;
+ std::string WorkDir;
+ bool KeepWorkDir;
+ bool WorkDirCreated;
+ };
+
+ /// \brief Describe a partition of the merged module.
+ ///
+ class IPOPartition {
+ public:
+ llvm::Module *getModule() const { return Mod; }
+ IPOFile &getIRFile() const;
+ IPOFile &getObjFile() const;
+ const std::string &getIRFilePath() const { return getIRFile().getPath(); }
+ const std::string &getObjFilePath() const { return getObjFile().getPath(); }
+
+ // If the bitcode reside in memory or disk
+ bool isInMemory() const { return Mod != 0; }
+
+ // Load/store bitcode from/to disk file.
+ bool saveBitCode();
+ bool loadBitCode();
+
+ private:
+ friend class IPOPartMgr;
+ IPOPartition(llvm::Module *M, const char *FileNameWoExt, IPOFileMgr &FM);
+
+ // The module associated with this partition
+ Module *Mod;
+ LLVMContext *Ctx;
+
+ // The bitcode file and its corresponding object file associated with
+ // this partition. The names of these two files are different only in
+ // extension; the "FileNameWoExt" record their (common) name without
+ // extension.
+ //
+ mutable IPOFile *IRFile;
+ mutable IPOFile *ObjFile;
+ std::string FileNameWoExt;
+
+ IPOFileMgr &FileMgr;
+ };
+
+ /// \brief To manage IPOPartitions
+ ///
+ class IPOPartMgr {
+ public:
+ IPOPartMgr(IPOFileMgr &IFM) : FileMgr(IFM), NextPartId(1) {}
+
+ typedef std::vector<IPOPartition *> IPOPartsTy;
+ typedef IPOPartsTy::iterator iterator;
+ typedef IPOPartsTy::const_iterator const_iterator;
+
+ iterator begin() { return IPOParts.begin(); }
+ iterator end() { return IPOParts.end(); }
+ const_iterator begin() const { return IPOParts.begin(); }
+ const_iterator end() const { return IPOParts.end(); }
+
+ IPOPartition *createIPOPart(Module *);
+ IPOPartition *getSinglePartition() {
+ assert(IPOParts.size() == 1 && "Has multiple partition");
+ return IPOParts[0];
+ }
+
+ private:
+ IPOPartsTy IPOParts;
+ IPOFileMgr &FileMgr;
+ int NextPartId;
+ };
+
+}
+
+#endif //LTO_PARTITION_H
Added: llvm/trunk/tools/lto/LTOPostIPODriver.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/LTOPostIPODriver.cpp?rev=188188&view=auto
==============================================================================
--- llvm/trunk/tools/lto/LTOPostIPODriver.cpp (added)
+++ llvm/trunk/tools/lto/LTOPostIPODriver.cpp Mon Aug 12 13:29:43 2013
@@ -0,0 +1,180 @@
+//===---------- LTOPostIPODriver.h - PostIPO Driver -----------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the PostIPODriver class which is the driver for Post-IPO
+// compilation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/PassManager.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/ObjCARC.h"
+#include "LTOPartition.h"
+#include "LTOPostIPODriver.h"
+
+using namespace llvm;
+using namespace lto;
+
+// /////////////////////////////////////////////////////////////////////////////
+//
+// Declare all variants of Post-IPO drivers
+//
+// /////////////////////////////////////////////////////////////////////////////
+//
+namespace {
+ /// \breif Base class for all driver variants.
+ ///
+ class PostIPODrvBaseImpl {
+ public:
+ PostIPODrvBaseImpl(TargetMachine *Targ, IPOPartMgr &IPM, IPOFileMgr &IFM,
+ bool ToMergeObjs):
+ PartMgr(IPM), FileMgr(IFM), MergedObjFile(0), Target(Targ),
+ MergeObjs(ToMergeObjs) {}
+
+ virtual ~PostIPODrvBaseImpl() {};
+
+ IPOPartMgr &getPartitionMgr() { return PartMgr; }
+ IPOFileMgr &getFileMgr() { return FileMgr; }
+
+ // Implement the PostIPODriver::getSingleObjFile()
+ virtual IPOFile *getSingleObjFile() const = 0;
+
+ bool IsToMergeObj() const { return MergeObjs; }
+
+ virtual bool Compile(std::string &ErrMsg) = 0;
+
+ protected:
+ // Populate post-IPO scalar optimization pass manager
+ bool PopulatePostIPOOptPM(PassManager &PM);
+
+ // Populate post-IPO machine-specific CodeGen pass manager
+ bool PopulateCodeGenPM(PassManager &PM, formatted_raw_ostream &OutFile,
+ std::string &Err);
+
+ protected:
+ IPOPartMgr &PartMgr;
+ IPOFileMgr &FileMgr;
+ IPOFile *MergedObjFile;
+ TargetMachine *Target;
+ bool MergeObjs;
+ };
+
+ /// \breif PostIPO driver for the compiling the entire program without
+ /// partition.
+ class PostIPODrvSerial : public PostIPODrvBaseImpl {
+ public:
+ PostIPODrvSerial(TargetMachine *T, IPOPartMgr &IPM, IPOFileMgr &IFM,
+ bool ToMergeObjs) :
+ PostIPODrvBaseImpl(T, IPM, IFM, ToMergeObjs) {}
+
+ virtual bool Compile(std::string &ErrMsg);
+ virtual IPOFile *getSingleObjFile() const;
+
+ private:
+ Module *getModule() const { return (*PartMgr.begin())->getModule(); }
+ };
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+//
+// Implemetation of PostIPODriver
+//
+// ////////////////////////////////////////////////////////////////////////////
+//
+PostIPODriver::PostIPODriver(VariantTy V, TargetMachine *TM, IPOPartMgr &IPM,
+ IPOFileMgr &IFM, bool ToMergeObjs) {
+ if (V == PIDV_SERIAL)
+ DrvImpl = new PostIPODrvSerial(TM, IPM, IFM, ToMergeObjs);
+ else
+ assert(false && "TBD");
+}
+
+bool PostIPODriver::Compile(std::string &ErrMsg) {
+ PostIPODrvBaseImpl *P = static_cast<PostIPODrvBaseImpl *>(DrvImpl);
+ return P->Compile(ErrMsg);
+}
+
+IPOFile *PostIPODriver::getSingleObjFile() const {
+ PostIPODrvBaseImpl *P = static_cast<PostIPODrvBaseImpl *>(DrvImpl);
+ return P->getSingleObjFile();
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+//
+// Implemetation of PostIPODrvBaseImpl
+//
+// ////////////////////////////////////////////////////////////////////////////
+//
+bool PostIPODrvBaseImpl::PopulatePostIPOOptPM(PassManager &PM) {
+ (void)PM;
+ return true;
+}
+
+bool PostIPODrvBaseImpl::PopulateCodeGenPM(PassManager &PM,
+ formatted_raw_ostream &OutFile,
+ std::string &Err) {
+ PM.add(new DataLayout(*Target->getDataLayout()));
+ Target->addAnalysisPasses(PM);
+
+ // If the bitcode files contain ARC code and were compiled with optimization,
+ // the ObjCARCContractPass must be run, so do it unconditionally here.
+ PM.add(createObjCARCContractPass());
+
+ if (Target->addPassesToEmitFile(PM, OutFile,
+ TargetMachine::CGFT_ObjectFile)) {
+ Err = "target file type not supported";
+ return false;
+ }
+ return true;
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+//
+// Implemetation of PostIPODrvSerial
+//
+// ////////////////////////////////////////////////////////////////////////////
+//
+bool PostIPODrvSerial::Compile(std::string &ErrMsg) {
+ Module *M = getModule();
+
+ // Step 1: Run the post-IPO scalar optimizations
+ {
+ PassManager SoptPM;
+ PopulatePostIPOOptPM(SoptPM);
+ SoptPM.run(*M);
+ }
+
+ // Step 2: Run the post-IPO machine-specific code-generation passes
+ {
+ IPOFile &Obj = (*PartMgr.begin())->getObjFile();
+ raw_fd_ostream ros(Obj.getPath().c_str(), Obj.getLastErrStr(),
+ sys::fs::F_Binary);
+ formatted_raw_ostream OutFile(ros);
+
+ PassManager CodGenPM;
+ if (!PopulateCodeGenPM(CodGenPM, OutFile, ErrMsg)) {
+ ErrMsg += Obj.getLastErrStr();
+ return false;
+ }
+
+ CodGenPM.run(*M);
+ }
+
+ return true;
+}
+
+IPOFile *PostIPODrvSerial::getSingleObjFile() const {
+ assert(!MergedObjFile && "No need to *merge* a single object file");
+ IPOPartition *P = *PartMgr.begin();
+ return &P->getObjFile();
+}
Added: llvm/trunk/tools/lto/LTOPostIPODriver.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/LTOPostIPODriver.h?rev=188188&view=auto
==============================================================================
--- llvm/trunk/tools/lto/LTOPostIPODriver.h (added)
+++ llvm/trunk/tools/lto/LTOPostIPODriver.h Mon Aug 12 13:29:43 2013
@@ -0,0 +1,52 @@
+//===---------- LTOPostIPODriver.h - PostIPO Driver -----------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declare the PostIPODriver class which is the driver for
+// Post-IPO compilation phase.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LTO_POSTIPO_DRIVER_H
+#define LTO_POSTIPO_DRIVER_H
+
+#include "llvm/Target/TargetMachine.h"
+
+namespace lto {
+ class IPOPartMgr;
+ class IPOFileMgr;
+ class IPOFile;
+
+ class PostIPODriver {
+ public:
+ typedef enum {
+ PIDV_Invalid,
+ PIDV_SERIAL, // No partition
+ PIDV_MultiThread, // Each partition is compiled by a thread
+ PIDV_MultiProc, // Each partition is compiled by a process
+ PIDV_MakeUtil // Partitions compilation is driven by a make-utility
+ } VariantTy;
+
+ PostIPODriver(VariantTy Var, TargetMachine *TM, IPOPartMgr &IPM,
+ IPOFileMgr &IFM, bool ToMergeObjs = false);
+
+ // Return the single resulting object file. If there is no prior
+ // compilation failure, this function may return NULL iff:
+ // 1) Partition is enabled, and
+ // 2) Multiple partitions are generated, and
+ // 3) It is not asked to merge together the objects corresponding to the
+ // the partions.
+ IPOFile *getSingleObjFile() const;
+
+ bool Compile(std::string &ErrMsg);
+
+ private:
+ void *DrvImpl;
+ VariantTy DrvStyle;
+ };
+}
+
+#endif // LTO_POSTIPO_DRIVER_H
Modified: llvm/trunk/tools/lto/lto.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/lto.cpp?rev=188188&r1=188187&r2=188188&view=diff
==============================================================================
--- llvm/trunk/tools/lto/lto.cpp (original)
+++ llvm/trunk/tools/lto/lto.cpp Mon Aug 12 13:29:43 2013
@@ -207,6 +207,19 @@ bool lto_codegen_compile_to_file(lto_cod
return !cg->compile_to_file(name, sLastErrorString);
}
+/// Get intermediate files that need to be removed before linker exit. Upon
+/// return, the paths of the files need to be removed is written to "paths". The
+/// paths are separated by a single '\0', and the last path is ended by double
+/// '\0's. A file could be a directory; in this case, the entire directory needs
+/// to be removed recursively.
+///
+/// It is only necessary to call this function after \p lto_codegen_compile was
+/// successfully called.
+void
+lto_codegen_get_files_need_remove(lto_code_gen_t cg, const char **paths) {
+ *paths = cg->getFilesNeedToRemove();
+}
+
/// lto_codegen_debug_options - Used to pass extra options to the code
/// generator.
void lto_codegen_debug_options(lto_code_gen_t cg, const char *opt) {
Modified: llvm/trunk/tools/lto/lto.exports
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/lto.exports?rev=188188&r1=188187&r2=188188&view=diff
==============================================================================
--- llvm/trunk/tools/lto/lto.exports (original)
+++ llvm/trunk/tools/lto/lto.exports Mon Aug 12 13:29:43 2013
@@ -20,6 +20,7 @@ lto_codegen_add_must_preserve_symbol
lto_codegen_compile
lto_codegen_create
lto_codegen_dispose
+lto_codegen_get_files_need_remove
lto_codegen_set_debug_model
lto_codegen_set_pic_model
lto_codegen_write_merged_modules
More information about the llvm-commits
mailing list