[PATCH] LTO: introduce object file-based on-disk module format.
Alp Toker
alp at nuanti.com
Thu Jul 3 18:44:42 PDT 2014
On 03/07/2014 10:56, Peter Collingbourne wrote:
> Hi rafael,
>
> This format is simply a regular object file with the bitcode stored in a
> section named ".llvmbc", plus any number of other (non-allocated) sections.
>
> One immediate use case for this is to accommodate compilation processes
> which expect the object file to contain metadata in non-allocated sections,
> such as the ".go_export" section used by some Go compilers [1], although I
> imagine that in the future we could consider compiling parts of the module
> (such as large non-inlinable functions) directly into the object file to
> improve LTO efficiency.
>
> [1]http://golang.org/doc/install/gccgo#Imports
>
> http://reviews.llvm.org/D4371
>
> Files:
> include/llvm/LTO/LTOModule.h
> lib/LTO/LTOModule.cpp
> test/LTO/Inputs/bcsection.macho.s
> test/LTO/Inputs/bcsection.s
> test/LTO/bcsection.ll
>
> D4371.11045.patch
>
>
> Index: include/llvm/LTO/LTOModule.h
> ===================================================================
> --- include/llvm/LTO/LTOModule.h
> +++ include/llvm/LTO/LTOModule.h
> @@ -202,12 +202,11 @@
> /// Get string that the data pointer points to.
> bool objcClassNameFromExpression(const Constant *c, std::string &name);
>
> - /// Returns 'true' if the memory buffer is for the specified target triple.
> - static bool isTargetMatch(MemoryBuffer *memBuffer, const char *triplePrefix);
> + /// Returns 'true' if the bitcode bc is for the specified target triple.
> + static bool isTargetMatch(StringRef bc, const char *triplePrefix);
>
> - /// Create an LTOModule (private version). N.B. This method takes ownership of
> - /// the buffer.
> - static LTOModule *makeLTOModule(MemoryBuffer *buffer, TargetOptions options,
> + /// Create an LTOModule (private version).
> + static LTOModule *makeLTOModule(StringRef data, TargetOptions options,
> std::string &errMsg);
>
> /// Create a MemoryBuffer from a memory range with an optional name.
> Index: lib/LTO/LTOModule.cpp
> ===================================================================
> --- lib/LTO/LTOModule.cpp
> +++ lib/LTO/LTOModule.cpp
> @@ -29,6 +29,7 @@
> #include "llvm/MC/MCSymbol.h"
> #include "llvm/MC/MCTargetAsmParser.h"
> #include "llvm/MC/SubtargetFeature.h"
> +#include "llvm/Object/ObjectFile.h"
> #include "llvm/Support/CommandLine.h"
> #include "llvm/Support/FileSystem.h"
> #include "llvm/Support/Host.h"
> @@ -43,6 +44,7 @@
> #include "llvm/Transforms/Utils/GlobalStatus.h"
> #include <system_error>
> using namespace llvm;
> +using namespace llvm::object;
>
> LTOModule::LTOModule(llvm::Module *m, llvm::TargetMachine *t)
> : _module(m), _target(t),
> @@ -53,43 +55,107 @@
> _context);
> }
>
> +static StringRef getObjectBitcodeData(ObjectFile *obj,
> + std::string *errMsg = 0) {
> + for (auto sec : obj->sections()) {
> + StringRef secName;
> + if (std::error_code ec = sec.getName(secName)) {
> + if (errMsg)
> + *errMsg = ec.message();
> + return StringRef();
> + }
> + if (secName == ".llvmbc") {
> + StringRef secContents;
> + if (std::error_code ec = sec.getContents(secContents)) {
> + if (errMsg)
> + *errMsg = ec.message();
> + return StringRef();
> + }
> + return secContents;
> + }
> + }
> +
> + if (errMsg)
> + *errMsg = "section .llvmbc not found";
> + return StringRef();
> +}
I feel the object file-based bitcode support should be layered in to a
separate source file.
> +
> +static StringRef getBitcodeData(StringRef str, std::string *errMsg = 0) {
> + switch (sys::fs::identify_magic(str)) {
> + case sys::fs::file_magic::bitcode:
> + return str;
> + case sys::fs::file_magic::elf_relocatable:
> + case sys::fs::file_magic::macho_object:
> + case sys::fs::file_magic::coff_object: {
> + std::unique_ptr<MemoryBuffer> buffer(
> + MemoryBuffer::getMemBuffer(str, "", false));
> + auto obj = ObjectFile::createObjectFile(buffer);
> + if (!obj) {
> + if (errMsg)
> + *errMsg = obj.getError().message();
> + return StringRef();
> + }
> + return getObjectBitcodeData(*obj, errMsg);
> + }
> + default:
> + return StringRef();
> + }
> +}
> +
> /// isBitcodeFile - Returns 'true' if the file (or memory contents) is LLVM
> /// bitcode.
> bool LTOModule::isBitcodeFile(const void *mem, size_t length) {
> - return sys::fs::identify_magic(StringRef((const char *)mem, length)) ==
> - sys::fs::file_magic::bitcode;
> + return !getBitcodeData(StringRef((const char *)mem, length)).empty();
This is potentially less efficient. Why change it from using magic?
> }
>
> bool LTOModule::isBitcodeFile(const char *path) {
> sys::fs::file_magic type;
> if (sys::fs::identify_magic(path, type))
> return false;
> - return type == sys::fs::file_magic::bitcode;
> + switch (type) {
> + case sys::fs::file_magic::bitcode:
> + return true;
> + case sys::fs::file_magic::elf_relocatable:
> + case sys::fs::file_magic::macho_object:
> + case sys::fs::file_magic::coff_object: {
> + auto Obj = ObjectFile::createObjectFile(path);
> + if (!Obj)
> + return false;
> + return !getObjectBitcodeData(*Obj).empty();
I don't like this at all. The behaviour of existing isBitcodeFile()
shouldn't just change.
This looks much more like it should be a isObjectFileWithBitcode() that
consumers can optionally call. It's really up to the library user what
input formats they want to accept.
Alp.
> + }
> + default:
> + return false;
> + }
> }
>
> /// isBitcodeFileForTarget - Returns 'true' if the file (or memory contents) is
> /// LLVM bitcode for the specified triple.
> bool LTOModule::isBitcodeFileForTarget(const void *mem, size_t length,
> const char *triplePrefix) {
> - MemoryBuffer *buffer = makeBuffer(mem, length);
> - if (!buffer)
> + StringRef data =
> + getBitcodeData(StringRef((const char *)mem, length));
> + if (data.empty())
> return false;
> - return isTargetMatch(buffer, triplePrefix);
> + return isTargetMatch(data, triplePrefix);
> }
>
> bool LTOModule::isBitcodeFileForTarget(const char *path,
> const char *triplePrefix) {
> std::unique_ptr<MemoryBuffer> buffer;
> if (MemoryBuffer::getFile(path, buffer))
> return false;
> - return isTargetMatch(buffer.release(), triplePrefix);
> + StringRef data = getBitcodeData(buffer->getBuffer());
> + if (data.empty())
> + return false;
> + return isTargetMatch(data, triplePrefix);
> }
>
> -/// isTargetMatch - Returns 'true' if the memory buffer is for the specified
> +/// isTargetMatch - Returns 'true' if the bitcode bc is for the specified
> /// target triple.
> -bool LTOModule::isTargetMatch(MemoryBuffer *buffer, const char *triplePrefix) {
> - std::string Triple = getBitcodeTargetTriple(buffer, getGlobalContext());
> - delete buffer;
> +bool LTOModule::isTargetMatch(StringRef bc, const char *triplePrefix) {
> + std::unique_ptr<MemoryBuffer> buffer(
> + MemoryBuffer::getMemBuffer(bc, "", false));
> + std::string Triple = getBitcodeTargetTriple(buffer.get(), getGlobalContext());
> return strncmp(Triple.c_str(), triplePrefix, strlen(triplePrefix)) == 0;
> }
>
> @@ -102,7 +168,7 @@
> errMsg = ec.message();
> return nullptr;
> }
> - return makeLTOModule(buffer.release(), options, errMsg);
> + return makeLTOModule(buffer->getBuffer(), options, errMsg);
> }
>
> LTOModule *LTOModule::createFromOpenFile(int fd, const char *path, size_t size,
> @@ -121,27 +187,33 @@
> errMsg = ec.message();
> return nullptr;
> }
> - return makeLTOModule(buffer.release(), options, errMsg);
> + return makeLTOModule(buffer->getBuffer(), options, errMsg);
> }
>
> LTOModule *LTOModule::createFromBuffer(const void *mem, size_t length,
> TargetOptions options,
> std::string &errMsg, StringRef path) {
> std::unique_ptr<MemoryBuffer> buffer(makeBuffer(mem, length, path));
> if (!buffer)
> return nullptr;
> - return makeLTOModule(buffer.release(), options, errMsg);
> + return makeLTOModule(StringRef((const char *)mem, length), options, errMsg);
> }
>
> -LTOModule *LTOModule::makeLTOModule(MemoryBuffer *buffer,
> +LTOModule *LTOModule::makeLTOModule(StringRef data,
> TargetOptions options,
> std::string &errMsg) {
> + StringRef bc = getBitcodeData(data, &errMsg);
> + if (bc.empty())
> + return nullptr;
> +
> + std::unique_ptr<MemoryBuffer> buffer(
> + MemoryBuffer::getMemBuffer(bc, "", false));
> +
> // parse bitcode buffer
> ErrorOr<Module *> ModuleOrErr =
> - getLazyBitcodeModule(buffer, getGlobalContext());
> + parseBitcodeFile(buffer.get(), getGlobalContext());
> if (std::error_code EC = ModuleOrErr.getError()) {
> errMsg = EC.message();
> - delete buffer;
> return nullptr;
> }
> std::unique_ptr<Module> m(ModuleOrErr.get());
> @@ -174,8 +246,6 @@
>
> TargetMachine *target = march->createTargetMachine(TripleStr, CPU, FeatureStr,
> options);
> - m->materializeAllPermanently();
> -
> LTOModule *Ret = new LTOModule(m.release(), target);
>
> // We need a MCContext set up in order to get mangled names of private
> Index: test/LTO/Inputs/bcsection.macho.s
> ===================================================================
> --- /dev/null
> +++ test/LTO/Inputs/bcsection.macho.s
> @@ -0,0 +1,2 @@
> +.section .llvmbc,.llvmbc
> +.incbin "bcsection.bc"
> Index: test/LTO/Inputs/bcsection.s
> ===================================================================
> --- /dev/null
> +++ test/LTO/Inputs/bcsection.s
> @@ -0,0 +1,2 @@
> +.section .llvmbc
> +.incbin "bcsection.bc"
> Index: test/LTO/bcsection.ll
> ===================================================================
> --- /dev/null
> +++ test/LTO/bcsection.ll
> @@ -0,0 +1,18 @@
> +; RUN: llvm-as -o %T/bcsection.bc %s
> +
> +; RUN: llvm-mc -I=%T -filetype=obj -triple=x86_64-pc-win32 -o %T/bcsection.coff.bco %p/Inputs/bcsection.s
> +; RUN: llvm-lto -exported-symbol=main -o %T/bcsection.coff.o %T/bcsection.coff.bco
> +; RUN: llvm-nm %T/bcsection.coff.o | FileCheck %s
> +
> +; RUN: llvm-mc -I=%T -filetype=obj -triple=x86_64-unknown-linux-gnu -o %T/bcsection.elf.bco %p/Inputs/bcsection.s
> +; RUN: llvm-lto -exported-symbol=main -o %T/bcsection.elf.o %T/bcsection.elf.bco
> +; RUN: llvm-nm %T/bcsection.elf.o | FileCheck %s
> +
> +; RUN: llvm-mc -I=%T -filetype=obj -triple=x86_64-apple-darwin11 -o %T/bcsection.macho.bco %p/Inputs/bcsection.macho.s
> +; RUN: llvm-lto -exported-symbol=main -o %T/bcsection.macho.o %T/bcsection.macho.bco
> +; RUN: llvm-nm %T/bcsection.macho.o | FileCheck %s
> +
> +; CHECK: main
> +define i32 @main() {
> + ret i32 0
> +}
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits
--
http://www.nuanti.com
the browser experts
More information about the llvm-commits
mailing list