[llvm] r221733 - libLTO: Allow linker to choose context of modules and codegen

[Duncan P. N. Exon Smith]

Author: dexonsmith
Date: Tue Nov 11 17:19:23 2014
New Revision: 221733

URL: http://llvm.org/viewvc/llvm-project?rev=221733&view=rev
Log:
libLTO: Allow linker to choose context of modules and codegen

Add API for specifying which `LLVMContext` each `lto_module_t` and
`lto_code_gen_t` is in.

In particular, this enables the following flow:

    for (auto &File : Files) {
      lto_module_t M = lto_module_create_in_local_context(File...);
      querySymbols(M);
      lto_module_dispose(M);
    }

    lto_code_gen_t CG = lto_codegen_create_in_local_context();
    for (auto &File : FilesToLink) {
      lto_module_t M = lto_module_create_in_codegen_context(File..., CG);
      lto_codegen_add_module(CG, M);
      lto_module_dispose(M);
    }
    lto_codegen_compile(CG);
    lto_codegen_write_merged_modules(CG, ...);
    lto_codegen_dispose(CG);

This flow has a few benefits.

  - Only one module (two if you count the combined module in the code
    generator) is in memory at a time.

  - Metadata (and constants) from files that are parsed to query symbols
    but not linked into the code generator don't pollute the global
    context.

  - The first for loop can be parallelized, since each module is in its
    own context.

  - When the code generator is disposed, the memory from LTO gets freed.

rdar://problem/18767512

Modified:
    llvm/trunk/include/llvm-c/lto.h
    llvm/trunk/tools/lto/lto.cpp

Modified: llvm/trunk/include/llvm-c/lto.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm-c/lto.h?rev=221733&r1=221732&r2=221733&view=diff
==============================================================================
--- llvm/trunk/include/llvm-c/lto.h (original)
+++ llvm/trunk/include/llvm-c/lto.h Tue Nov 11 17:19:23 2014
@@ -40,7 +40,7 @@ typedef bool lto_bool_t;
  * @{
  */
 
-#define LTO_API_VERSION 10
+#define LTO_API_VERSION 11
 
 /**
  * \since prior to LTO_API_VERSION=3
@@ -178,6 +178,35 @@ lto_module_create_from_memory_with_path(
                                         const char *path);
 
 /**
+ * \brief Loads an object file in its own context.
+ *
+ * Loads an object file in its own LLVMContext.  This function call is
+ * thread-safe.  However, modules created this way should not be merged into an
+ * lto_code_gen_t using \a lto_codegen_add_module().
+ *
+ * Returns NULL on error (check lto_get_error_message() for details).
+ *
+ * \since LTO_API_VERSION=11
+ */
+extern lto_module_t
+lto_module_create_in_local_context(const void *mem, size_t length,
+                                   const char *path);
+
+/**
+ * \brief Loads an object file in the codegen context.
+ *
+ * Loads an object file into the same context as \c cg.  The module is safe to
+ * add using \a lto_codegen_add_module().
+ *
+ * Returns NULL on error (check lto_get_error_message() for details).
+ *
+ * \since LTO_API_VERSION=11
+ */
+extern lto_module_t
+lto_module_create_in_codegen_context(const void *mem, size_t length,
+                                     const char *path, lto_code_gen_t cg);
+
+/**
  * Loads an object file from disk. The seek point of fd is not preserved.
  * Returns NULL on error (check lto_get_error_message() for details).
  *
@@ -324,12 +353,27 @@ extern void lto_codegen_set_diagnostic_h
  * Instantiates a code generator.
  * Returns NULL on error (check lto_get_error_message() for details).
  *
+ * All modules added using \a lto_codegen_add_module() must have been created
+ * in the same context as the codegen.
+ *
  * \since prior to LTO_API_VERSION=3
  */
 extern lto_code_gen_t
 lto_codegen_create(void);
 
 /**
+ * \brief Instantiate a code generator in its own context.
+ *
+ * Instantiates a code generator in its own context.  Modules added via \a
+ * lto_codegen_add_module() must have all been created in the same context,
+ * using \a lto_module_create_in_codegen_context().
+ *
+ * \since LTO_API_VERSION=11
+ */
+extern lto_code_gen_t
+lto_codegen_create_in_local_context(void);
+
+/**
  * Frees all code generator and all memory it internally allocated.
  * Upon return the lto_code_gen_t is no longer valid.
  *
@@ -342,6 +386,10 @@ lto_codegen_dispose(lto_code_gen_t);
  * Add an object module to the set of modules for which code will be generated.
  * Returns true on error (check lto_get_error_message() for details).
  *
+ * \c cg and \c mod must both be in the same context.  See \a
+ * lto_codegen_create_in_local_context() and \a
+ * lto_module_create_in_codegen_context().
+ *
  * \since prior to LTO_API_VERSION=3
  */
 extern lto_bool_t

Modified: llvm/trunk/tools/lto/lto.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/lto/lto.cpp?rev=221733&r1=221732&r2=221733&view=diff
==============================================================================
--- llvm/trunk/tools/lto/lto.cpp (original)
+++ llvm/trunk/tools/lto/lto.cpp Tue Nov 11 17:19:23 2014
@@ -150,6 +150,24 @@ lto_module_t lto_module_create_from_memo
       LTOModule::createFromBuffer(mem, length, Options, sLastErrorString, path));
 }
 
+lto_module_t lto_module_create_in_local_context(const void *mem, size_t length,
+                                                const char *path) {
+  lto_initialize();
+  llvm::TargetOptions Options = InitTargetOptionsFromCodeGenFlags();
+  return wrap(LTOModule::createInLocalContext(mem, length, Options,
+                                              sLastErrorString, path));
+}
+
+lto_module_t lto_module_create_in_codegen_context(const void *mem,
+                                                  size_t length,
+                                                  const char *path,
+                                                  lto_code_gen_t cg) {
+  lto_initialize();
+  llvm::TargetOptions Options = InitTargetOptionsFromCodeGenFlags();
+  return wrap(LTOModule::createInContext(mem, length, Options, sLastErrorString,
+                                         path, &unwrap(cg)->getContext()));
+}
+
 void lto_module_dispose(lto_module_t mod) { delete unwrap(mod); }
 
 const char* lto_module_get_target_triple(lto_module_t mod) {