<div dir="ltr">Hi Peter, I'm getting compilation failures with this.<div><br></div><div><div>[748/3638] Building CXX object lib\LTO\CMakeFiles\LLVMLTO.dir\Caching.cpp.obj</div><div>FAILED: C:\PROGRA~2\MICROS~1.0\VC\bin\AMD64_~2\cl.exe /nologo /TP -DGTEST_HAS_RTTI=0 -DUNICODE -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS -D_HAS_EXCEPTIONS=0 -D_SCL_SECURE_NO_DEPRECATE -D_SCL_SECURE_NO_WARNINGS -D_UNICODE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -Ilib\LTO -ID:\src\llvm\lib\LTO -Iinclude -ID:\src\llvm\include /DWIN32 /D_WINDOWS /W4 -wd4141 -wd4146 -wd4180 -wd4244 -wd4258 -wd4267 -wd4291 -wd4345 -wd4351 -wd4355 -wd4456 -wd4457 -wd4458 -wd4459 -wd4503 -wd4624 -wd4722 -wd4800 -wd4100 -wd4127 -wd4512 -wd4505 -wd4610 -wd4510 -wd4702 -wd4245 -wd4706 -wd4310 -wd4701 -wd4703 -wd4389 -wd4611 -wd4805 -wd4204 -wd4577 -wd4091 -wd4592 -wd4319 -wd4324 -w14062 -we4238 /Zc:inline /Zc:strictStrings /Oi /Zc:rvalueCast /Zc:sizedDealloc- /D_DEBUG /MDd /Zi /Ob0 /Od /RTC1 /EHs-c- /GR- /showIncludes /Folib\LTO\CMakeFiles\LLVMLTO.dir\Caching.cpp.obj /Fdlib\LTO\CMakeFiles\LLVMLTO.dir\ /FS -c D:\src\llvm\lib\LTO\Caching.cpp</div><div>D:\src\llvm\lib\LTO\Caching.cpp(96): error C2668: 'llvm::make_unique': ambiguous call to overloaded function D:\src\llvm\include\llvm/ADT/STLExtras.h(563): note: could be 'std::unique_ptr<llvm::lto::localCache::<lambda_24b0b9154e270ae79e71d53f81fee0dc>::()::CacheStream,std::default_delete<_Ty>> llvm::make_unique<llvm::lto::localCache::<lambda_24b0b9154e270ae79e71d53f81fee0dc>::()::CacheStream,std::unique_ptr<llvm::raw_fd_ostream,std::default_delete<llvm::raw_fd_ostream>>,const llvm::lto::AddFileFn&,llvm::StringRef,llvm::StringRef,size_t&>(std::unique_ptr<llvm::raw_fd_ostream,std::default_delete<llvm::raw_fd_ostream>> &&,const llvm::lto::AddFileFn &,llvm::StringRef &&,llvm::StringRef &&,size_t &)' with [ _Ty=llvm::lto::localCache::<lambda_24b0b9154e270ae79e71d53f81fee0dc>::()::CacheStream</div><div> ] C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\INCLUDE\memory(1628): note: or 'std::unique_ptr<llvm::lto::localCache::<lambda_24b0b9154e270ae79e71d53f81fee0dc>::()::CacheStream,std::default_delete<_Ty>> std::make_unique<llvm::lto::localCache::<lambda_24b0b9154e270ae79e71d53f81fee0dc>::()::CacheStream,std::unique_ptr<llvm::raw_fd_ostream,std::default_delete<llvm::raw_fd_ostream>>,const llvm::lto::AddFileFn&,llvm::StringRef,llvm::StringRef,size_t&>(std::unique_ptr<llvm::raw_fd_ostream,std::default_delete<llvm::raw_fd_ostream>> &&,const llvm::lto::AddFileFn &,llvm::StringRef &&,llvm::StringRef &&,size_t &)' [found using argument-dependent lookup] with [</div><div> _Ty=llvm::lto::localCache::<lambda_24b0b9154e270ae79e71d53f81fee0dc>::()::CacheStream ] D:\src\llvm\lib\LTO\Caching.cpp(96): note: while trying to match the argument list '(std::unique_ptr<llvm::raw_fd_ostream,std::default_delete<_Ty>>, const llvm::lto::AddFileFn, llvm::StringRef, llvm::StringRef, std::size_t)' with [ _Ty=llvm::raw_fd_ostream ] [748/3638] Building AMDGPUGenInstrInfo.inc...</div><div>ninja: build stopped: subcommand failed. </div></div></div><br><div class="gmail_quote"><div dir="ltr">On Fri, Sep 23, 2016 at 2:42 PM Peter Collingbourne via llvm-commits <<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: pcc<br class="gmail_msg">
Date: Fri Sep 23 16:33:43 2016<br class="gmail_msg">
New Revision: 282299<br class="gmail_msg">
<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=282299&view=rev" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project?rev=282299&view=rev</a><br class="gmail_msg">
Log:<br class="gmail_msg">
LTO: Simplify caching interface.<br class="gmail_msg">
<br class="gmail_msg">
The NativeObjectOutput class has a design problem: it mixes up the caching<br class="gmail_msg">
policy with the interface for output streams, which makes the client-side<br class="gmail_msg">
code hard to follow and would for example make it harder to replace the<br class="gmail_msg">
cache implementation in an arbitrary client.<br class="gmail_msg">
<br class="gmail_msg">
This change separates the two aspects by moving the caching policy<br class="gmail_msg">
to a separate field in Config, replacing NativeObjectOutput with a<br class="gmail_msg">
NativeObjectStream class which only deals with streams and does not need to<br class="gmail_msg">
be overridden by most clients and introducing an AddFile callback for adding<br class="gmail_msg">
files (e.g. from the cache) to the link.<br class="gmail_msg">
<br class="gmail_msg">
Differential Revision: <a href="https://reviews.llvm.org/D24622" rel="noreferrer" class="gmail_msg" target="_blank">https://reviews.llvm.org/D24622</a><br class="gmail_msg">
<br class="gmail_msg">
Modified:<br class="gmail_msg">
llvm/trunk/include/llvm/LTO/Caching.h<br class="gmail_msg">
llvm/trunk/include/llvm/LTO/Config.h<br class="gmail_msg">
llvm/trunk/include/llvm/LTO/LTO.h<br class="gmail_msg">
llvm/trunk/include/llvm/LTO/LTOBackend.h<br class="gmail_msg">
llvm/trunk/lib/LTO/Caching.cpp<br class="gmail_msg">
llvm/trunk/lib/LTO/LTO.cpp<br class="gmail_msg">
llvm/trunk/lib/LTO/LTOBackend.cpp<br class="gmail_msg">
llvm/trunk/tools/gold/gold-plugin.cpp<br class="gmail_msg">
llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/include/llvm/LTO/Caching.h<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/Caching.h?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/Caching.h?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/include/llvm/LTO/Caching.h (original)<br class="gmail_msg">
+++ llvm/trunk/include/llvm/LTO/Caching.h Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -7,92 +7,29 @@<br class="gmail_msg">
//<br class="gmail_msg">
//===----------------------------------------------------------------------===//<br class="gmail_msg">
//<br class="gmail_msg">
-// This file defines the lto::CacheObjectOutput data structure, which allows<br class="gmail_msg">
-// clients to add a filesystem cache to ThinLTO<br class="gmail_msg">
+// This file defines the localCache function, which allows clients to add a<br class="gmail_msg">
+// filesystem cache to ThinLTO.<br class="gmail_msg">
//<br class="gmail_msg">
//===----------------------------------------------------------------------===//<br class="gmail_msg">
<br class="gmail_msg">
#ifndef LLVM_LTO_CACHING_H<br class="gmail_msg">
#define LLVM_LTO_CACHING_H<br class="gmail_msg">
<br class="gmail_msg">
-#include "llvm/ADT/SmallString.h"<br class="gmail_msg">
-#include "llvm/LTO/Config.h"<br class="gmail_msg">
-#include "llvm/Support/MemoryBuffer.h"<br class="gmail_msg">
+#include "llvm/LTO/LTO.h"<br class="gmail_msg">
+#include <string><br class="gmail_msg">
<br class="gmail_msg">
namespace llvm {<br class="gmail_msg">
namespace lto {<br class="gmail_msg">
-/// Type for client-supplied callback when a buffer is loaded from the cache.<br class="gmail_msg">
-typedef std::function<void(std::string)> AddBufferFn;<br class="gmail_msg">
<br class="gmail_msg">
-/// Manage caching on the filesystem.<br class="gmail_msg">
+/// This type defines the callback to add a pre-existing native object file<br class="gmail_msg">
+/// (e.g. in a cache).<br class="gmail_msg">
///<br class="gmail_msg">
-/// The general scheme is the following:<br class="gmail_msg">
-///<br class="gmail_msg">
-/// void do_stuff(AddBufferFn CallBack) {<br class="gmail_msg">
-/// /* ... */<br class="gmail_msg">
-/// {<br class="gmail_msg">
-/// /* Create the CacheObjectOutput pointing to a cache directory */<br class="gmail_msg">
-/// auto Output = CacheObjectOutput("/tmp/cache", CallBack)<br class="gmail_msg">
-///<br class="gmail_msg">
-/// /* Call some processing function */<br class="gmail_msg">
-/// process(Output);<br class="gmail_msg">
-///<br class="gmail_msg">
-/// } /* Callback is only called now, on destruction of the Output object */<br class="gmail_msg">
-/// /* ... */<br class="gmail_msg">
-/// }<br class="gmail_msg">
-///<br class="gmail_msg">
-///<br class="gmail_msg">
-/// void process(NativeObjectOutput &Output) {<br class="gmail_msg">
-/// /* check if caching is supported */<br class="gmail_msg">
-/// if (Output.isCachingEnabled()) {<br class="gmail_msg">
-/// auto Key = ComputeKeyForEntry(...); // "expensive" call<br class="gmail_msg">
-/// if (Output.tryLoadFromCache())<br class="gmail_msg">
-/// return; // Cache hit<br class="gmail_msg">
-/// }<br class="gmail_msg">
-///<br class="gmail_msg">
-/// auto OS = Output.getStream();<br class="gmail_msg">
-///<br class="gmail_msg">
-/// OS << ...;<br class="gmail_msg">
-/// /* Note that the callback is not called here, but only when the caller<br class="gmail_msg">
-/// destroys Output */<br class="gmail_msg">
-/// }<br class="gmail_msg">
-///<br class="gmail_msg">
-class CacheObjectOutput : public NativeObjectOutput {<br class="gmail_msg">
- /// Path to the on-disk cache directory<br class="gmail_msg">
- StringRef CacheDirectoryPath;<br class="gmail_msg">
- /// Path to this entry in the cache, initialized by tryLoadFromCache().<br class="gmail_msg">
- SmallString<128> EntryPath;<br class="gmail_msg">
- /// Path to temporary file used to buffer output that will be committed to the<br class="gmail_msg">
- /// cache entry when this object is destroyed<br class="gmail_msg">
- SmallString<128> TempFilename;<br class="gmail_msg">
- /// User-supplied callback, used to provide path to cache entry<br class="gmail_msg">
- /// (potentially after creating it).<br class="gmail_msg">
- AddBufferFn AddBuffer;<br class="gmail_msg">
-<br class="gmail_msg">
-public:<br class="gmail_msg">
- /// The destructor pulls the entry from the cache and calls the AddBuffer<br class="gmail_msg">
- /// callback, after committing the entry into the cache on miss.<br class="gmail_msg">
- ~CacheObjectOutput();<br class="gmail_msg">
-<br class="gmail_msg">
- /// Create a CacheObjectOutput: the client is supposed to create it in the<br class="gmail_msg">
- /// callback supplied to LTO::run. The \p CacheDirectoryPath points to the<br class="gmail_msg">
- /// directory on disk where to store the cache, and \p AddBuffer will be<br class="gmail_msg">
- /// called when the buffer is ready to be pulled out of the cache<br class="gmail_msg">
- /// (potentially after creating it).<br class="gmail_msg">
- CacheObjectOutput(StringRef CacheDirectoryPath, AddBufferFn AddBuffer)<br class="gmail_msg">
- : CacheDirectoryPath(CacheDirectoryPath), AddBuffer(AddBuffer) {}<br class="gmail_msg">
-<br class="gmail_msg">
- /// Return an allocated stream for the output, or null in case of failure.<br class="gmail_msg">
- std::unique_ptr<raw_pwrite_stream> getStream() override;<br class="gmail_msg">
-<br class="gmail_msg">
- /// Set EntryPath, try loading from a possible cache first, return true on<br class="gmail_msg">
- /// cache hit.<br class="gmail_msg">
- bool tryLoadFromCache(StringRef Key) override;<br class="gmail_msg">
-<br class="gmail_msg">
- /// Returns true to signal that this implementation of NativeObjectFile<br class="gmail_msg">
- /// support caching.<br class="gmail_msg">
- bool isCachingEnabled() const override { return true; }<br class="gmail_msg">
-};<br class="gmail_msg">
+/// File callbacks must be thread safe.<br class="gmail_msg">
+typedef std::function<void(unsigned Task, StringRef Path)> AddFileFn;<br class="gmail_msg">
+<br class="gmail_msg">
+/// Create a local file system cache which uses the given cache directory and<br class="gmail_msg">
+/// file callback.<br class="gmail_msg">
+NativeObjectCache localCache(std::string CacheDirectoryPath, AddFileFn AddFile);<br class="gmail_msg">
<br class="gmail_msg">
} // namespace lto<br class="gmail_msg">
} // namespace llvm<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/include/llvm/LTO/Config.h<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/Config.h?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/Config.h?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/include/llvm/LTO/Config.h (original)<br class="gmail_msg">
+++ llvm/trunk/include/llvm/LTO/Config.h Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -30,38 +30,6 @@ class raw_pwrite_stream;<br class="gmail_msg">
<br class="gmail_msg">
namespace lto {<br class="gmail_msg">
<br class="gmail_msg">
-/// Abstract class representing a single Task output to be implemented by the<br class="gmail_msg">
-/// client of the LTO API.<br class="gmail_msg">
-///<br class="gmail_msg">
-/// The general scheme the API is called is the following:<br class="gmail_msg">
-///<br class="gmail_msg">
-/// void process(NativeObjectOutput &Output) {<br class="gmail_msg">
-/// /* check if caching is supported */<br class="gmail_msg">
-/// if (Output.isCachingEnabled()) {<br class="gmail_msg">
-/// auto Key = ComputeKeyForEntry(...); // "expensive" call<br class="gmail_msg">
-/// if (Output.tryLoadFromCache())<br class="gmail_msg">
-/// return; // Cache hit<br class="gmail_msg">
-/// }<br class="gmail_msg">
-///<br class="gmail_msg">
-/// auto OS = Output.getStream();<br class="gmail_msg">
-///<br class="gmail_msg">
-/// OS << ....;<br class="gmail_msg">
-/// }<br class="gmail_msg">
-///<br class="gmail_msg">
-class NativeObjectOutput {<br class="gmail_msg">
-public:<br class="gmail_msg">
- // Return an allocated stream for the output, or null in case of failure.<br class="gmail_msg">
- virtual std::unique_ptr<raw_pwrite_stream> getStream() = 0;<br class="gmail_msg">
-<br class="gmail_msg">
- // Try loading from a possible cache first, return true on cache hit.<br class="gmail_msg">
- virtual bool tryLoadFromCache(StringRef Key) { return false; }<br class="gmail_msg">
-<br class="gmail_msg">
- // Returns true if a cache is available<br class="gmail_msg">
- virtual bool isCachingEnabled() const { return false; }<br class="gmail_msg">
-<br class="gmail_msg">
- virtual ~NativeObjectOutput() = default;<br class="gmail_msg">
-};<br class="gmail_msg">
-<br class="gmail_msg">
/// LTO configuration. A linker can configure LTO by setting fields in this data<br class="gmail_msg">
/// structure and passing it to the lto::LTO constructor.<br class="gmail_msg">
struct Config {<br class="gmail_msg">
@@ -235,13 +203,6 @@ struct Config {<br class="gmail_msg">
bool UseInputModulePath = false);<br class="gmail_msg">
};<br class="gmail_msg">
<br class="gmail_msg">
-/// This type defines the callback to add a native object that is generated on<br class="gmail_msg">
-/// the fly.<br class="gmail_msg">
-///<br class="gmail_msg">
-/// Output callbacks must be thread safe.<br class="gmail_msg">
-typedef std::function<std::unique_ptr<NativeObjectOutput>(unsigned Task)><br class="gmail_msg">
- AddOutputFn;<br class="gmail_msg">
-<br class="gmail_msg">
/// A derived class of LLVMContext that initializes itself according to a given<br class="gmail_msg">
/// Config object. The purpose of this class is to tie ownership of the<br class="gmail_msg">
/// diagnostic handler to the context, as opposed to the Config object (which<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/include/llvm/LTO/LTO.h<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/LTO.h?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/LTO.h?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/include/llvm/LTO/LTO.h (original)<br class="gmail_msg">
+++ llvm/trunk/include/llvm/LTO/LTO.h Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -247,13 +247,47 @@ public:<br class="gmail_msg">
}<br class="gmail_msg">
};<br class="gmail_msg">
<br class="gmail_msg">
+/// This class wraps an output stream for a native object. Most clients should<br class="gmail_msg">
+/// just be able to return an instance of this base class from the stream<br class="gmail_msg">
+/// callback, but if a client needs to perform some action after the stream is<br class="gmail_msg">
+/// written to, that can be done by deriving from this class and overriding the<br class="gmail_msg">
+/// destructor.<br class="gmail_msg">
+class NativeObjectStream {<br class="gmail_msg">
+public:<br class="gmail_msg">
+ NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS) : OS(std::move(OS)) {}<br class="gmail_msg">
+ std::unique_ptr<raw_pwrite_stream> OS;<br class="gmail_msg">
+ virtual ~NativeObjectStream() = default;<br class="gmail_msg">
+};<br class="gmail_msg">
+<br class="gmail_msg">
+/// This type defines the callback to add a native object that is generated on<br class="gmail_msg">
+/// the fly.<br class="gmail_msg">
+///<br class="gmail_msg">
+/// Stream callbacks must be thread safe.<br class="gmail_msg">
+typedef std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)><br class="gmail_msg">
+ AddStreamFn;<br class="gmail_msg">
+<br class="gmail_msg">
+/// This is the type of a native object cache. To request an item from the<br class="gmail_msg">
+/// cache, pass a unique string as the Key. For hits, the cached file will be<br class="gmail_msg">
+/// added to the link and this function will return AddStreamFn(). For misses,<br class="gmail_msg">
+/// the cache will return a stream callback which must be called at most once to<br class="gmail_msg">
+/// produce content for the stream. The native object stream produced by the<br class="gmail_msg">
+/// stream callback will add the file to the link after the stream is written<br class="gmail_msg">
+/// to.<br class="gmail_msg">
+///<br class="gmail_msg">
+/// Clients generally look like this:<br class="gmail_msg">
+///<br class="gmail_msg">
+/// if (AddStreamFn AddStream = Cache(Task, Key))<br class="gmail_msg">
+/// ProduceContent(AddStream);<br class="gmail_msg">
+typedef std::function<AddStreamFn(unsigned Task, StringRef Key)><br class="gmail_msg">
+ NativeObjectCache;<br class="gmail_msg">
+<br class="gmail_msg">
/// A ThinBackend defines what happens after the thin-link phase during ThinLTO.<br class="gmail_msg">
/// The details of this type definition aren't important; clients can only<br class="gmail_msg">
/// create a ThinBackend using one of the create*ThinBackend() functions below.<br class="gmail_msg">
typedef std::function<std::unique_ptr<ThinBackendProc>(<br class="gmail_msg">
Config &C, ModuleSummaryIndex &CombinedIndex,<br class="gmail_msg">
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,<br class="gmail_msg">
- AddOutputFn AddOutput)><br class="gmail_msg">
+ AddStreamFn AddStream, NativeObjectCache Cache)><br class="gmail_msg">
ThinBackend;<br class="gmail_msg">
<br class="gmail_msg">
/// This ThinBackend runs the individual backend jobs in-process.<br class="gmail_msg">
@@ -286,8 +320,9 @@ ThinBackend createWriteIndexesThinBacken<br class="gmail_msg">
/// and pass it and an array of symbol resolutions to the add() function.<br class="gmail_msg">
/// - Call the getMaxTasks() function to get an upper bound on the number of<br class="gmail_msg">
/// native object files that LTO may add to the link.<br class="gmail_msg">
-/// - Call the run() function. This function will use the supplied AddOutput<br class="gmail_msg">
-/// function to add up to getMaxTasks() native object files to the link.<br class="gmail_msg">
+/// - Call the run() function. This function will use the supplied AddStream<br class="gmail_msg">
+/// and Cache functions to add up to getMaxTasks() native object files to<br class="gmail_msg">
+/// the link.<br class="gmail_msg">
class LTO {<br class="gmail_msg">
friend InputFile;<br class="gmail_msg">
<br class="gmail_msg">
@@ -310,9 +345,15 @@ public:<br class="gmail_msg">
/// full description of tasks see LTOBackend.h.<br class="gmail_msg">
unsigned getMaxTasks() const;<br class="gmail_msg">
<br class="gmail_msg">
- /// Runs the LTO pipeline. This function calls the supplied AddOutput function<br class="gmail_msg">
- /// to add native object files to the link.<br class="gmail_msg">
- Error run(AddOutputFn AddOutput);<br class="gmail_msg">
+ /// Runs the LTO pipeline. This function calls the supplied AddStream<br class="gmail_msg">
+ /// function to add native object files to the link.<br class="gmail_msg">
+ ///<br class="gmail_msg">
+ /// The Cache parameter is optional. If supplied, it will be used to cache<br class="gmail_msg">
+ /// native object files and add them to the link.<br class="gmail_msg">
+ ///<br class="gmail_msg">
+ /// The client will receive at most one callback (via either AddStream or<br class="gmail_msg">
+ /// Cache) for each task identifier.<br class="gmail_msg">
+ Error run(AddStreamFn AddStream, NativeObjectCache Cache = nullptr);<br class="gmail_msg">
<br class="gmail_msg">
private:<br class="gmail_msg">
Config Conf;<br class="gmail_msg">
@@ -393,8 +434,9 @@ private:<br class="gmail_msg">
Error addThinLTO(std::unique_ptr<InputFile> Input,<br class="gmail_msg">
ArrayRef<SymbolResolution> Res);<br class="gmail_msg">
<br class="gmail_msg">
- Error runRegularLTO(AddOutputFn AddOutput);<br class="gmail_msg">
- Error runThinLTO(AddOutputFn AddOutput, bool HasRegularLTO);<br class="gmail_msg">
+ Error runRegularLTO(AddStreamFn AddStream);<br class="gmail_msg">
+ Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,<br class="gmail_msg">
+ bool HasRegularLTO);<br class="gmail_msg">
<br class="gmail_msg">
mutable bool CalledGetMaxTasks = false;<br class="gmail_msg">
};<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/include/llvm/LTO/LTOBackend.h<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/LTOBackend.h?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/LTO/LTOBackend.h?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/include/llvm/LTO/LTOBackend.h (original)<br class="gmail_msg">
+++ llvm/trunk/include/llvm/LTO/LTOBackend.h Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -20,7 +20,7 @@<br class="gmail_msg">
#include "llvm/ADT/MapVector.h"<br class="gmail_msg">
#include "llvm/IR/DiagnosticInfo.h"<br class="gmail_msg">
#include "llvm/IR/ModuleSummaryIndex.h"<br class="gmail_msg">
-#include "llvm/LTO/Config.h"<br class="gmail_msg">
+#include "llvm/LTO/LTO.h"<br class="gmail_msg">
#include "llvm/Support/MemoryBuffer.h"<br class="gmail_msg">
#include "llvm/Target/TargetOptions.h"<br class="gmail_msg">
#include "llvm/Transforms/IPO/FunctionImport.h"<br class="gmail_msg">
@@ -34,12 +34,12 @@ class Target;<br class="gmail_msg">
namespace lto {<br class="gmail_msg">
<br class="gmail_msg">
/// Runs a regular LTO backend.<br class="gmail_msg">
-Error backend(Config &C, AddOutputFn AddStream,<br class="gmail_msg">
+Error backend(Config &C, AddStreamFn AddStream,<br class="gmail_msg">
unsigned ParallelCodeGenParallelismLevel,<br class="gmail_msg">
std::unique_ptr<Module> M);<br class="gmail_msg">
<br class="gmail_msg">
/// Runs a ThinLTO backend.<br class="gmail_msg">
-Error thinBackend(Config &C, unsigned Task, AddOutputFn AddStream, Module &M,<br class="gmail_msg">
+Error thinBackend(Config &C, unsigned Task, AddStreamFn AddStream, Module &M,<br class="gmail_msg">
ModuleSummaryIndex &CombinedIndex,<br class="gmail_msg">
const FunctionImporter::ImportMapTy &ImportList,<br class="gmail_msg">
const GVSummaryMapTy &DefinedGlobals,<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/lib/LTO/Caching.cpp<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/Caching.cpp?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/Caching.cpp?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/lib/LTO/Caching.cpp (original)<br class="gmail_msg">
+++ llvm/trunk/lib/LTO/Caching.cpp Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -12,13 +12,9 @@<br class="gmail_msg">
//===----------------------------------------------------------------------===//<br class="gmail_msg">
<br class="gmail_msg">
#include "llvm/LTO/Caching.h"<br class="gmail_msg">
-<br class="gmail_msg">
-#ifdef HAVE_LLVM_REVISION<br class="gmail_msg">
-#include "LLVMLTORevision.h"<br class="gmail_msg">
-#endif<br class="gmail_msg">
-<br class="gmail_msg">
#include "llvm/ADT/StringExtras.h"<br class="gmail_msg">
#include "llvm/Support/FileSystem.h"<br class="gmail_msg">
+#include "llvm/Support/MemoryBuffer.h"<br class="gmail_msg">
#include "llvm/Support/Path.h"<br class="gmail_msg">
#include "llvm/Support/raw_ostream.h"<br class="gmail_msg">
<br class="gmail_msg">
@@ -30,6 +26,8 @@ static void commitEntry(StringRef TempFi<br class="gmail_msg">
auto EC = sys::fs::rename(TempFilename, EntryPath);<br class="gmail_msg">
if (EC) {<br class="gmail_msg">
// Renaming failed, probably not the same filesystem, copy and delete.<br class="gmail_msg">
+ // FIXME: Avoid needing to do this by creating the temporary file in the<br class="gmail_msg">
+ // cache directory.<br class="gmail_msg">
{<br class="gmail_msg">
auto ReloadedBufferOrErr = MemoryBuffer::getFile(TempFilename);<br class="gmail_msg">
if (auto EC = ReloadedBufferOrErr.getError())<br class="gmail_msg">
@@ -48,51 +46,54 @@ static void commitEntry(StringRef TempFi<br class="gmail_msg">
}<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-CacheObjectOutput::~CacheObjectOutput() {<br class="gmail_msg">
- if (EntryPath.empty())<br class="gmail_msg">
- // The entry was never used by the client (tryLoadFromCache() wasn't called)<br class="gmail_msg">
- return;<br class="gmail_msg">
- // TempFilename is only set if getStream() was called, i.e. on cache miss when<br class="gmail_msg">
- // tryLoadFromCache() returned false. And EntryPath is valid if a Key was<br class="gmail_msg">
- // submitted, otherwise it has been set to CacheDirectoryPath in<br class="gmail_msg">
- // tryLoadFromCache.<br class="gmail_msg">
- if (!TempFilename.empty()) {<br class="gmail_msg">
- if (EntryPath == CacheDirectoryPath)<br class="gmail_msg">
- // The Key supplied to tryLoadFromCache was empty, do not commit the temp.<br class="gmail_msg">
- EntryPath = TempFilename;<br class="gmail_msg">
- else<br class="gmail_msg">
- // We commit the tempfile into the cache now, by moving it to EntryPath.<br class="gmail_msg">
- commitEntry(TempFilename, EntryPath);<br class="gmail_msg">
- }<br class="gmail_msg">
- // Supply the cache path to the user.<br class="gmail_msg">
- AddBuffer(EntryPath.str());<br class="gmail_msg">
-}<br class="gmail_msg">
-<br class="gmail_msg">
-// Return an allocated stream for the output, or null in case of failure.<br class="gmail_msg">
-std::unique_ptr<raw_pwrite_stream> CacheObjectOutput::getStream() {<br class="gmail_msg">
- assert(!EntryPath.empty() && "API Violation: client didn't call "<br class="gmail_msg">
- "tryLoadFromCache() before getStream()");<br class="gmail_msg">
- // Write to a temporary to avoid race condition<br class="gmail_msg">
- int TempFD;<br class="gmail_msg">
- std::error_code EC =<br class="gmail_msg">
- sys::fs::createTemporaryFile("Thin", "tmp.o", TempFD, TempFilename);<br class="gmail_msg">
- if (EC) {<br class="gmail_msg">
- errs() << "Error: " << EC.message() << "\n";<br class="gmail_msg">
- report_fatal_error("ThinLTO: Can't get a temporary file");<br class="gmail_msg">
- }<br class="gmail_msg">
- return llvm::make_unique<raw_fd_ostream>(TempFD, /* ShouldClose */ true);<br class="gmail_msg">
-}<br class="gmail_msg">
+NativeObjectCache lto::localCache(std::string CacheDirectoryPath,<br class="gmail_msg">
+ AddFileFn AddFile) {<br class="gmail_msg">
+ return [=](unsigned Task, StringRef Key) -> AddStreamFn {<br class="gmail_msg">
+ // First, see if we have a cache hit.<br class="gmail_msg">
+ SmallString<64> EntryPath;<br class="gmail_msg">
+ sys::path::append(EntryPath, CacheDirectoryPath, Key);<br class="gmail_msg">
+ if (sys::fs::exists(EntryPath)) {<br class="gmail_msg">
+ AddFile(Task, EntryPath);<br class="gmail_msg">
+ return AddStreamFn();<br class="gmail_msg">
+ }<br class="gmail_msg">
<br class="gmail_msg">
-// Try loading from a possible cache first, return true on cache hit.<br class="gmail_msg">
-bool CacheObjectOutput::tryLoadFromCache(StringRef Key) {<br class="gmail_msg">
- assert(!CacheDirectoryPath.empty() &&<br class="gmail_msg">
- "CacheObjectOutput was initialized without a cache path");<br class="gmail_msg">
- if (Key.empty()) {<br class="gmail_msg">
- // Client didn't compute a valid key. EntryPath has been set to<br class="gmail_msg">
- // CacheDirectoryPath.<br class="gmail_msg">
- EntryPath = CacheDirectoryPath;<br class="gmail_msg">
- return false;<br class="gmail_msg">
- }<br class="gmail_msg">
- sys::path::append(EntryPath, CacheDirectoryPath, Key);<br class="gmail_msg">
- return sys::fs::exists(EntryPath);<br class="gmail_msg">
+ // This native object stream is responsible for commiting the resulting<br class="gmail_msg">
+ // file to the cache and calling AddFile to add it to the link.<br class="gmail_msg">
+ struct CacheStream : NativeObjectStream {<br class="gmail_msg">
+ AddFileFn AddFile;<br class="gmail_msg">
+ std::string TempFilename;<br class="gmail_msg">
+ std::string EntryPath;<br class="gmail_msg">
+ unsigned Task;<br class="gmail_msg">
+<br class="gmail_msg">
+ CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddFileFn AddFile,<br class="gmail_msg">
+ std::string TempFilename, std::string EntryPath,<br class="gmail_msg">
+ unsigned Task)<br class="gmail_msg">
+ : NativeObjectStream(std::move(OS)), AddFile(AddFile),<br class="gmail_msg">
+ TempFilename(TempFilename), EntryPath(EntryPath), Task(Task) {}<br class="gmail_msg">
+<br class="gmail_msg">
+ ~CacheStream() {<br class="gmail_msg">
+ // Make sure the file is closed before committing it.<br class="gmail_msg">
+ OS.reset();<br class="gmail_msg">
+ commitEntry(TempFilename, EntryPath);<br class="gmail_msg">
+ AddFile(Task, EntryPath);<br class="gmail_msg">
+ }<br class="gmail_msg">
+ };<br class="gmail_msg">
+<br class="gmail_msg">
+ return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> {<br class="gmail_msg">
+ // Write to a temporary to avoid race condition<br class="gmail_msg">
+ int TempFD;<br class="gmail_msg">
+ SmallString<64> TempFilename;<br class="gmail_msg">
+ std::error_code EC =<br class="gmail_msg">
+ sys::fs::createTemporaryFile("Thin", "tmp.o", TempFD, TempFilename);<br class="gmail_msg">
+ if (EC) {<br class="gmail_msg">
+ errs() << "Error: " << EC.message() << "\n";<br class="gmail_msg">
+ report_fatal_error("ThinLTO: Can't get a temporary file");<br class="gmail_msg">
+ }<br class="gmail_msg">
+<br class="gmail_msg">
+ // This CacheStream will move the temporary file into the cache when done.<br class="gmail_msg">
+ return make_unique<CacheStream>(<br class="gmail_msg">
+ llvm::make_unique<raw_fd_ostream>(TempFD, /* ShouldClose */ true),<br class="gmail_msg">
+ AddFile, TempFilename.str(), EntryPath.str(), Task);<br class="gmail_msg">
+ };<br class="gmail_msg">
+ };<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/lib/LTO/LTO.cpp<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/LTO.cpp?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/LTO.cpp?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/lib/LTO/LTO.cpp (original)<br class="gmail_msg">
+++ llvm/trunk/lib/LTO/LTO.cpp Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -409,19 +409,19 @@ unsigned LTO::getMaxTasks() const {<br class="gmail_msg">
return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size();<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-Error LTO::run(AddOutputFn AddOutput) {<br class="gmail_msg">
+Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {<br class="gmail_msg">
// Save the status of having a regularLTO combined module, as<br class="gmail_msg">
// this is needed for generating the ThinLTO Task ID, and<br class="gmail_msg">
// the CombinedModule will be moved at the end of runRegularLTO.<br class="gmail_msg">
bool HasRegularLTO = RegularLTO.CombinedModule != nullptr;<br class="gmail_msg">
// Invoke regular LTO if there was a regular LTO module to start with.<br class="gmail_msg">
if (HasRegularLTO)<br class="gmail_msg">
- if (auto E = runRegularLTO(AddOutput))<br class="gmail_msg">
+ if (auto E = runRegularLTO(AddStream))<br class="gmail_msg">
return E;<br class="gmail_msg">
- return runThinLTO(AddOutput, HasRegularLTO);<br class="gmail_msg">
+ return runThinLTO(AddStream, Cache, HasRegularLTO);<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-Error LTO::runRegularLTO(AddOutputFn AddOutput) {<br class="gmail_msg">
+Error LTO::runRegularLTO(AddStreamFn AddStream) {<br class="gmail_msg">
// Make sure commons have the right size/alignment: we kept the largest from<br class="gmail_msg">
// all the prevailing when adding the inputs, and we apply it here.<br class="gmail_msg">
const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();<br class="gmail_msg">
@@ -478,7 +478,7 @@ Error LTO::runRegularLTO(AddOutputFn Add<br class="gmail_msg">
!Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))<br class="gmail_msg">
return Error();<br class="gmail_msg">
}<br class="gmail_msg">
- return backend(Conf, AddOutput, RegularLTO.ParallelCodeGenParallelismLevel,<br class="gmail_msg">
+ return backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,<br class="gmail_msg">
std::move(RegularLTO.CombinedModule));<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
@@ -507,7 +507,8 @@ public:<br class="gmail_msg">
<br class="gmail_msg">
class InProcessThinBackend : public ThinBackendProc {<br class="gmail_msg">
ThreadPool BackendThreadPool;<br class="gmail_msg">
- AddOutputFn AddOutput;<br class="gmail_msg">
+ AddStreamFn AddStream;<br class="gmail_msg">
+ NativeObjectCache Cache;<br class="gmail_msg">
<br class="gmail_msg">
Optional<Error> Err;<br class="gmail_msg">
std::mutex ErrMu;<br class="gmail_msg">
@@ -517,42 +518,40 @@ public:<br class="gmail_msg">
Config &Conf, ModuleSummaryIndex &CombinedIndex,<br class="gmail_msg">
unsigned ThinLTOParallelismLevel,<br class="gmail_msg">
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,<br class="gmail_msg">
- AddOutputFn AddOutput)<br class="gmail_msg">
+ AddStreamFn AddStream, NativeObjectCache Cache)<br class="gmail_msg">
: ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),<br class="gmail_msg">
BackendThreadPool(ThinLTOParallelismLevel),<br class="gmail_msg">
- AddOutput(std::move(AddOutput)) {}<br class="gmail_msg">
+ AddStream(std::move(AddStream)), Cache(std::move(Cache)) {}<br class="gmail_msg">
<br class="gmail_msg">
Error runThinLTOBackendThread(<br class="gmail_msg">
- AddOutputFn AddOutput, unsigned Task, MemoryBufferRef MBRef,<br class="gmail_msg">
- ModuleSummaryIndex &CombinedIndex,<br class="gmail_msg">
+ AddStreamFn AddStream, NativeObjectCache Cache, unsigned Task,<br class="gmail_msg">
+ MemoryBufferRef MBRef, ModuleSummaryIndex &CombinedIndex,<br class="gmail_msg">
const FunctionImporter::ImportMapTy &ImportList,<br class="gmail_msg">
const FunctionImporter::ExportSetTy &ExportList,<br class="gmail_msg">
const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,<br class="gmail_msg">
const GVSummaryMapTy &DefinedGlobals,<br class="gmail_msg">
MapVector<StringRef, MemoryBufferRef> &ModuleMap) {<br class="gmail_msg">
+ auto RunThinBackend = [&](AddStreamFn AddStream) {<br class="gmail_msg">
+ LTOLLVMContext BackendContext(Conf);<br class="gmail_msg">
+ ErrorOr<std::unique_ptr<Module>> MOrErr =<br class="gmail_msg">
+ parseBitcodeFile(MBRef, BackendContext);<br class="gmail_msg">
+ assert(MOrErr && "Unable to load module in thread?");<br class="gmail_msg">
<br class="gmail_msg">
- auto ModuleIdentifier = MBRef.getBufferIdentifier();<br class="gmail_msg">
- auto Output = AddOutput(Task);<br class="gmail_msg">
- if (Output->isCachingEnabled()) {<br class="gmail_msg">
- SmallString<40> Key;<br class="gmail_msg">
- // The module may be cached, this helps handling it.<br class="gmail_msg">
- computeCacheKey(Key, CombinedIndex, ModuleIdentifier, ImportList,<br class="gmail_msg">
- ExportList, ResolvedODR, DefinedGlobals);<br class="gmail_msg">
- if (Output->tryLoadFromCache(Key))<br class="gmail_msg">
- return Error();<br class="gmail_msg">
- }<br class="gmail_msg">
-<br class="gmail_msg">
- LTOLLVMContext BackendContext(Conf);<br class="gmail_msg">
- ErrorOr<std::unique_ptr<Module>> MOrErr =<br class="gmail_msg">
- parseBitcodeFile(MBRef, BackendContext);<br class="gmail_msg">
- assert(MOrErr && "Unable to load module in thread?");<br class="gmail_msg">
-<br class="gmail_msg">
- auto AddOutputWrapper = [&](unsigned TaskId) {<br class="gmail_msg">
- assert(Task == TaskId && "Unexpexted TaskId mismatch");<br class="gmail_msg">
- return std::move(Output);<br class="gmail_msg">
+ return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,<br class="gmail_msg">
+ ImportList, DefinedGlobals, ModuleMap);<br class="gmail_msg">
};<br class="gmail_msg">
- return thinBackend(Conf, Task, AddOutputWrapper, **MOrErr, CombinedIndex,<br class="gmail_msg">
- ImportList, DefinedGlobals, ModuleMap);<br class="gmail_msg">
+<br class="gmail_msg">
+ if (!Cache)<br class="gmail_msg">
+ return RunThinBackend(AddStream);<br class="gmail_msg">
+<br class="gmail_msg">
+ SmallString<40> Key;<br class="gmail_msg">
+ // The module may be cached, this helps handling it.<br class="gmail_msg">
+ computeCacheKey(Key, CombinedIndex, MBRef.getBufferIdentifier(),<br class="gmail_msg">
+ ImportList, ExportList, ResolvedODR, DefinedGlobals);<br class="gmail_msg">
+ if (AddStreamFn CacheAddStream = Cache(Task, Key))<br class="gmail_msg">
+ return RunThinBackend(CacheAddStream);<br class="gmail_msg">
+<br class="gmail_msg">
+ return Error();<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
Error start(<br class="gmail_msg">
@@ -574,8 +573,8 @@ public:<br class="gmail_msg">
const GVSummaryMapTy &DefinedGlobals,<br class="gmail_msg">
MapVector<StringRef, MemoryBufferRef> &ModuleMap) {<br class="gmail_msg">
Error E = runThinLTOBackendThread(<br class="gmail_msg">
- AddOutput, Task, MBRef, CombinedIndex, ImportList, ExportList,<br class="gmail_msg">
- ResolvedODR, DefinedGlobals, ModuleMap);<br class="gmail_msg">
+ AddStream, Cache, Task, MBRef, CombinedIndex, ImportList,<br class="gmail_msg">
+ ExportList, ResolvedODR, DefinedGlobals, ModuleMap);<br class="gmail_msg">
if (E) {<br class="gmail_msg">
std::unique_lock<std::mutex> L(ErrMu);<br class="gmail_msg">
if (Err)<br class="gmail_msg">
@@ -602,10 +601,10 @@ public:<br class="gmail_msg">
ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) {<br class="gmail_msg">
return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,<br class="gmail_msg">
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,<br class="gmail_msg">
- AddOutputFn AddOutput) {<br class="gmail_msg">
+ AddStreamFn AddStream, NativeObjectCache Cache) {<br class="gmail_msg">
return llvm::make_unique<InProcessThinBackend>(<br class="gmail_msg">
Conf, CombinedIndex, ParallelismLevel, ModuleToDefinedGVSummaries,<br class="gmail_msg">
- AddOutput);<br class="gmail_msg">
+ AddStream, Cache);<br class="gmail_msg">
};<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
@@ -693,14 +692,15 @@ ThinBackend lto::createWriteIndexesThinB<br class="gmail_msg">
std::string LinkedObjectsFile) {<br class="gmail_msg">
return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,<br class="gmail_msg">
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,<br class="gmail_msg">
- AddOutputFn AddOutput) {<br class="gmail_msg">
+ AddStreamFn AddStream, NativeObjectCache Cache) {<br class="gmail_msg">
return llvm::make_unique<WriteIndexesThinBackend>(<br class="gmail_msg">
Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,<br class="gmail_msg">
ShouldEmitImportsFiles, LinkedObjectsFile);<br class="gmail_msg">
};<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-Error LTO::runThinLTO(AddOutputFn AddOutput, bool HasRegularLTO) {<br class="gmail_msg">
+Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,<br class="gmail_msg">
+ bool HasRegularLTO) {<br class="gmail_msg">
if (ThinLTO.ModuleMap.empty())<br class="gmail_msg">
return Error();<br class="gmail_msg">
<br class="gmail_msg">
@@ -759,8 +759,9 @@ Error LTO::runThinLTO(AddOutputFn AddOut<br class="gmail_msg">
thinLTOResolveWeakForLinkerInIndex(ThinLTO.CombinedIndex, isPrevailing,<br class="gmail_msg">
recordNewLinkage);<br class="gmail_msg">
<br class="gmail_msg">
- std::unique_ptr<ThinBackendProc> BackendProc = ThinLTO.Backend(<br class="gmail_msg">
- Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, AddOutput);<br class="gmail_msg">
+ std::unique_ptr<ThinBackendProc> BackendProc =<br class="gmail_msg">
+ ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,<br class="gmail_msg">
+ AddStream, Cache);<br class="gmail_msg">
<br class="gmail_msg">
// Partition numbers for ThinLTO jobs start at 1 (see comments for<br class="gmail_msg">
// GlobalResolution in LTO.h). Task numbers, however, start at<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/lib/LTO/LTOBackend.cpp<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/LTOBackend.cpp?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/LTOBackend.cpp?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/lib/LTO/LTOBackend.cpp (original)<br class="gmail_msg">
+++ llvm/trunk/lib/LTO/LTOBackend.cpp Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -199,34 +199,20 @@ bool opt(Config &Conf, TargetMachine *TM<br class="gmail_msg">
return !Conf.PostOptModuleHook || Conf.PostOptModuleHook(Task, Mod);<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-/// Monolithic LTO does not support caching (yet), this is a convenient wrapper<br class="gmail_msg">
-/// around AddOutput to workaround this.<br class="gmail_msg">
-static AddOutputFn getUncachedOutputWrapper(AddOutputFn &AddOutput,<br class="gmail_msg">
- unsigned Task) {<br class="gmail_msg">
- return [Task, &AddOutput](unsigned TaskId) {<br class="gmail_msg">
- auto Output = AddOutput(Task);<br class="gmail_msg">
- if (Output->isCachingEnabled() && Output->tryLoadFromCache(""))<br class="gmail_msg">
- report_fatal_error("Cache hit without a valid key?");<br class="gmail_msg">
- assert(Task == TaskId && "Unexpexted TaskId mismatch");<br class="gmail_msg">
- return Output;<br class="gmail_msg">
- };<br class="gmail_msg">
-}<br class="gmail_msg">
-<br class="gmail_msg">
-void codegen(Config &Conf, TargetMachine *TM, AddOutputFn AddOutput,<br class="gmail_msg">
+void codegen(Config &Conf, TargetMachine *TM, AddStreamFn AddStream,<br class="gmail_msg">
unsigned Task, Module &Mod) {<br class="gmail_msg">
if (Conf.PreCodeGenModuleHook && !Conf.PreCodeGenModuleHook(Task, Mod))<br class="gmail_msg">
return;<br class="gmail_msg">
<br class="gmail_msg">
- auto Output = AddOutput(Task);<br class="gmail_msg">
- std::unique_ptr<raw_pwrite_stream> OS = Output->getStream();<br class="gmail_msg">
+ auto Stream = AddStream(Task);<br class="gmail_msg">
legacy::PassManager CodeGenPasses;<br class="gmail_msg">
- if (TM->addPassesToEmitFile(CodeGenPasses, *OS,<br class="gmail_msg">
+ if (TM->addPassesToEmitFile(CodeGenPasses, *Stream->OS,<br class="gmail_msg">
TargetMachine::CGFT_ObjectFile))<br class="gmail_msg">
report_fatal_error("Failed to setup codegen");<br class="gmail_msg">
CodeGenPasses.run(Mod);<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-void splitCodeGen(Config &C, TargetMachine *TM, AddOutputFn AddOutput,<br class="gmail_msg">
+void splitCodeGen(Config &C, TargetMachine *TM, AddStreamFn AddStream,<br class="gmail_msg">
unsigned ParallelCodeGenParallelismLevel,<br class="gmail_msg">
std::unique_ptr<Module> Mod) {<br class="gmail_msg">
ThreadPool CodegenThreadPool(ParallelCodeGenParallelismLevel);<br class="gmail_msg">
@@ -260,9 +246,7 @@ void splitCodeGen(Config &C, TargetMachi<br class="gmail_msg">
std::unique_ptr<TargetMachine> TM =<br class="gmail_msg">
createTargetMachine(C, MPartInCtx->getTargetTriple(), T);<br class="gmail_msg">
<br class="gmail_msg">
- codegen(C, TM.get(),<br class="gmail_msg">
- getUncachedOutputWrapper(AddOutput, ThreadId), ThreadId,<br class="gmail_msg">
- *MPartInCtx);<br class="gmail_msg">
+ codegen(C, TM.get(), AddStream, ThreadId, *MPartInCtx);<br class="gmail_msg">
},<br class="gmail_msg">
// Pass BC using std::move to ensure that it get moved rather than<br class="gmail_msg">
// copied into the thread's context.<br class="gmail_msg">
@@ -299,7 +283,7 @@ static void handleAsmUndefinedRefs(Modul<br class="gmail_msg">
updateCompilerUsed(Mod, TM, AsmUndefinedRefs);<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-Error lto::backend(Config &C, AddOutputFn AddOutput,<br class="gmail_msg">
+Error lto::backend(Config &C, AddStreamFn AddStream,<br class="gmail_msg">
unsigned ParallelCodeGenParallelismLevel,<br class="gmail_msg">
std::unique_ptr<Module> Mod) {<br class="gmail_msg">
Expected<const Target *> TOrErr = initAndLookupTarget(C, *Mod);<br class="gmail_msg">
@@ -316,15 +300,15 @@ Error lto::backend(Config &C, AddOutputF<br class="gmail_msg">
return Error();<br class="gmail_msg">
<br class="gmail_msg">
if (ParallelCodeGenParallelismLevel == 1) {<br class="gmail_msg">
- codegen(C, TM.get(), getUncachedOutputWrapper(AddOutput, 0), 0, *Mod);<br class="gmail_msg">
+ codegen(C, TM.get(), AddStream, 0, *Mod);<br class="gmail_msg">
} else {<br class="gmail_msg">
- splitCodeGen(C, TM.get(), AddOutput, ParallelCodeGenParallelismLevel,<br class="gmail_msg">
+ splitCodeGen(C, TM.get(), AddStream, ParallelCodeGenParallelismLevel,<br class="gmail_msg">
std::move(Mod));<br class="gmail_msg">
}<br class="gmail_msg">
return Error();<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-Error lto::thinBackend(Config &Conf, unsigned Task, AddOutputFn AddOutput,<br class="gmail_msg">
+Error lto::thinBackend(Config &Conf, unsigned Task, AddStreamFn AddStream,<br class="gmail_msg">
Module &Mod, ModuleSummaryIndex &CombinedIndex,<br class="gmail_msg">
const FunctionImporter::ImportMapTy &ImportList,<br class="gmail_msg">
const GVSummaryMapTy &DefinedGlobals,<br class="gmail_msg">
@@ -339,7 +323,7 @@ Error lto::thinBackend(Config &Conf, uns<br class="gmail_msg">
handleAsmUndefinedRefs(Mod, *TM);<br class="gmail_msg">
<br class="gmail_msg">
if (Conf.CodeGenOnly) {<br class="gmail_msg">
- codegen(Conf, TM.get(), AddOutput, Task, Mod);<br class="gmail_msg">
+ codegen(Conf, TM.get(), AddStream, Task, Mod);<br class="gmail_msg">
return Error();<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
@@ -379,6 +363,6 @@ Error lto::thinBackend(Config &Conf, uns<br class="gmail_msg">
if (!opt(Conf, TM.get(), Task, Mod, /*IsThinLto=*/true))<br class="gmail_msg">
return Error();<br class="gmail_msg">
<br class="gmail_msg">
- codegen(Conf, TM.get(), AddOutput, Task, Mod);<br class="gmail_msg">
+ codegen(Conf, TM.get(), AddStream, Task, Mod);<br class="gmail_msg">
return Error();<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/tools/gold/gold-plugin.cpp<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/gold/gold-plugin.cpp?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/gold/gold-plugin.cpp?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/tools/gold/gold-plugin.cpp (original)<br class="gmail_msg">
+++ llvm/trunk/tools/gold/gold-plugin.cpp Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -675,24 +675,6 @@ static void getThinLTOOldAndNewPrefix(st<br class="gmail_msg">
NewPrefix = Split.second.str();<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-namespace {<br class="gmail_msg">
-// Define the LTOOutput handling<br class="gmail_msg">
-class LTOOutput : public lto::NativeObjectOutput {<br class="gmail_msg">
- StringRef Path;<br class="gmail_msg">
-<br class="gmail_msg">
-public:<br class="gmail_msg">
- LTOOutput(StringRef Path) : Path(Path) {}<br class="gmail_msg">
- // Open the filename \p Path and allocate a stream.<br class="gmail_msg">
- std::unique_ptr<raw_pwrite_stream> getStream() override {<br class="gmail_msg">
- int FD;<br class="gmail_msg">
- std::error_code EC = sys::fs::openFileForWrite(Path, FD, sys::fs::F_None);<br class="gmail_msg">
- if (EC)<br class="gmail_msg">
- message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str());<br class="gmail_msg">
- return llvm::make_unique<llvm::raw_fd_ostream>(FD, true);<br class="gmail_msg">
- }<br class="gmail_msg">
-};<br class="gmail_msg">
-}<br class="gmail_msg">
-<br class="gmail_msg">
static std::unique_ptr<LTO> createLTO() {<br class="gmail_msg">
Config Conf;<br class="gmail_msg">
ThinBackend Backend;<br class="gmail_msg">
@@ -831,21 +813,27 @@ static ld_plugin_status allSymbolsReadHo<br class="gmail_msg">
std::vector<uintptr_t> IsTemporary(MaxTasks);<br class="gmail_msg">
std::vector<SmallString<128>> Filenames(MaxTasks);<br class="gmail_msg">
<br class="gmail_msg">
- auto AddOutput =<br class="gmail_msg">
- [&](size_t Task) -> std::unique_ptr<lto::NativeObjectOutput> {<br class="gmail_msg">
- auto &OutputName = Filenames[Task];<br class="gmail_msg">
- getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps, OutputName,<br class="gmail_msg">
+ auto AddStream =<br class="gmail_msg">
+ [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {<br class="gmail_msg">
+ IsTemporary[Task] = !SaveTemps;<br class="gmail_msg">
+ getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps, Filenames[Task],<br class="gmail_msg">
MaxTasks > 1 ? Task : -1);<br class="gmail_msg">
- IsTemporary[Task] = !SaveTemps && options::cache_dir.empty();<br class="gmail_msg">
- if (options::cache_dir.empty())<br class="gmail_msg">
- return llvm::make_unique<LTOOutput>(OutputName);<br class="gmail_msg">
-<br class="gmail_msg">
- return llvm::make_unique<CacheObjectOutput>(<br class="gmail_msg">
- options::cache_dir,<br class="gmail_msg">
- [&OutputName](std::string EntryPath) { OutputName = EntryPath; });<br class="gmail_msg">
+ int FD;<br class="gmail_msg">
+ std::error_code EC =<br class="gmail_msg">
+ sys::fs::openFileForWrite(Filenames[Task], FD, sys::fs::F_None);<br class="gmail_msg">
+ if (EC)<br class="gmail_msg">
+ message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str());<br class="gmail_msg">
+ return llvm::make_unique<lto::NativeObjectStream>(<br class="gmail_msg">
+ llvm::make_unique<llvm::raw_fd_ostream>(FD, true));<br class="gmail_msg">
};<br class="gmail_msg">
<br class="gmail_msg">
- check(Lto->run(AddOutput));<br class="gmail_msg">
+ auto AddFile = [&](size_t Task, StringRef Path) { Filenames[Task] = Path; };<br class="gmail_msg">
+<br class="gmail_msg">
+ NativeObjectCache Cache;<br class="gmail_msg">
+ if (!options::cache_dir.empty())<br class="gmail_msg">
+ Cache = localCache(options::cache_dir, AddFile);<br class="gmail_msg">
+<br class="gmail_msg">
+ check(Lto->run(AddStream, Cache));<br class="gmail_msg">
<br class="gmail_msg">
if (options::TheOutputType == options::OT_DISABLE ||<br class="gmail_msg">
options::TheOutputType == options::OT_BC_ONLY)<br class="gmail_msg">
<br class="gmail_msg">
Modified: llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp<br class="gmail_msg">
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp?rev=282299&r1=282298&r2=282299&view=diff" rel="noreferrer" class="gmail_msg" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp?rev=282299&r1=282298&r2=282299&view=diff</a><br class="gmail_msg">
==============================================================================<br class="gmail_msg">
--- llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp (original)<br class="gmail_msg">
+++ llvm/trunk/tools/llvm-lto2/llvm-lto2.cpp Fri Sep 23 16:33:43 2016<br class="gmail_msg">
@@ -95,22 +95,6 @@ template <typename T> static T check(Err<br class="gmail_msg">
return T();<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
-namespace {<br class="gmail_msg">
-// Define the LTOOutput handling<br class="gmail_msg">
-class LTOOutput : public lto::NativeObjectOutput {<br class="gmail_msg">
- std::string Path;<br class="gmail_msg">
-<br class="gmail_msg">
-public:<br class="gmail_msg">
- LTOOutput(std::string Path) : Path(std::move(Path)) {}<br class="gmail_msg">
- std::unique_ptr<raw_pwrite_stream> getStream() override {<br class="gmail_msg">
- std::error_code EC;<br class="gmail_msg">
- auto S = llvm::make_unique<raw_fd_ostream>(Path, EC, sys::fs::F_None);<br class="gmail_msg">
- check(EC, Path);<br class="gmail_msg">
- return std::move(S);<br class="gmail_msg">
- }<br class="gmail_msg">
-};<br class="gmail_msg">
-}<br class="gmail_msg">
-<br class="gmail_msg">
int main(int argc, char **argv) {<br class="gmail_msg">
InitializeAllTargets();<br class="gmail_msg">
InitializeAllTargetMCs();<br class="gmail_msg">
@@ -203,23 +187,28 @@ int main(int argc, char **argv) {<br class="gmail_msg">
if (HasErrors)<br class="gmail_msg">
return 1;<br class="gmail_msg">
<br class="gmail_msg">
- auto AddOutput =<br class="gmail_msg">
- [&](size_t Task) -> std::unique_ptr<lto::NativeObjectOutput> {<br class="gmail_msg">
+ auto AddStream =<br class="gmail_msg">
+ [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {<br class="gmail_msg">
std::string Path = OutputFilename + "." + utostr(Task);<br class="gmail_msg">
- if (CacheDir.empty())<br class="gmail_msg">
- return llvm::make_unique<LTOOutput>(std::move(Path));<br class="gmail_msg">
<br class="gmail_msg">
- return llvm::make_unique<CacheObjectOutput>(<br class="gmail_msg">
- CacheDir, [Path](std::string EntryPath) {<br class="gmail_msg">
- // Load the entry from the cache now.<br class="gmail_msg">
- auto ReloadedBufferOrErr = MemoryBuffer::getFile(EntryPath);<br class="gmail_msg">
- if (auto EC = ReloadedBufferOrErr.getError())<br class="gmail_msg">
- report_fatal_error(Twine("Can't reload cached file '") + EntryPath +<br class="gmail_msg">
- "': " + EC.message() + "\n");<br class="gmail_msg">
+ std::error_code EC;<br class="gmail_msg">
+ auto S = llvm::make_unique<raw_fd_ostream>(Path, EC, sys::fs::F_None);<br class="gmail_msg">
+ check(EC, Path);<br class="gmail_msg">
+ return llvm::make_unique<lto::NativeObjectStream>(std::move(S));<br class="gmail_msg">
+ };<br class="gmail_msg">
+<br class="gmail_msg">
+ auto AddFile = [&](size_t Task, StringRef Path) {<br class="gmail_msg">
+ auto ReloadedBufferOrErr = MemoryBuffer::getFile(Path);<br class="gmail_msg">
+ if (auto EC = ReloadedBufferOrErr.getError())<br class="gmail_msg">
+ report_fatal_error(Twine("Can't reload cached file '") + Path + "': " +<br class="gmail_msg">
+ EC.message() + "\n");<br class="gmail_msg">
<br class="gmail_msg">
- *LTOOutput(Path).getStream() << (*ReloadedBufferOrErr)->getBuffer();<br class="gmail_msg">
- });<br class="gmail_msg">
+ *AddStream(Task)->OS << (*ReloadedBufferOrErr)->getBuffer();<br class="gmail_msg">
};<br class="gmail_msg">
<br class="gmail_msg">
- check(Lto.run(AddOutput), "LTO::run failed");<br class="gmail_msg">
+ NativeObjectCache Cache;<br class="gmail_msg">
+ if (!CacheDir.empty())<br class="gmail_msg">
+ Cache = localCache(CacheDir, AddFile);<br class="gmail_msg">
+<br class="gmail_msg">
+ check(Lto.run(AddStream, Cache), "LTO::run failed");<br class="gmail_msg">
}<br class="gmail_msg">
<br class="gmail_msg">
<br class="gmail_msg">
_______________________________________________<br class="gmail_msg">
llvm-commits mailing list<br class="gmail_msg">
<a href="mailto:llvm-commits@lists.llvm.org" class="gmail_msg" target="_blank">llvm-commits@lists.llvm.org</a><br class="gmail_msg">
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" class="gmail_msg" target="_blank">http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits</a><br class="gmail_msg">
</blockquote></div>