r318411 - Allow to store precompiled preambles in memory.

Ilya Biryukov via cfe-commits cfe-commits at lists.llvm.org
Thu Nov 16 08:25:01 PST 2017


Author: ibiryukov
Date: Thu Nov 16 08:25:01 2017
New Revision: 318411

URL: http://llvm.org/viewvc/llvm-project?rev=318411&view=rev
Log:
Allow to store precompiled preambles in memory.

Summary:
These preambles are built by ASTUnit and clangd. Previously, preambles
were always stored on disk.

In-memory preambles are routed back to the compiler as virtual files in
a custom VFS.

Interface of ASTUnit does not allow to use in-memory preambles, as
ASTUnit::CodeComplete receives FileManager as a parameter, so we can't
change VFS used by the compiler inside the CodeComplete method.

A follow-up commit will update clangd in clang-tools-extra to use
in-memory preambles.

Reviewers: klimek, sammccall, bkramer

Reviewed By: klimek

Subscribers: ioeric, cfe-commits

Differential Revision: https://reviews.llvm.org/D39842

Modified:
    cfe/trunk/include/clang/Frontend/FrontendActions.h
    cfe/trunk/include/clang/Frontend/PrecompiledPreamble.h
    cfe/trunk/lib/Frontend/ASTUnit.cpp
    cfe/trunk/lib/Frontend/FrontendActions.cpp
    cfe/trunk/lib/Frontend/PrecompiledPreamble.cpp

Modified: cfe/trunk/include/clang/Frontend/FrontendActions.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/FrontendActions.h?rev=318411&r1=318410&r2=318411&view=diff
==============================================================================
--- cfe/trunk/include/clang/Frontend/FrontendActions.h (original)
+++ cfe/trunk/include/clang/Frontend/FrontendActions.h Thu Nov 16 08:25:01 2017
@@ -86,10 +86,15 @@ public:
   /// \brief Compute the AST consumer arguments that will be used to
   /// create the PCHGenerator instance returned by CreateASTConsumer.
   ///
-  /// \returns true if an error occurred, false otherwise.
-  static std::unique_ptr<raw_pwrite_stream>
-  ComputeASTConsumerArguments(CompilerInstance &CI, StringRef InFile,
-                              std::string &Sysroot, std::string &OutputFile);
+  /// \returns false if an error occurred, true otherwise.
+  static bool ComputeASTConsumerArguments(CompilerInstance &CI,
+                                          std::string &Sysroot);
+
+  /// \brief Creates file to write the PCH into and returns a stream to write it
+  /// into. On error, returns null.
+  static std::unique_ptr<llvm::raw_pwrite_stream>
+  CreateOutputFile(CompilerInstance &CI, StringRef InFile,
+                   std::string &OutputFile);
 
   bool BeginSourceFileAction(CompilerInstance &CI) override;
 };

Modified: cfe/trunk/include/clang/Frontend/PrecompiledPreamble.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/PrecompiledPreamble.h?rev=318411&r1=318410&r2=318411&view=diff
==============================================================================
--- cfe/trunk/include/clang/Frontend/PrecompiledPreamble.h (original)
+++ cfe/trunk/include/clang/Frontend/PrecompiledPreamble.h Thu Nov 16 08:25:01 2017
@@ -17,6 +17,7 @@
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/MD5.h"
 #include <memory>
 #include <system_error>
@@ -47,7 +48,7 @@ class PreambleCallbacks;
 /// reuse the PCH for the subsequent runs. Use BuildPreamble to create PCH and
 /// CanReusePreamble + AddImplicitPreamble to make use of it.
 class PrecompiledPreamble {
-  class TempPCHFile;
+  class PCHStorage;
   struct PreambleFileHash;
 
 public:
@@ -70,6 +71,9 @@ public:
   ///
   /// \param PCHContainerOps An instance of PCHContainerOperations.
   ///
+  /// \param StoreInMemory Store PCH in memory. If false, PCH will be stored in
+  /// a temporary file.
+  ///
   /// \param Callbacks A set of callbacks to be executed when building
   /// the preamble.
   static llvm::ErrorOr<PrecompiledPreamble>
@@ -77,7 +81,7 @@ public:
         const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
         DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
         std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-        PreambleCallbacks &Callbacks);
+        bool StoreInMemory, PreambleCallbacks &Callbacks);
 
   PrecompiledPreamble(PrecompiledPreamble &&) = default;
   PrecompiledPreamble &operator=(PrecompiledPreamble &&) = default;
@@ -85,9 +89,6 @@ public:
   /// PreambleBounds used to build the preamble.
   PreambleBounds getBounds() const;
 
-  /// The temporary file path at which the preamble PCH was placed.
-  StringRef GetPCHPath() const { return PCHFile.getFilePath(); }
-
   /// Check whether PrecompiledPreamble can be reused for the new contents(\p
   /// MainFileBuffer) of the main file.
   bool CanReuse(const CompilerInvocation &Invocation,
@@ -95,12 +96,14 @@ public:
                 vfs::FileSystem *VFS) const;
 
   /// Changes options inside \p CI to use PCH from this preamble. Also remaps
-  /// main file to \p MainFileBuffer.
+  /// main file to \p MainFileBuffer and updates \p VFS to ensure the preamble
+  /// is accessible.
   void AddImplicitPreamble(CompilerInvocation &CI,
+                           IntrusiveRefCntPtr<vfs::FileSystem> &VFS,
                            llvm::MemoryBuffer *MainFileBuffer) const;
 
 private:
-  PrecompiledPreamble(TempPCHFile PCHFile, std::vector<char> PreambleBytes,
+  PrecompiledPreamble(PCHStorage Storage, std::vector<char> PreambleBytes,
                       bool PreambleEndsAtStartOfLine,
                       llvm::StringMap<PreambleFileHash> FilesInPreamble);
 
@@ -142,6 +145,44 @@ private:
     llvm::Optional<std::string> FilePath;
   };
 
+  class InMemoryPreamble {
+  public:
+    std::string Data;
+  };
+
+  class PCHStorage {
+  public:
+    enum class Kind { Empty, InMemory, TempFile };
+
+    PCHStorage() = default;
+    PCHStorage(TempPCHFile File);
+    PCHStorage(InMemoryPreamble Memory);
+
+    PCHStorage(const PCHStorage &) = delete;
+    PCHStorage &operator=(const PCHStorage &) = delete;
+
+    PCHStorage(PCHStorage &&Other);
+    PCHStorage &operator=(PCHStorage &&Other);
+
+    ~PCHStorage();
+
+    Kind getKind() const;
+
+    TempPCHFile &asFile();
+    const TempPCHFile &asFile() const;
+
+    InMemoryPreamble &asMemory();
+    const InMemoryPreamble &asMemory() const;
+
+  private:
+    void destroy();
+    void setEmpty();
+
+  private:
+    Kind StorageKind = Kind::Empty;
+    llvm::AlignedCharArrayUnion<TempPCHFile, InMemoryPreamble> Storage = {};
+  };
+
   /// Data used to determine if a file used in the preamble has been changed.
   struct PreambleFileHash {
     /// All files have size set.
@@ -171,8 +212,15 @@ private:
     }
   };
 
-  /// Manages the lifetime of temporary file that stores a PCH.
-  TempPCHFile PCHFile;
+  /// Sets up the PreprocessorOptions and changes VFS, so that PCH stored in \p
+  /// Storage is accessible to clang. This method is an implementation detail of
+  /// AddImplicitPreamble.
+  static void setupPreambleStorage(const PCHStorage &Storage,
+                                   PreprocessorOptions &PreprocessorOpts,
+                                   IntrusiveRefCntPtr<vfs::FileSystem> &VFS);
+
+  /// Manages the memory buffer or temporary file that stores the PCH.
+  PCHStorage Storage;
   /// Keeps track of the files that were used when computing the
   /// preamble, with both their buffer size and their modification time.
   ///

Modified: cfe/trunk/lib/Frontend/ASTUnit.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/ASTUnit.cpp?rev=318411&r1=318410&r2=318411&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/ASTUnit.cpp (original)
+++ cfe/trunk/lib/Frontend/ASTUnit.cpp Thu Nov 16 08:25:01 2017
@@ -1013,24 +1013,6 @@ static void checkAndSanitizeDiags(SmallV
   }
 }
 
-static IntrusiveRefCntPtr<vfs::FileSystem> createVFSOverlayForPreamblePCH(
-    StringRef PCHFilename,
-    IntrusiveRefCntPtr<vfs::FileSystem> RealFS,
-    IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
-  // We want only the PCH file from the real filesystem to be available,
-  // so we create an in-memory VFS with just that and overlay it on top.
-  auto Buf = RealFS->getBufferForFile(PCHFilename);
-  if (!Buf)
-    return VFS;
-  IntrusiveRefCntPtr<vfs::InMemoryFileSystem>
-      PCHFS(new vfs::InMemoryFileSystem());
-  PCHFS->addFile(PCHFilename, 0, std::move(*Buf));
-  IntrusiveRefCntPtr<vfs::OverlayFileSystem>
-      Overlay(new vfs::OverlayFileSystem(VFS));
-  Overlay->pushOverlay(PCHFS);
-  return Overlay;
-}
-
 /// Parse the source file into a translation unit using the given compiler
 /// invocation, replacing the current translation unit.
 ///
@@ -1042,6 +1024,19 @@ bool ASTUnit::Parse(std::shared_ptr<PCHC
   if (!Invocation)
     return true;
 
+  auto CCInvocation = std::make_shared<CompilerInvocation>(*Invocation);
+  if (OverrideMainBuffer) {
+    assert(Preamble &&
+           "No preamble was built, but OverrideMainBuffer is not null");
+    IntrusiveRefCntPtr<vfs::FileSystem> OldVFS = VFS;
+    Preamble->AddImplicitPreamble(*CCInvocation, VFS, OverrideMainBuffer.get());
+    if (OldVFS != VFS && FileMgr) {
+      assert(OldVFS == FileMgr->getVirtualFileSystem() &&
+             "VFS passed to Parse and VFS in FileMgr are different");
+      FileMgr = new FileManager(FileMgr->getFileSystemOpts(), VFS);
+    }
+  }
+
   // Create the compiler instance to use for building the AST.
   std::unique_ptr<CompilerInstance> Clang(
       new CompilerInstance(std::move(PCHContainerOps)));
@@ -1052,29 +1047,11 @@ bool ASTUnit::Parse(std::shared_ptr<PCHC
     Clang->setVirtualFileSystem(VFS);
   }
 
-  // Make sure we can access the PCH file even if we're using a VFS
-  if (!VFS && FileMgr)
-    VFS = FileMgr->getVirtualFileSystem();
-  IntrusiveRefCntPtr<vfs::FileSystem> RealFS = vfs::getRealFileSystem();
-  if (OverrideMainBuffer && VFS && RealFS && VFS != RealFS &&
-      !VFS->exists(Preamble->GetPCHPath())) {
-    // We have a slight inconsistency here -- we're using the VFS to
-    // read files, but the PCH was generated in the real file system.
-    VFS = createVFSOverlayForPreamblePCH(Preamble->GetPCHPath(), RealFS, VFS);
-    if (FileMgr) {
-      FileMgr = new FileManager(FileMgr->getFileSystemOpts(), VFS);
-      Clang->setFileManager(FileMgr.get());
-    }
-    else {
-      Clang->setVirtualFileSystem(VFS);
-    }
-  }
-
   // Recover resources if we crash before exiting this method.
   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance>
     CICleanup(Clang.get());
 
-  Clang->setInvocation(std::make_shared<CompilerInvocation>(*Invocation));
+  Clang->setInvocation(CCInvocation);
   OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].getFile();
     
   // Set up diagnostics, capturing any diagnostics that would
@@ -1128,9 +1105,6 @@ bool ASTUnit::Parse(std::shared_ptr<PCHC
   // If the main file has been overridden due to the use of a preamble,
   // make that override happen and introduce the preamble.
   if (OverrideMainBuffer) {
-    assert(Preamble && "No preamble was built, but OverrideMainBuffer is not null");
-    Preamble->AddImplicitPreamble(Clang->getInvocation(), OverrideMainBuffer.get());
-    
     // The stored diagnostic has the old source manager in it; update
     // the locations to refer into the new source manager. Since we've
     // been careful to make sure that the source manager's state
@@ -1319,7 +1293,7 @@ ASTUnit::getMainBufferWithPrecompiledPre
 
     llvm::ErrorOr<PrecompiledPreamble> NewPreamble = PrecompiledPreamble::Build(
         PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics, VFS,
-        PCHContainerOps, Callbacks);
+        PCHContainerOps, /*StoreInMemory=*/false, Callbacks);
     if (NewPreamble) {
       Preamble = std::move(*NewPreamble);
       PreambleRebuildCounter = 1;
@@ -2195,8 +2169,16 @@ void ASTUnit::CodeComplete(
   // If the main file has been overridden due to the use of a preamble,
   // make that override happen and introduce the preamble.
   if (OverrideMainBuffer) {
-    assert(Preamble && "No preamble was built, but OverrideMainBuffer is not null");
-    Preamble->AddImplicitPreamble(Clang->getInvocation(), OverrideMainBuffer.get());
+    assert(Preamble &&
+           "No preamble was built, but OverrideMainBuffer is not null");
+
+    auto VFS = FileMgr.getVirtualFileSystem();
+    Preamble->AddImplicitPreamble(Clang->getInvocation(), VFS,
+                                  OverrideMainBuffer.get());
+    // FIXME: there is no way to update VFS if it was changed by
+    // AddImplicitPreamble as FileMgr is accepted as a parameter by this method.
+    // We use on-disk preambles instead and rely on FileMgr's VFS to ensure the
+    // PCH files are always readable.
     OwnedBuffers.push_back(OverrideMainBuffer.release());
   } else {
     PreprocessorOpts.PrecompiledPreambleBytes.first = 0;

Modified: cfe/trunk/lib/Frontend/FrontendActions.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/FrontendActions.cpp?rev=318411&r1=318410&r2=318411&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/FrontendActions.cpp (original)
+++ cfe/trunk/lib/Frontend/FrontendActions.cpp Thu Nov 16 08:25:01 2017
@@ -80,9 +80,12 @@ DeclContextPrintAction::CreateASTConsume
 std::unique_ptr<ASTConsumer>
 GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   std::string Sysroot;
+  if (!ComputeASTConsumerArguments(CI, /*ref*/ Sysroot))
+    return nullptr;
+
   std::string OutputFile;
   std::unique_ptr<raw_pwrite_stream> OS =
-      ComputeASTConsumerArguments(CI, InFile, Sysroot, OutputFile);
+      CreateOutputFile(CI, InFile, /*ref*/ OutputFile);
   if (!OS)
     return nullptr;
 
@@ -103,17 +106,20 @@ GeneratePCHAction::CreateASTConsumer(Com
   return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
 }
 
-std::unique_ptr<raw_pwrite_stream>
-GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI,
-                                               StringRef InFile,
-                                               std::string &Sysroot,
-                                               std::string &OutputFile) {
+bool GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI,
+                                                    std::string &Sysroot) {
   Sysroot = CI.getHeaderSearchOpts().Sysroot;
   if (CI.getFrontendOpts().RelocatablePCH && Sysroot.empty()) {
     CI.getDiagnostics().Report(diag::err_relocatable_without_isysroot);
-    return nullptr;
+    return false;
   }
 
+  return true;
+}
+
+std::unique_ptr<llvm::raw_pwrite_stream>
+GeneratePCHAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile,
+                                    std::string &OutputFile) {
   // We use createOutputFile here because this is exposed via libclang, and we
   // must disable the RemoveFileOnSignal behavior.
   // We use a temporary to avoid race conditions.

Modified: cfe/trunk/lib/Frontend/PrecompiledPreamble.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/PrecompiledPreamble.cpp?rev=318411&r1=318410&r2=318411&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/PrecompiledPreamble.cpp (original)
+++ cfe/trunk/lib/Frontend/PrecompiledPreamble.cpp Thu Nov 16 08:25:01 2017
@@ -24,16 +24,45 @@
 #include "clang/Serialization/ASTWriter.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Config/config.h"
 #include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/MutexGuard.h"
 #include "llvm/Support/Process.h"
 
+#include <utility>
+
 using namespace clang;
 
 namespace {
 
+StringRef getInMemoryPreamblePath() {
+#if defined(LLVM_ON_UNIX)
+  return "/__clang_tmp/___clang_inmemory_preamble___";
+#elif defined(LLVM_ON_WIN32)
+  return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
+#else
+#warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
+  return "/__clang_tmp/___clang_inmemory_preamble___";
+#endif
+}
+
+IntrusiveRefCntPtr<vfs::FileSystem>
+createVFSOverlayForPreamblePCH(StringRef PCHFilename,
+                               std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
+                               IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
+  // We want only the PCH file from the real filesystem to be available,
+  // so we create an in-memory VFS with just that and overlay it on top.
+  IntrusiveRefCntPtr<vfs::InMemoryFileSystem> PCHFS(
+      new vfs::InMemoryFileSystem());
+  PCHFS->addFile(PCHFilename, 0, std::move(PCHBuffer));
+  IntrusiveRefCntPtr<vfs::OverlayFileSystem> Overlay(
+      new vfs::OverlayFileSystem(VFS));
+  Overlay->pushOverlay(PCHFS);
+  return Overlay;
+}
+
 /// Keeps a track of files to be deleted in destructor.
 class TemporaryFiles {
 public:
@@ -101,8 +130,9 @@ private:
 
 class PrecompilePreambleAction : public ASTFrontendAction {
 public:
-  PrecompilePreambleAction(PreambleCallbacks &Callbacks)
-      : Callbacks(Callbacks) {}
+  PrecompilePreambleAction(std::string *InMemStorage,
+                           PreambleCallbacks &Callbacks)
+      : InMemStorage(InMemStorage), Callbacks(Callbacks) {}
 
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
@@ -123,6 +153,7 @@ private:
   friend class PrecompilePreambleConsumer;
 
   bool HasEmittedPreamblePCH = false;
+  std::string *InMemStorage;
   PreambleCallbacks &Callbacks;
 };
 
@@ -164,13 +195,18 @@ private:
 
 std::unique_ptr<ASTConsumer>
 PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
-
                                             StringRef InFile) {
   std::string Sysroot;
-  std::string OutputFile;
-  std::unique_ptr<raw_ostream> OS =
-      GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot,
-                                                     OutputFile);
+  if (!GeneratePCHAction::ComputeASTConsumerArguments(CI, Sysroot))
+    return nullptr;
+
+  std::unique_ptr<llvm::raw_ostream> OS;
+  if (InMemStorage) {
+    OS = llvm::make_unique<llvm::raw_string_ostream>(*InMemStorage);
+  } else {
+    std::string OutputFile;
+    OS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
+  }
   if (!OS)
     return nullptr;
 
@@ -202,7 +238,7 @@ llvm::ErrorOr<PrecompiledPreamble> Preco
     const CompilerInvocation &Invocation,
     const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
     DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
-    std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+    std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
     PreambleCallbacks &Callbacks) {
   assert(VFS && "VFS is null");
 
@@ -214,12 +250,19 @@ llvm::ErrorOr<PrecompiledPreamble> Preco
   PreprocessorOptions &PreprocessorOpts =
       PreambleInvocation->getPreprocessorOpts();
 
-  // Create a temporary file for the precompiled preamble. In rare
-  // circumstances, this can fail.
-  llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
-      PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
-  if (!PreamblePCHFile)
-    return BuildPreambleError::CouldntCreateTempFile;
+  llvm::Optional<TempPCHFile> TempFile;
+  if (!StoreInMemory) {
+    // Create a temporary file for the precompiled preamble. In rare
+    // circumstances, this can fail.
+    llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
+        PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
+    if (!PreamblePCHFile)
+      return BuildPreambleError::CouldntCreateTempFile;
+    TempFile = std::move(*PreamblePCHFile);
+  }
+
+  PCHStorage Storage = StoreInMemory ? PCHStorage(InMemoryPreamble())
+                                     : PCHStorage(std::move(*TempFile));
 
   // Save the preamble text for later; we'll need to compare against it for
   // subsequent reparses.
@@ -230,8 +273,8 @@ llvm::ErrorOr<PrecompiledPreamble> Preco
 
   // Tell the compiler invocation to generate a temporary precompiled header.
   FrontendOpts.ProgramAction = frontend::GeneratePCH;
-  // FIXME: Generate the precompiled header into memory?
-  FrontendOpts.OutputFile = PreamblePCHFile->getFilePath();
+  FrontendOpts.OutputFile = StoreInMemory ? getInMemoryPreamblePath()
+                                          : Storage.asFile().getFilePath();
   PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
   PreprocessorOpts.PrecompiledPreambleBytes.second = false;
   // Inform preprocessor to record conditional stack when building the preamble.
@@ -303,7 +346,8 @@ llvm::ErrorOr<PrecompiledPreamble> Preco
   }
 
   std::unique_ptr<PrecompilePreambleAction> Act;
-  Act.reset(new PrecompilePreambleAction(Callbacks));
+  Act.reset(new PrecompilePreambleAction(
+      StoreInMemory ? &Storage.asMemory().Data : nullptr, Callbacks));
   if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
     return BuildPreambleError::BeginSourceFileFailed;
 
@@ -337,9 +381,9 @@ llvm::ErrorOr<PrecompiledPreamble> Preco
     }
   }
 
-  return PrecompiledPreamble(
-      std::move(*PreamblePCHFile), std::move(PreambleBytes),
-      PreambleEndsAtStartOfLine, std::move(FilesInPreamble));
+  return PrecompiledPreamble(std::move(Storage), std::move(PreambleBytes),
+                             PreambleEndsAtStartOfLine,
+                             std::move(FilesInPreamble));
 }
 
 PreambleBounds PrecompiledPreamble::getBounds() const {
@@ -427,27 +471,33 @@ bool PrecompiledPreamble::CanReuse(const
 }
 
 void PrecompiledPreamble::AddImplicitPreamble(
-    CompilerInvocation &CI, llvm::MemoryBuffer *MainFileBuffer) const {
+    CompilerInvocation &CI, IntrusiveRefCntPtr<vfs::FileSystem> &VFS,
+    llvm::MemoryBuffer *MainFileBuffer) const {
+  assert(VFS && "VFS must not be null");
+
   auto &PreprocessorOpts = CI.getPreprocessorOpts();
 
+  // Remap main file to point to MainFileBuffer.
+  auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
+  PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
+
   // Configure ImpicitPCHInclude.
   PreprocessorOpts.PrecompiledPreambleBytes.first = PreambleBytes.size();
   PreprocessorOpts.PrecompiledPreambleBytes.second = PreambleEndsAtStartOfLine;
-  PreprocessorOpts.ImplicitPCHInclude = PCHFile.getFilePath();
   PreprocessorOpts.DisablePCHValidation = true;
 
-  // Remap main file to point to MainFileBuffer.
-  auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
-  PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
+  setupPreambleStorage(Storage, PreprocessorOpts, VFS);
 }
 
 PrecompiledPreamble::PrecompiledPreamble(
-    TempPCHFile PCHFile, std::vector<char> PreambleBytes,
+    PCHStorage Storage, std::vector<char> PreambleBytes,
     bool PreambleEndsAtStartOfLine,
     llvm::StringMap<PreambleFileHash> FilesInPreamble)
-    : PCHFile(std::move(PCHFile)), FilesInPreamble(std::move(FilesInPreamble)),
+    : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
       PreambleBytes(std::move(PreambleBytes)),
-      PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {}
+      PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
+  assert(this->Storage.getKind() != PCHStorage::Kind::Empty);
+}
 
 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
 PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile() {
@@ -468,8 +518,7 @@ PrecompiledPreamble::TempPCHFile::create
   // that we would never get a race condition in a multi-threaded setting (i.e.,
   // multiple threads getting the same temporary path).
   int FD;
-  auto EC = llvm::sys::fs::createTemporaryFile(Prefix, Suffix, /*ref*/ FD,
-                                               /*ref*/ File);
+  auto EC = llvm::sys::fs::createTemporaryFile(Prefix, Suffix, FD, File);
   if (EC)
     return EC;
   // We only needed to make sure the file exists, close the file right away.
@@ -515,6 +564,87 @@ llvm::StringRef PrecompiledPreamble::Tem
   return *FilePath;
 }
 
+PrecompiledPreamble::PCHStorage::PCHStorage(TempPCHFile File)
+    : StorageKind(Kind::TempFile) {
+  new (&asFile()) TempPCHFile(std::move(File));
+}
+
+PrecompiledPreamble::PCHStorage::PCHStorage(InMemoryPreamble Memory)
+    : StorageKind(Kind::InMemory) {
+  new (&asMemory()) InMemoryPreamble(std::move(Memory));
+}
+
+PrecompiledPreamble::PCHStorage::PCHStorage(PCHStorage &&Other) : PCHStorage() {
+  *this = std::move(Other);
+}
+
+PrecompiledPreamble::PCHStorage &PrecompiledPreamble::PCHStorage::
+operator=(PCHStorage &&Other) {
+  destroy();
+
+  StorageKind = Other.StorageKind;
+  switch (StorageKind) {
+  case Kind::Empty:
+    // do nothing;
+    break;
+  case Kind::TempFile:
+    new (&asFile()) TempPCHFile(std::move(Other.asFile()));
+    break;
+  case Kind::InMemory:
+    new (&asMemory()) InMemoryPreamble(std::move(Other.asMemory()));
+    break;
+  }
+
+  Other.setEmpty();
+  return *this;
+}
+
+PrecompiledPreamble::PCHStorage::~PCHStorage() { destroy(); }
+
+PrecompiledPreamble::PCHStorage::Kind
+PrecompiledPreamble::PCHStorage::getKind() const {
+  return StorageKind;
+}
+
+PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::PCHStorage::asFile() {
+  assert(getKind() == Kind::TempFile);
+  return *reinterpret_cast<TempPCHFile *>(Storage.buffer);
+}
+
+const PrecompiledPreamble::TempPCHFile &
+PrecompiledPreamble::PCHStorage::asFile() const {
+  return const_cast<PCHStorage *>(this)->asFile();
+}
+
+PrecompiledPreamble::InMemoryPreamble &
+PrecompiledPreamble::PCHStorage::asMemory() {
+  assert(getKind() == Kind::InMemory);
+  return *reinterpret_cast<InMemoryPreamble *>(Storage.buffer);
+}
+
+const PrecompiledPreamble::InMemoryPreamble &
+PrecompiledPreamble::PCHStorage::asMemory() const {
+  return const_cast<PCHStorage *>(this)->asMemory();
+}
+
+void PrecompiledPreamble::PCHStorage::destroy() {
+  switch (StorageKind) {
+  case Kind::Empty:
+    return;
+  case Kind::TempFile:
+    asFile().~TempPCHFile();
+    return;
+  case Kind::InMemory:
+    asMemory().~InMemoryPreamble();
+    return;
+  }
+}
+
+void PrecompiledPreamble::PCHStorage::setEmpty() {
+  destroy();
+  StorageKind = Kind::Empty;
+}
+
 PrecompiledPreamble::PreambleFileHash
 PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
                                                      time_t ModTime) {
@@ -539,6 +669,43 @@ PrecompiledPreamble::PreambleFileHash::c
   return Result;
 }
 
+void PrecompiledPreamble::setupPreambleStorage(
+    const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
+    IntrusiveRefCntPtr<vfs::FileSystem> &VFS) {
+  if (Storage.getKind() == PCHStorage::Kind::TempFile) {
+    const TempPCHFile &PCHFile = Storage.asFile();
+    PreprocessorOpts.ImplicitPCHInclude = PCHFile.getFilePath();
+
+    // Make sure we can access the PCH file even if we're using a VFS
+    IntrusiveRefCntPtr<vfs::FileSystem> RealFS = vfs::getRealFileSystem();
+    auto PCHPath = PCHFile.getFilePath();
+    if (VFS == RealFS || VFS->exists(PCHPath))
+      return;
+    auto Buf = RealFS->getBufferForFile(PCHPath);
+    if (!Buf) {
+      // We can't read the file even from RealFS, this is clearly an error,
+      // but we'll just leave the current VFS as is and let clang's code
+      // figure out what to do with missing PCH.
+      return;
+    }
+
+    // We have a slight inconsistency here -- we're using the VFS to
+    // read files, but the PCH was generated in the real file system.
+    VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(*Buf), VFS);
+  } else {
+    assert(Storage.getKind() == PCHStorage::Kind::InMemory);
+    // For in-memory preamble, we have to provide a VFS overlay that makes it
+    // accessible.
+    StringRef PCHPath = getInMemoryPreamblePath();
+    PreprocessorOpts.ImplicitPCHInclude = PCHPath;
+
+    // FIMXE(ibiryukov): Preambles can be large. We should allow shared access
+    // to the preamble data instead of copying it here.
+    auto Buf = llvm::MemoryBuffer::getMemBufferCopy(Storage.asMemory().Data);
+    VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(Buf), VFS);
+  }
+}
+
 void PreambleCallbacks::AfterExecute(CompilerInstance &CI) {}
 void PreambleCallbacks::AfterPCHEmitted(ASTWriter &Writer) {}
 void PreambleCallbacks::HandleTopLevelDecl(DeclGroupRef DG) {}




More information about the cfe-commits mailing list