[clang] 112b619 - [clang] Use the module cache's view of built PCMs (#242383)

via cfe-commits cfe-commits at lists.llvm.org
Mon Nov 16 10:52:24 PST 2026


Author: Jan Svoboda
Date: 2026-11-16T10:52:16-08:00
New Revision: 112b619dac2d073df046fe2d0cb5bc0b21046994

URL: https://github.com/llvm/llvm-project/commit/112b619dac2d073df046fe2d0cb5bc0b21046994
DIFF: https://github.com/llvm/llvm-project/commit/112b619dac2d073df046fe2d0cb5bc0b21046994.diff

LOG: [clang] Use the module cache's view of built PCMs (#242383)

When building a module implicitly, the compiler hands the PCM contents
to
`ModuleCache::write()` and then extracts the AST file from the PCH
container
of its own buffer, storing a copy of it in the `InMemoryModuleCache` of
the
importing instance.

This PR makes `ModuleCache::write()` return a buffer with the written
contents, which the importing instance now extracts the AST file from.
The
in-process module cache returns a buffer sharing ownership of its entry,
the
cross-process module cache returns the written file, read back from the
same
file descriptor it was written through (memory-mapped for all but small
files). This is a prerequisite for storing the module cache's view in
the
`InMemoryModuleCache` without copying it.

To support this, `writeImpl()` can now read the written file back from
the
same file descriptor, guaranteeing we map exactly the contents we wrote
and
making it immune to concurrent replacement of the file.

`ModuleCache::write()` now also takes ownership of the compiled buffer,
making
it clear the caller isn't supposed to use it anymore. The in-process
module
cache now stores it without copying, and the cross-process module cache
frees
it as soon as it's written.

Added: 
    

Modified: 
    clang/include/clang/Serialization/ModuleCache.h
    clang/lib/DependencyScanning/InProcessModuleCache.cpp
    clang/lib/Frontend/CompilerInstance.cpp
    clang/lib/Serialization/ModuleCache.cpp
    clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
    clang/unittests/Serialization/ModuleCacheTest.cpp

Removed: 
    


################################################################################
diff  --git a/clang/include/clang/Serialization/ModuleCache.h b/clang/include/clang/Serialization/ModuleCache.h
index 8fdd36911cce9d..2faae46926db0c 100644
--- a/clang/include/clang/Serialization/ModuleCache.h
+++ b/clang/include/clang/Serialization/ModuleCache.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem/UniqueID.h"
 
 #include <ctime>
@@ -89,9 +90,14 @@ class ModuleCache {
   virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
   virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
 
-  /// Write the PCM contents to the given path in the module cache.
-  virtual std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
-                                off_t &Size, time_t &ModTime) = 0;
+  /// Write the PCM contents to the given path in the module cache. Takes
+  /// ownership of \p Buffer, so that it can be freed as soon as possible.
+  ///
+  /// Returns a buffer with the written contents, which readers in this process
+  /// should use instead of their own copy.
+  virtual llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+        time_t &ModTime) = 0;
 
   virtual Expected<std::unique_ptr<llvm::MemoryBuffer>>
   read(StringRef FileName, off_t &Size, time_t &ModTime) = 0;
@@ -117,8 +123,13 @@ void maybePruneImpl(StringRef Path, time_t PruneInterval, time_t PruneAfter,
                     llvm::function_ref<void(StringRef)> OnPrune = {});
 
 /// Shared implementation of `ModuleCache::write()`.
-std::error_code writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
-                          off_t &Size, time_t &ModTime);
+///
+/// If \p ReadBack is true, returns the contents of the written file, read back
+/// from the same file descriptor (and therefore immune to concurrent
+/// replacement of the file at \p Path). Otherwise, returns nullptr on success.
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer, off_t &Size,
+          time_t &ModTime, bool ReadBack);
 
 /// Shared implementation of `ModuleCache::read()`.
 Expected<std::unique_ptr<llvm::MemoryBuffer>>

diff  --git a/clang/lib/DependencyScanning/InProcessModuleCache.cpp b/clang/lib/DependencyScanning/InProcessModuleCache.cpp
index 78510180525305..76cbdddfda5230 100644
--- a/clang/lib/DependencyScanning/InProcessModuleCache.cpp
+++ b/clang/lib/DependencyScanning/InProcessModuleCache.cpp
@@ -57,7 +57,8 @@ void ModuleCacheEntries::flush() {
       off_t Size;
       time_t ModTime;
       // Best-effort: ignore errors (e.g. read-only cache directory).
-      (void)writeImpl(Path, *Entry->Buffer, Size, ModTime);
+      (void)writeImpl(Path, *Entry->Buffer, Size, ModTime,
+                      /*ReadBack=*/false);
     }
   }
 }
@@ -194,27 +195,27 @@ class InProcessModuleCache : public ModuleCache {
     return InMemory;
   }
 
-  std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
-                        off_t &Size, time_t &ModTime) override {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+        time_t &ModTime) override {
     ModuleCacheEntry &Entry = getOrCreateEntry(Path);
     std::lock_guard<std::mutex> Lock(Entry.Mutex);
     Logger.log() << "pcm_write: " << Path;
     if (Entry.State == ModuleCacheEntry::S_Written) {
       assert(Entry.Buffer && "Wrote PCM with no contents");
-      assert(Entry.Buffer->getBuffer() == Buffer.getBuffer() &&
+      assert(Entry.Buffer->getBuffer() == Buffer->getBuffer() &&
              "Wrote the same PCM with 
diff erent contents");
       Size = Entry.Buffer->getBufferSize();
       ModTime = Entry.ModTime;
-      return {};
+      return std::make_unique<SharedMemoryBuffer>(Entry.Buffer);
     }
     // Readers of the previous buffer (if any) keep it alive on their own.
-    Entry.Buffer =
-        llvm::MemoryBuffer::getMemBufferCopy(Buffer.getBuffer(), Path);
+    Entry.Buffer = std::move(Buffer);
     Entry.ModTime = llvm::sys::toTimeT(std::chrono::system_clock::now());
     Entry.State = ModuleCacheEntry::S_Written;
     Size = Entry.Buffer->getBufferSize();
     ModTime = Entry.ModTime;
-    return {};
+    return std::make_unique<SharedMemoryBuffer>(Entry.Buffer);
   }
 
   Expected<std::unique_ptr<llvm::MemoryBuffer>>

diff  --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 53579714fc24af..7d23b43a2fbeda 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1559,15 +1559,17 @@ static bool compileModuleImpl(CompilerInstance &ImportingInstance,
 
   off_t Size;
   time_t ModTime;
-  std::error_code EC = ImportingInstance.getModuleCache().write(
-      ModuleFileName, *Buffer, Size, ModTime);
-  if (EC) {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Written =
+      ImportingInstance.getModuleCache().write(
+          ModuleFileName, std::move(Buffer), Size, ModTime);
+  if (!Written) {
     ImportingInstance.getDiagnostics().Report(ModuleNameLoc,
                                               diag::err_module_not_written)
-        << Module->Name << ModuleFileName << EC.message()
+        << Module->Name << ModuleFileName << Written.getError().message()
         << SourceRange(ImportLoc, ModuleNameLoc);
     return false;
   }
+  Buffer = std::move(*Written);
 
   // The module is built successfully, we can update its timestamp now.
   if (ImportingInstance.getPreprocessor()

diff  --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp
index b98fa656e9b77d..b360ac5f42f163 100644
--- a/clang/lib/Serialization/ModuleCache.cpp
+++ b/clang/lib/Serialization/ModuleCache.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/IOSandbox.h"
 #include "llvm/Support/LockFileManager.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 
 #include <chrono>
@@ -160,8 +161,9 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval,
   }
 }
 
-std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
-                                 off_t &Size, time_t &ModTime) {
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer, off_t &Size,
+                 time_t &ModTime, bool ReadBack) {
   StringRef Extension = llvm::sys::path::extension(Path);
   SmallString<128> ModelPath = StringRef(Path).drop_back(Extension.size());
   ModelPath += "-%%%%%%%%";
@@ -184,6 +186,7 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
   }
 
   llvm::sys::fs::file_status Status;
+  std::unique_ptr<llvm::MemoryBuffer> Written;
   {
     llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
     OS << Buffer.getBuffer();
@@ -192,6 +195,15 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
     // Using the status from an open file descriptor ensures this is not racy.
     if ((EC = llvm::sys::fs::status(FD, Status)))
       return EC;
+    if (ReadBack) {
+      llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf =
+          llvm::MemoryBuffer::getOpenFile(
+              llvm::sys::fs::convertFDToNativeFile(FD), Path, Status.getSize(),
+              /*RequiresNullTerminator=*/false);
+      if (!Buf)
+        return Buf.getError();
+      Written = std::move(*Buf);
+    }
   }
 
   Size = Status.getSize();
@@ -201,7 +213,7 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
   if ((EC = llvm::sys::fs::rename(TmpPath, Path)))
     return EC;
 
-  return {};
+  return std::move(Written);
 }
 
 /// The access time of a module file is only refreshed explicitly when it is
@@ -307,12 +319,13 @@ class CrossProcessModuleCache : public ModuleCache {
     return InMemory;
   }
 
-  std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
-                        off_t &Size, time_t &ModTime) override {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+        time_t &ModTime) override {
     // This is a compiler-internal input/output, let's bypass the sandbox.
     auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
 
-    return writeImpl(Path, Buffer, Size, ModTime);
+    return writeImpl(Path, *Buffer, Size, ModTime, /*ReadBack=*/true);
   }
 
   Expected<std::unique_ptr<llvm::MemoryBuffer>>

diff  --git a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
index 5b667a4989f50e..fd2f24e9851291 100644
--- a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
+++ b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
@@ -33,7 +33,8 @@ TEST(InProcessModuleCache, ReadWriteInvalidation) {
   EXPECT_EQ(Buf->getBuffer(), "original\n");
 
   auto NewBuf = llvm::MemoryBuffer::getMemBufferCopy("modified\n", Path);
-  ASSERT_EQ(ModCache->write(Path, *NewBuf, Size, ModTime), std::error_code{});
+  ASSERT_EQ(ModCache->write(Path, std::move(NewBuf), Size, ModTime).getError(),
+            std::error_code{});
 
   // Writing a new buffer should not invalidate the previously-read buffer.
   EXPECT_EQ(Buf->getBuffer(), "original\n");
@@ -74,3 +75,40 @@ TEST(InProcessModuleCache, ReadReadInvalidation) {
   EXPECT_EQ(Buf1->getBuffer().begin(), Buf2->getBuffer().begin());
   EXPECT_EQ(Buf1->getBuffer().end(), Buf2->getBuffer().end());
 }
+
+TEST(InProcessModuleCache, WriteReturnsSharedBuffer) {
+  ModuleCacheEntries Entries;
+  AtomicLineLogger Logger;
+  std::shared_ptr<ModuleCache> ModCache =
+      makeInProcessModuleCache(Entries, Logger);
+
+  llvm::SmallString<256> Path;
+  llvm::sys::fs::createUniquePath("M-%%%%%%.pcm", Path, /*MakeAbsolute=*/true);
+
+  auto Given = llvm::MemoryBuffer::getMemBuffer("contents\n", Path);
+  const char *GivenBufStart = Given->getBufferStart();
+
+  off_t Size;
+  time_t ModTime;
+  auto WrittenOrErr = ModCache->write(Path, std::move(Given), Size, ModTime);
+  ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
+  auto Written = std::move(*WrittenOrErr);
+  ASSERT_TRUE(Written);
+  EXPECT_EQ(Written->getBuffer(), "contents\n");
+  // Written buffers share the memory of the given buffer.
+  EXPECT_EQ(Written->getBufferStart(), GivenBufStart);
+
+  // Read buffers share the memory of the written buffer.
+  std::unique_ptr<llvm::MemoryBuffer> Read;
+  ASSERT_THAT_ERROR(ModCache->read(Path, Size, ModTime).moveInto(Read),
+                    llvm::Succeeded());
+  EXPECT_EQ(Read->getBufferStart(), Written->getBufferStart());
+
+  // Repeatedly-read buffers share the memory of the buffer written first.
+  auto RewrittenOrErr = ModCache->write(
+      Path, llvm::MemoryBuffer::getMemBufferCopy("contents\n", Path), Size,
+      ModTime);
+  ASSERT_EQ(RewrittenOrErr.getError(), std::error_code{});
+  ASSERT_TRUE(*RewrittenOrErr);
+  EXPECT_EQ((*RewrittenOrErr)->getBufferStart(), Written->getBufferStart());
+}

diff  --git a/clang/unittests/Serialization/ModuleCacheTest.cpp b/clang/unittests/Serialization/ModuleCacheTest.cpp
index df943fef7caba4..ef16e492b6b349 100644
--- a/clang/unittests/Serialization/ModuleCacheTest.cpp
+++ b/clang/unittests/Serialization/ModuleCacheTest.cpp
@@ -338,9 +338,33 @@ TEST_F(ModuleCacheTest, WriteReportsFullSize) {
   StringRef Contents = "contents\n";
   off_t Size;
   time_t ModTime;
-  ASSERT_EQ(writeImpl(Path, MemoryBufferRef(Contents, Path), Size, ModTime),
+  ASSERT_EQ(writeImpl(Path, MemoryBufferRef(Contents, Path), Size, ModTime,
+                      /*ReadBack=*/false)
+                .getError(),
             std::error_code{});
   EXPECT_EQ(Size, off_t(Contents.size()));
 }
 
+// Test that the buffer returned from write() is memory-mapped to the written
+// file (for files large enough to be mmapped).
+TEST_F(ModuleCacheTest, WriteReturnsMappedBuffer) {
+  SmallString<256> Path(ModuleCachePath);
+  sys::path::append(Path, "M.pcm");
+
+  std::string Contents(1024 * 1024, 'x');
+  auto Given = MemoryBuffer::getMemBuffer(Contents, Path);
+  ASSERT_NE(Given->getBufferKind(), MemoryBuffer::MemoryBuffer_MMap);
+
+  off_t Size;
+  time_t ModTime;
+  std::shared_ptr<ModuleCache> ModCache = createCrossProcessModuleCache();
+  auto WrittenOrErr = ModCache->write(Path, std::move(Given), Size, ModTime);
+  ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
+  auto Written = std::move(*WrittenOrErr);
+  ASSERT_TRUE(Written);
+  // Written buffer refers to the mmaped file contents.
+  EXPECT_EQ(Written->getBufferKind(), MemoryBuffer::MemoryBuffer_MMap);
+  EXPECT_EQ(Written->getBuffer(), Contents);
+}
+
 } // anonymous namespace


        


More information about the cfe-commits mailing list