[clang] [clang] Use the module cache's view of built PCMs (PR #242383)

Jan Svoboda via cfe-commits cfe-commits at lists.llvm.org
Fri Nov 13 17:12:37 PST 2026


https://github.com/jansvoboda11 updated https://github.com/llvm/llvm-project/pull/242383

>From e5d6baa8d8bbdb8378d35d29b78ff06e401dd339 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Thu, 12 Nov 2026 13:07:34 -0800
Subject: [PATCH 1/3] [clang] Use the module cache's view of built PCMs

When building a module implicitly, the compiler hands the PCM contents to
`ModuleCache::write()` and then extracts the AST file from the PCH container
of its own buffer, storing a copy of it in the `InMemoryModuleCache` of the
importing instance.

This PR makes `ModuleCache::write()` return a buffer with the written
contents, which the importing instance now extracts the AST file from. The
in-process module cache returns a buffer sharing ownership of its entry, the
cross-process module cache returns the written file, read back from the same
file descriptor it was written through (memory-mapped for all but small
files). This is a prerequisite for storing the module cache's view in the
`InMemoryModuleCache` without copying it.

To support this, `writeImpl()` can now read the written file back from the
same file descriptor, guaranteeing we map exactly the contents we wrote and
making it immune to concurrent replacement of the file.

`ModuleCache::write()` now also takes ownership of the compiled buffer, making
it clear the caller isn't supposed to use it anymore. The in-process module
cache now stores it without copying, and the cross-process module cache frees
it as soon as it's written.
---
 .../include/clang/Serialization/ModuleCache.h | 21 +++++++---
 .../InProcessModuleCache.cpp                  | 17 ++++----
 clang/lib/Frontend/CompilerInstance.cpp       | 10 +++--
 clang/lib/Serialization/ModuleCache.cpp       | 25 ++++++++---
 .../InProcessModuleCacheTest.cpp              | 42 ++++++++++++++++++-
 .../Serialization/ModuleCacheTest.cpp         |  4 +-
 6 files changed, 94 insertions(+), 25 deletions(-)

diff --git a/clang/include/clang/Serialization/ModuleCache.h b/clang/include/clang/Serialization/ModuleCache.h
index 8fdd36911cce9..2faae46926db0 100644
--- a/clang/include/clang/Serialization/ModuleCache.h
+++ b/clang/include/clang/Serialization/ModuleCache.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem/UniqueID.h"
 
 #include <ctime>
@@ -89,9 +90,14 @@ class ModuleCache {
   virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
   virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
 
-  /// Write the PCM contents to the given path in the module cache.
-  virtual std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
-                                off_t &Size, time_t &ModTime) = 0;
+  /// Write the PCM contents to the given path in the module cache. Takes
+  /// ownership of \p Buffer, so that it can be freed as soon as possible.
+  ///
+  /// Returns a buffer with the written contents, which readers in this process
+  /// should use instead of their own copy.
+  virtual llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+        time_t &ModTime) = 0;
 
   virtual Expected<std::unique_ptr<llvm::MemoryBuffer>>
   read(StringRef FileName, off_t &Size, time_t &ModTime) = 0;
@@ -117,8 +123,13 @@ void maybePruneImpl(StringRef Path, time_t PruneInterval, time_t PruneAfter,
                     llvm::function_ref<void(StringRef)> OnPrune = {});
 
 /// Shared implementation of `ModuleCache::write()`.
-std::error_code writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
-                          off_t &Size, time_t &ModTime);
+///
+/// If \p ReadBack is true, returns the contents of the written file, read back
+/// from the same file descriptor (and therefore immune to concurrent
+/// replacement of the file at \p Path). Otherwise, returns nullptr on success.
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer, off_t &Size,
+          time_t &ModTime, bool ReadBack);
 
 /// Shared implementation of `ModuleCache::read()`.
 Expected<std::unique_ptr<llvm::MemoryBuffer>>
diff --git a/clang/lib/DependencyScanning/InProcessModuleCache.cpp b/clang/lib/DependencyScanning/InProcessModuleCache.cpp
index 7851018052530..76cbdddfda523 100644
--- a/clang/lib/DependencyScanning/InProcessModuleCache.cpp
+++ b/clang/lib/DependencyScanning/InProcessModuleCache.cpp
@@ -57,7 +57,8 @@ void ModuleCacheEntries::flush() {
       off_t Size;
       time_t ModTime;
       // Best-effort: ignore errors (e.g. read-only cache directory).
-      (void)writeImpl(Path, *Entry->Buffer, Size, ModTime);
+      (void)writeImpl(Path, *Entry->Buffer, Size, ModTime,
+                      /*ReadBack=*/false);
     }
   }
 }
@@ -194,27 +195,27 @@ class InProcessModuleCache : public ModuleCache {
     return InMemory;
   }
 
-  std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
-                        off_t &Size, time_t &ModTime) override {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+        time_t &ModTime) override {
     ModuleCacheEntry &Entry = getOrCreateEntry(Path);
     std::lock_guard<std::mutex> Lock(Entry.Mutex);
     Logger.log() << "pcm_write: " << Path;
     if (Entry.State == ModuleCacheEntry::S_Written) {
       assert(Entry.Buffer && "Wrote PCM with no contents");
-      assert(Entry.Buffer->getBuffer() == Buffer.getBuffer() &&
+      assert(Entry.Buffer->getBuffer() == Buffer->getBuffer() &&
              "Wrote the same PCM with different contents");
       Size = Entry.Buffer->getBufferSize();
       ModTime = Entry.ModTime;
-      return {};
+      return std::make_unique<SharedMemoryBuffer>(Entry.Buffer);
     }
     // Readers of the previous buffer (if any) keep it alive on their own.
-    Entry.Buffer =
-        llvm::MemoryBuffer::getMemBufferCopy(Buffer.getBuffer(), Path);
+    Entry.Buffer = std::move(Buffer);
     Entry.ModTime = llvm::sys::toTimeT(std::chrono::system_clock::now());
     Entry.State = ModuleCacheEntry::S_Written;
     Size = Entry.Buffer->getBufferSize();
     ModTime = Entry.ModTime;
-    return {};
+    return std::make_unique<SharedMemoryBuffer>(Entry.Buffer);
   }
 
   Expected<std::unique_ptr<llvm::MemoryBuffer>>
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 7e9d43c418a0f..b66dcc158c9be 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1562,15 +1562,17 @@ static bool compileModuleImpl(CompilerInstance &ImportingInstance,
 
   off_t Size;
   time_t ModTime;
-  std::error_code EC = ImportingInstance.getModuleCache().write(
-      ModuleFileName, *Buffer, Size, ModTime);
-  if (EC) {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Written =
+      ImportingInstance.getModuleCache().write(
+          ModuleFileName, std::move(Buffer), Size, ModTime);
+  if (!Written) {
     ImportingInstance.getDiagnostics().Report(ModuleNameLoc,
                                               diag::err_module_not_written)
-        << Module->Name << ModuleFileName << EC.message()
+        << Module->Name << ModuleFileName << Written.getError().message()
         << SourceRange(ImportLoc, ModuleNameLoc);
     return false;
   }
+  Buffer = std::move(*Written);
 
   // The module is built successfully, we can update its timestamp now.
   if (ImportingInstance.getPreprocessor()
diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp
index b98fa656e9b77..b360ac5f42f16 100644
--- a/clang/lib/Serialization/ModuleCache.cpp
+++ b/clang/lib/Serialization/ModuleCache.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/IOSandbox.h"
 #include "llvm/Support/LockFileManager.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 
 #include <chrono>
@@ -160,8 +161,9 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval,
   }
 }
 
-std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
-                                 off_t &Size, time_t &ModTime) {
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer, off_t &Size,
+                 time_t &ModTime, bool ReadBack) {
   StringRef Extension = llvm::sys::path::extension(Path);
   SmallString<128> ModelPath = StringRef(Path).drop_back(Extension.size());
   ModelPath += "-%%%%%%%%";
@@ -184,6 +186,7 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
   }
 
   llvm::sys::fs::file_status Status;
+  std::unique_ptr<llvm::MemoryBuffer> Written;
   {
     llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
     OS << Buffer.getBuffer();
@@ -192,6 +195,15 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
     // Using the status from an open file descriptor ensures this is not racy.
     if ((EC = llvm::sys::fs::status(FD, Status)))
       return EC;
+    if (ReadBack) {
+      llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf =
+          llvm::MemoryBuffer::getOpenFile(
+              llvm::sys::fs::convertFDToNativeFile(FD), Path, Status.getSize(),
+              /*RequiresNullTerminator=*/false);
+      if (!Buf)
+        return Buf.getError();
+      Written = std::move(*Buf);
+    }
   }
 
   Size = Status.getSize();
@@ -201,7 +213,7 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
   if ((EC = llvm::sys::fs::rename(TmpPath, Path)))
     return EC;
 
-  return {};
+  return std::move(Written);
 }
 
 /// The access time of a module file is only refreshed explicitly when it is
@@ -307,12 +319,13 @@ class CrossProcessModuleCache : public ModuleCache {
     return InMemory;
   }
 
-  std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
-                        off_t &Size, time_t &ModTime) override {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+        time_t &ModTime) override {
     // This is a compiler-internal input/output, let's bypass the sandbox.
     auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
 
-    return writeImpl(Path, Buffer, Size, ModTime);
+    return writeImpl(Path, *Buffer, Size, ModTime, /*ReadBack=*/true);
   }
 
   Expected<std::unique_ptr<llvm::MemoryBuffer>>
diff --git a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
index 5b667a4989f50..665c39b49f73e 100644
--- a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
+++ b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
@@ -33,7 +33,8 @@ TEST(InProcessModuleCache, ReadWriteInvalidation) {
   EXPECT_EQ(Buf->getBuffer(), "original\n");
 
   auto NewBuf = llvm::MemoryBuffer::getMemBufferCopy("modified\n", Path);
-  ASSERT_EQ(ModCache->write(Path, *NewBuf, Size, ModTime), std::error_code{});
+  ASSERT_EQ(ModCache->write(Path, std::move(NewBuf), Size, ModTime).getError(),
+            std::error_code{});
 
   // Writing a new buffer should not invalidate the previously-read buffer.
   EXPECT_EQ(Buf->getBuffer(), "original\n");
@@ -74,3 +75,42 @@ TEST(InProcessModuleCache, ReadReadInvalidation) {
   EXPECT_EQ(Buf1->getBuffer().begin(), Buf2->getBuffer().begin());
   EXPECT_EQ(Buf1->getBuffer().end(), Buf2->getBuffer().end());
 }
+
+TEST(InProcessModuleCache, WriteReturnsSharedBuffer) {
+  ModuleCacheEntries Entries;
+  AtomicLineLogger Logger;
+  std::shared_ptr<ModuleCache> ModCache =
+      makeInProcessModuleCache(Entries, Logger);
+
+  llvm::SmallString<256> Path;
+  llvm::sys::fs::createUniquePath("M-%%%%%%.pcm", Path, /*MakeAbsolute=*/true);
+
+  off_t Size;
+  time_t ModTime;
+  std::unique_ptr<llvm::MemoryBuffer> Written;
+  {
+    std::string Contents = "contents\n";
+    auto WrittenOrErr = ModCache->write(
+        Path, llvm::MemoryBuffer::getMemBufferCopy(Contents, Path), Size,
+        ModTime);
+    ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
+    Written = std::move(*WrittenOrErr);
+  }
+  // The returned buffer doesn't reference the written contents.
+  ASSERT_TRUE(Written);
+  EXPECT_EQ(Written->getBuffer(), "contents\n");
+
+  // Reads share the memory of the written buffer.
+  std::unique_ptr<llvm::MemoryBuffer> Read;
+  ASSERT_THAT_ERROR(ModCache->read(Path, Size, ModTime).moveInto(Read),
+                    llvm::Succeeded());
+  EXPECT_EQ(Read->getBufferStart(), Written->getBufferStart());
+
+  // Writing the same PCM again shares the memory of the first write.
+  auto RewrittenOrErr = ModCache->write(
+      Path, llvm::MemoryBuffer::getMemBufferCopy("contents\n", Path), Size,
+      ModTime);
+  ASSERT_EQ(RewrittenOrErr.getError(), std::error_code{});
+  ASSERT_TRUE(*RewrittenOrErr);
+  EXPECT_EQ((*RewrittenOrErr)->getBufferStart(), Written->getBufferStart());
+}
diff --git a/clang/unittests/Serialization/ModuleCacheTest.cpp b/clang/unittests/Serialization/ModuleCacheTest.cpp
index df943fef7caba..79cda42872ea5 100644
--- a/clang/unittests/Serialization/ModuleCacheTest.cpp
+++ b/clang/unittests/Serialization/ModuleCacheTest.cpp
@@ -338,7 +338,9 @@ TEST_F(ModuleCacheTest, WriteReportsFullSize) {
   StringRef Contents = "contents\n";
   off_t Size;
   time_t ModTime;
-  ASSERT_EQ(writeImpl(Path, MemoryBufferRef(Contents, Path), Size, ModTime),
+  ASSERT_EQ(writeImpl(Path, MemoryBufferRef(Contents, Path), Size, ModTime,
+                      /*ReadBack=*/false)
+                .getError(),
             std::error_code{});
   EXPECT_EQ(Size, off_t(Contents.size()));
 }

>From c78c52d170412084df8b8ee554d8b15e402d33ad Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Fri, 13 Nov 2026 16:57:41 -0800
Subject: [PATCH 2/3] Simplify test

---
 .../InProcessModuleCacheTest.cpp                  | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
index 665c39b49f73e..69b3c78814b07 100644
--- a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
+++ b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
@@ -87,16 +87,11 @@ TEST(InProcessModuleCache, WriteReturnsSharedBuffer) {
 
   off_t Size;
   time_t ModTime;
-  std::unique_ptr<llvm::MemoryBuffer> Written;
-  {
-    std::string Contents = "contents\n";
-    auto WrittenOrErr = ModCache->write(
-        Path, llvm::MemoryBuffer::getMemBufferCopy(Contents, Path), Size,
-        ModTime);
-    ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
-    Written = std::move(*WrittenOrErr);
-  }
-  // The returned buffer doesn't reference the written contents.
+  auto WrittenOrErr = ModCache->write(
+      Path, llvm::MemoryBuffer::getMemBuffer("contents\n", Path), Size,
+      ModTime);
+  ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
+  auto Written = std::move(*WrittenOrErr);
   ASSERT_TRUE(Written);
   EXPECT_EQ(Written->getBuffer(), "contents\n");
 

>From 424b675c1999fe99ca0bd219c3e09fbc80c85161 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Fri, 13 Nov 2026 17:12:21 -0800
Subject: [PATCH 3/3] Tweak assertions

---
 .../DependencyScanning/InProcessModuleCacheTest.cpp | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
index 69b3c78814b07..fd2f24e985129 100644
--- a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
+++ b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
@@ -85,23 +85,26 @@ TEST(InProcessModuleCache, WriteReturnsSharedBuffer) {
   llvm::SmallString<256> Path;
   llvm::sys::fs::createUniquePath("M-%%%%%%.pcm", Path, /*MakeAbsolute=*/true);
 
+  auto Given = llvm::MemoryBuffer::getMemBuffer("contents\n", Path);
+  const char *GivenBufStart = Given->getBufferStart();
+
   off_t Size;
   time_t ModTime;
-  auto WrittenOrErr = ModCache->write(
-      Path, llvm::MemoryBuffer::getMemBuffer("contents\n", Path), Size,
-      ModTime);
+  auto WrittenOrErr = ModCache->write(Path, std::move(Given), Size, ModTime);
   ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
   auto Written = std::move(*WrittenOrErr);
   ASSERT_TRUE(Written);
   EXPECT_EQ(Written->getBuffer(), "contents\n");
+  // Written buffers share the memory of the given buffer.
+  EXPECT_EQ(Written->getBufferStart(), GivenBufStart);
 
-  // Reads share the memory of the written buffer.
+  // Read buffers share the memory of the written buffer.
   std::unique_ptr<llvm::MemoryBuffer> Read;
   ASSERT_THAT_ERROR(ModCache->read(Path, Size, ModTime).moveInto(Read),
                     llvm::Succeeded());
   EXPECT_EQ(Read->getBufferStart(), Written->getBufferStart());
 
-  // Writing the same PCM again shares the memory of the first write.
+  // Repeatedly-read buffers share the memory of the buffer written first.
   auto RewrittenOrErr = ModCache->write(
       Path, llvm::MemoryBuffer::getMemBufferCopy("contents\n", Path), Size,
       ModTime);



More information about the cfe-commits mailing list