[clang] [clang] Use the module cache's view of built PCMs (PR #242383)
Jan Svoboda via cfe-commits
cfe-commits at lists.llvm.org
Fri Nov 13 17:12:37 PST 2026
https://github.com/jansvoboda11 updated https://github.com/llvm/llvm-project/pull/242383
>From e5d6baa8d8bbdb8378d35d29b78ff06e401dd339 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Thu, 12 Nov 2026 13:07:34 -0800
Subject: [PATCH 1/3] [clang] Use the module cache's view of built PCMs
When building a module implicitly, the compiler hands the PCM contents to
`ModuleCache::write()` and then extracts the AST file from the PCH container
of its own buffer, storing a copy of it in the `InMemoryModuleCache` of the
importing instance.
This PR makes `ModuleCache::write()` return a buffer with the written
contents, which the importing instance now extracts the AST file from. The
in-process module cache returns a buffer sharing ownership of its entry, the
cross-process module cache returns the written file, read back from the same
file descriptor it was written through (memory-mapped for all but small
files). This is a prerequisite for storing the module cache's view in the
`InMemoryModuleCache` without copying it.
To support this, `writeImpl()` can now read the written file back from the
same file descriptor, guaranteeing we map exactly the contents we wrote and
making it immune to concurrent replacement of the file.
`ModuleCache::write()` now also takes ownership of the compiled buffer, making
it clear the caller isn't supposed to use it anymore. The in-process module
cache now stores it without copying, and the cross-process module cache frees
it as soon as it's written.
---
.../include/clang/Serialization/ModuleCache.h | 21 +++++++---
.../InProcessModuleCache.cpp | 17 ++++----
clang/lib/Frontend/CompilerInstance.cpp | 10 +++--
clang/lib/Serialization/ModuleCache.cpp | 25 ++++++++---
.../InProcessModuleCacheTest.cpp | 42 ++++++++++++++++++-
.../Serialization/ModuleCacheTest.cpp | 4 +-
6 files changed, 94 insertions(+), 25 deletions(-)
diff --git a/clang/include/clang/Serialization/ModuleCache.h b/clang/include/clang/Serialization/ModuleCache.h
index 8fdd36911cce9..2faae46926db0 100644
--- a/clang/include/clang/Serialization/ModuleCache.h
+++ b/clang/include/clang/Serialization/ModuleCache.h
@@ -13,6 +13,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/FileSystem/UniqueID.h"
#include <ctime>
@@ -89,9 +90,14 @@ class ModuleCache {
virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
- /// Write the PCM contents to the given path in the module cache.
- virtual std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
- off_t &Size, time_t &ModTime) = 0;
+ /// Write the PCM contents to the given path in the module cache. Takes
+ /// ownership of \p Buffer, so that it can be freed as soon as possible.
+ ///
+ /// Returns a buffer with the written contents, which readers in this process
+ /// should use instead of their own copy.
+ virtual llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+ time_t &ModTime) = 0;
virtual Expected<std::unique_ptr<llvm::MemoryBuffer>>
read(StringRef FileName, off_t &Size, time_t &ModTime) = 0;
@@ -117,8 +123,13 @@ void maybePruneImpl(StringRef Path, time_t PruneInterval, time_t PruneAfter,
llvm::function_ref<void(StringRef)> OnPrune = {});
/// Shared implementation of `ModuleCache::write()`.
-std::error_code writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
- off_t &Size, time_t &ModTime);
+///
+/// If \p ReadBack is true, returns the contents of the written file, read back
+/// from the same file descriptor (and therefore immune to concurrent
+/// replacement of the file at \p Path). Otherwise, returns nullptr on success.
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer, off_t &Size,
+ time_t &ModTime, bool ReadBack);
/// Shared implementation of `ModuleCache::read()`.
Expected<std::unique_ptr<llvm::MemoryBuffer>>
diff --git a/clang/lib/DependencyScanning/InProcessModuleCache.cpp b/clang/lib/DependencyScanning/InProcessModuleCache.cpp
index 7851018052530..76cbdddfda523 100644
--- a/clang/lib/DependencyScanning/InProcessModuleCache.cpp
+++ b/clang/lib/DependencyScanning/InProcessModuleCache.cpp
@@ -57,7 +57,8 @@ void ModuleCacheEntries::flush() {
off_t Size;
time_t ModTime;
// Best-effort: ignore errors (e.g. read-only cache directory).
- (void)writeImpl(Path, *Entry->Buffer, Size, ModTime);
+ (void)writeImpl(Path, *Entry->Buffer, Size, ModTime,
+ /*ReadBack=*/false);
}
}
}
@@ -194,27 +195,27 @@ class InProcessModuleCache : public ModuleCache {
return InMemory;
}
- std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
- off_t &Size, time_t &ModTime) override {
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+ time_t &ModTime) override {
ModuleCacheEntry &Entry = getOrCreateEntry(Path);
std::lock_guard<std::mutex> Lock(Entry.Mutex);
Logger.log() << "pcm_write: " << Path;
if (Entry.State == ModuleCacheEntry::S_Written) {
assert(Entry.Buffer && "Wrote PCM with no contents");
- assert(Entry.Buffer->getBuffer() == Buffer.getBuffer() &&
+ assert(Entry.Buffer->getBuffer() == Buffer->getBuffer() &&
"Wrote the same PCM with different contents");
Size = Entry.Buffer->getBufferSize();
ModTime = Entry.ModTime;
- return {};
+ return std::make_unique<SharedMemoryBuffer>(Entry.Buffer);
}
// Readers of the previous buffer (if any) keep it alive on their own.
- Entry.Buffer =
- llvm::MemoryBuffer::getMemBufferCopy(Buffer.getBuffer(), Path);
+ Entry.Buffer = std::move(Buffer);
Entry.ModTime = llvm::sys::toTimeT(std::chrono::system_clock::now());
Entry.State = ModuleCacheEntry::S_Written;
Size = Entry.Buffer->getBufferSize();
ModTime = Entry.ModTime;
- return {};
+ return std::make_unique<SharedMemoryBuffer>(Entry.Buffer);
}
Expected<std::unique_ptr<llvm::MemoryBuffer>>
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 7e9d43c418a0f..b66dcc158c9be 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1562,15 +1562,17 @@ static bool compileModuleImpl(CompilerInstance &ImportingInstance,
off_t Size;
time_t ModTime;
- std::error_code EC = ImportingInstance.getModuleCache().write(
- ModuleFileName, *Buffer, Size, ModTime);
- if (EC) {
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Written =
+ ImportingInstance.getModuleCache().write(
+ ModuleFileName, std::move(Buffer), Size, ModTime);
+ if (!Written) {
ImportingInstance.getDiagnostics().Report(ModuleNameLoc,
diag::err_module_not_written)
- << Module->Name << ModuleFileName << EC.message()
+ << Module->Name << ModuleFileName << Written.getError().message()
<< SourceRange(ImportLoc, ModuleNameLoc);
return false;
}
+ Buffer = std::move(*Written);
// The module is built successfully, we can update its timestamp now.
if (ImportingInstance.getPreprocessor()
diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp
index b98fa656e9b77..b360ac5f42f16 100644
--- a/clang/lib/Serialization/ModuleCache.cpp
+++ b/clang/lib/Serialization/ModuleCache.cpp
@@ -16,6 +16,7 @@
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/IOSandbox.h"
#include "llvm/Support/LockFileManager.h"
+#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include <chrono>
@@ -160,8 +161,9 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval,
}
}
-std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
- off_t &Size, time_t &ModTime) {
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer, off_t &Size,
+ time_t &ModTime, bool ReadBack) {
StringRef Extension = llvm::sys::path::extension(Path);
SmallString<128> ModelPath = StringRef(Path).drop_back(Extension.size());
ModelPath += "-%%%%%%%%";
@@ -184,6 +186,7 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
}
llvm::sys::fs::file_status Status;
+ std::unique_ptr<llvm::MemoryBuffer> Written;
{
llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
OS << Buffer.getBuffer();
@@ -192,6 +195,15 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
// Using the status from an open file descriptor ensures this is not racy.
if ((EC = llvm::sys::fs::status(FD, Status)))
return EC;
+ if (ReadBack) {
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf =
+ llvm::MemoryBuffer::getOpenFile(
+ llvm::sys::fs::convertFDToNativeFile(FD), Path, Status.getSize(),
+ /*RequiresNullTerminator=*/false);
+ if (!Buf)
+ return Buf.getError();
+ Written = std::move(*Buf);
+ }
}
Size = Status.getSize();
@@ -201,7 +213,7 @@ std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer,
if ((EC = llvm::sys::fs::rename(TmpPath, Path)))
return EC;
- return {};
+ return std::move(Written);
}
/// The access time of a module file is only refreshed explicitly when it is
@@ -307,12 +319,13 @@ class CrossProcessModuleCache : public ModuleCache {
return InMemory;
}
- std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer,
- off_t &Size, time_t &ModTime) override {
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ write(StringRef Path, std::unique_ptr<llvm::MemoryBuffer> Buffer, off_t &Size,
+ time_t &ModTime) override {
// This is a compiler-internal input/output, let's bypass the sandbox.
auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
- return writeImpl(Path, Buffer, Size, ModTime);
+ return writeImpl(Path, *Buffer, Size, ModTime, /*ReadBack=*/true);
}
Expected<std::unique_ptr<llvm::MemoryBuffer>>
diff --git a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
index 5b667a4989f50..665c39b49f73e 100644
--- a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
+++ b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
@@ -33,7 +33,8 @@ TEST(InProcessModuleCache, ReadWriteInvalidation) {
EXPECT_EQ(Buf->getBuffer(), "original\n");
auto NewBuf = llvm::MemoryBuffer::getMemBufferCopy("modified\n", Path);
- ASSERT_EQ(ModCache->write(Path, *NewBuf, Size, ModTime), std::error_code{});
+ ASSERT_EQ(ModCache->write(Path, std::move(NewBuf), Size, ModTime).getError(),
+ std::error_code{});
// Writing a new buffer should not invalidate the previously-read buffer.
EXPECT_EQ(Buf->getBuffer(), "original\n");
@@ -74,3 +75,42 @@ TEST(InProcessModuleCache, ReadReadInvalidation) {
EXPECT_EQ(Buf1->getBuffer().begin(), Buf2->getBuffer().begin());
EXPECT_EQ(Buf1->getBuffer().end(), Buf2->getBuffer().end());
}
+
+TEST(InProcessModuleCache, WriteReturnsSharedBuffer) {
+ ModuleCacheEntries Entries;
+ AtomicLineLogger Logger;
+ std::shared_ptr<ModuleCache> ModCache =
+ makeInProcessModuleCache(Entries, Logger);
+
+ llvm::SmallString<256> Path;
+ llvm::sys::fs::createUniquePath("M-%%%%%%.pcm", Path, /*MakeAbsolute=*/true);
+
+ off_t Size;
+ time_t ModTime;
+ std::unique_ptr<llvm::MemoryBuffer> Written;
+ {
+ std::string Contents = "contents\n";
+ auto WrittenOrErr = ModCache->write(
+ Path, llvm::MemoryBuffer::getMemBufferCopy(Contents, Path), Size,
+ ModTime);
+ ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
+ Written = std::move(*WrittenOrErr);
+ }
+ // The returned buffer doesn't reference the written contents.
+ ASSERT_TRUE(Written);
+ EXPECT_EQ(Written->getBuffer(), "contents\n");
+
+ // Reads share the memory of the written buffer.
+ std::unique_ptr<llvm::MemoryBuffer> Read;
+ ASSERT_THAT_ERROR(ModCache->read(Path, Size, ModTime).moveInto(Read),
+ llvm::Succeeded());
+ EXPECT_EQ(Read->getBufferStart(), Written->getBufferStart());
+
+ // Writing the same PCM again shares the memory of the first write.
+ auto RewrittenOrErr = ModCache->write(
+ Path, llvm::MemoryBuffer::getMemBufferCopy("contents\n", Path), Size,
+ ModTime);
+ ASSERT_EQ(RewrittenOrErr.getError(), std::error_code{});
+ ASSERT_TRUE(*RewrittenOrErr);
+ EXPECT_EQ((*RewrittenOrErr)->getBufferStart(), Written->getBufferStart());
+}
diff --git a/clang/unittests/Serialization/ModuleCacheTest.cpp b/clang/unittests/Serialization/ModuleCacheTest.cpp
index df943fef7caba..79cda42872ea5 100644
--- a/clang/unittests/Serialization/ModuleCacheTest.cpp
+++ b/clang/unittests/Serialization/ModuleCacheTest.cpp
@@ -338,7 +338,9 @@ TEST_F(ModuleCacheTest, WriteReportsFullSize) {
StringRef Contents = "contents\n";
off_t Size;
time_t ModTime;
- ASSERT_EQ(writeImpl(Path, MemoryBufferRef(Contents, Path), Size, ModTime),
+ ASSERT_EQ(writeImpl(Path, MemoryBufferRef(Contents, Path), Size, ModTime,
+ /*ReadBack=*/false)
+ .getError(),
std::error_code{});
EXPECT_EQ(Size, off_t(Contents.size()));
}
>From c78c52d170412084df8b8ee554d8b15e402d33ad Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Fri, 13 Nov 2026 16:57:41 -0800
Subject: [PATCH 2/3] Simplify test
---
.../InProcessModuleCacheTest.cpp | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
diff --git a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
index 665c39b49f73e..69b3c78814b07 100644
--- a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
+++ b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
@@ -87,16 +87,11 @@ TEST(InProcessModuleCache, WriteReturnsSharedBuffer) {
off_t Size;
time_t ModTime;
- std::unique_ptr<llvm::MemoryBuffer> Written;
- {
- std::string Contents = "contents\n";
- auto WrittenOrErr = ModCache->write(
- Path, llvm::MemoryBuffer::getMemBufferCopy(Contents, Path), Size,
- ModTime);
- ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
- Written = std::move(*WrittenOrErr);
- }
- // The returned buffer doesn't reference the written contents.
+ auto WrittenOrErr = ModCache->write(
+ Path, llvm::MemoryBuffer::getMemBuffer("contents\n", Path), Size,
+ ModTime);
+ ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
+ auto Written = std::move(*WrittenOrErr);
ASSERT_TRUE(Written);
EXPECT_EQ(Written->getBuffer(), "contents\n");
>From 424b675c1999fe99ca0bd219c3e09fbc80c85161 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda at apple.com>
Date: Fri, 13 Nov 2026 17:12:21 -0800
Subject: [PATCH 3/3] Tweak assertions
---
.../DependencyScanning/InProcessModuleCacheTest.cpp | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
index 69b3c78814b07..fd2f24e985129 100644
--- a/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
+++ b/clang/unittests/DependencyScanning/InProcessModuleCacheTest.cpp
@@ -85,23 +85,26 @@ TEST(InProcessModuleCache, WriteReturnsSharedBuffer) {
llvm::SmallString<256> Path;
llvm::sys::fs::createUniquePath("M-%%%%%%.pcm", Path, /*MakeAbsolute=*/true);
+ auto Given = llvm::MemoryBuffer::getMemBuffer("contents\n", Path);
+ const char *GivenBufStart = Given->getBufferStart();
+
off_t Size;
time_t ModTime;
- auto WrittenOrErr = ModCache->write(
- Path, llvm::MemoryBuffer::getMemBuffer("contents\n", Path), Size,
- ModTime);
+ auto WrittenOrErr = ModCache->write(Path, std::move(Given), Size, ModTime);
ASSERT_EQ(WrittenOrErr.getError(), std::error_code{});
auto Written = std::move(*WrittenOrErr);
ASSERT_TRUE(Written);
EXPECT_EQ(Written->getBuffer(), "contents\n");
+ // Written buffers share the memory of the given buffer.
+ EXPECT_EQ(Written->getBufferStart(), GivenBufStart);
- // Reads share the memory of the written buffer.
+ // Read buffers share the memory of the written buffer.
std::unique_ptr<llvm::MemoryBuffer> Read;
ASSERT_THAT_ERROR(ModCache->read(Path, Size, ModTime).moveInto(Read),
llvm::Succeeded());
EXPECT_EQ(Read->getBufferStart(), Written->getBufferStart());
- // Writing the same PCM again shares the memory of the first write.
+ // Repeatedly-read buffers share the memory of the buffer written first.
auto RewrittenOrErr = ModCache->write(
Path, llvm::MemoryBuffer::getMemBufferCopy("contents\n", Path), Size,
ModTime);
More information about the cfe-commits
mailing list