[lld] [llvm] [lld][MachO] Follow-up to use madvise() for threaded file page-in. (PR #157917)
John Holdsworth via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 12 01:08:48 PDT 2025
https://github.com/johnno1962 updated https://github.com/llvm/llvm-project/pull/157917
>From 65e433cf48c96cc08d65e51f384bde4f17c067e5 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github at johnholdsworth.com>
Date: Wed, 10 Sep 2025 20:03:03 +0200
Subject: [PATCH 1/7] Switch to use madvise() to page-in files.
---
lld/MachO/Driver.cpp | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 3db638e1ead96..2495d268cfb71 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -53,6 +53,10 @@
#include "llvm/TextAPI/Architecture.h"
#include "llvm/TextAPI/PackedVersion.h"
+#if !_WIN32
+#include <sys/mman.h>
+#endif
+
using namespace llvm;
using namespace llvm::MachO;
using namespace llvm::object;
@@ -334,11 +338,10 @@ class SerialBackgroundQueue {
// This code forces the page-ins on multiple threads so
// the process is not stalled waiting on disk buffer i/o.
void multiThreadedPageInBackground(DeferredFiles &deferred) {
- static const size_t pageSize = Process::getPageSizeEstimate();
static const size_t largeArchive = 10 * 1024 * 1024;
#ifndef NDEBUG
using namespace std::chrono;
- std::atomic_int numDeferedFilesTouched = 0;
+ std::atomic_int numDeferedFilesAdvised = 0;
static std::atomic_uint64_t totalBytes = 0;
auto t0 = high_resolution_clock::now();
#endif
@@ -349,13 +352,19 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) {
return;
#ifndef NDEBUG
totalBytes += buff.size();
- numDeferedFilesTouched += 1;
+ numDeferedFilesAdvised += 1;
#endif
+#if _WIN32
+ static const size_t pageSize = Process::getPageSizeEstimate();
// Reference all file's mmap'd pages to load them into memory.
for (const char *page = buff.data(), *end = page + buff.size(); page < end;
page += pageSize)
LLVM_ATTRIBUTE_UNUSED volatile char t = *page;
+#else
+ // Advise that mmap'd files should be loaded into memory.
+ madvise((void *)buff.data(), buff.size(), MADV_WILLNEED);
+#endif
};
#if LLVM_ENABLE_THREADS
{ // Create scope for waiting for the taskGroup
@@ -376,7 +385,7 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) {
auto dt = high_resolution_clock::now() - t0;
if (Process::GetEnv("LLD_MULTI_THREAD_PAGE"))
llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
- << numDeferedFilesTouched << "/" << deferred.size() << "/"
+ << numDeferedFilesAdvised << "/" << deferred.size() << "/"
<< duration_cast<milliseconds>(dt).count() / 1000. << "\n";
#endif
}
>From 59e77f5a5d7e09cf30a900d452ba0a15650a6588 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github at johnholdsworth.com>
Date: Thu, 11 Sep 2025 07:37:54 +0200
Subject: [PATCH 2/7] Response to review.
---
lld/MachO/Driver.cpp | 27 ++++++++++++++++-----------
1 file changed, 16 insertions(+), 11 deletions(-)
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 2495d268cfb71..03f1b4038beeb 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -338,34 +338,37 @@ class SerialBackgroundQueue {
// This code forces the page-ins on multiple threads so
// the process is not stalled waiting on disk buffer i/o.
void multiThreadedPageInBackground(DeferredFiles &deferred) {
- static const size_t largeArchive = 10 * 1024 * 1024;
-#ifndef NDEBUG
using namespace std::chrono;
- std::atomic_int numDeferedFilesAdvised = 0;
+ static const size_t pageSize = Process::getPageSizeEstimate();
+ static const size_t largeArchive = 10 * 1024 * 1024;
static std::atomic_uint64_t totalBytes = 0;
+ std::atomic_int numDeferedFilesAdvised = 0;
auto t0 = high_resolution_clock::now();
-#endif
auto preloadDeferredFile = [&](const DeferredFile &deferredFile) {
const StringRef &buff = deferredFile.buffer.getBuffer();
if (buff.size() > largeArchive)
return;
-#ifndef NDEBUG
+ if (((uintptr_t)buff.data() & (pageSize - 1)))
+ return; // Not mmap()'d (not page aligned).
+
totalBytes += buff.size();
numDeferedFilesAdvised += 1;
-#endif
#if _WIN32
- static const size_t pageSize = Process::getPageSizeEstimate();
// Reference all file's mmap'd pages to load them into memory.
for (const char *page = buff.data(), *end = page + buff.size(); page < end;
page += pageSize)
LLVM_ATTRIBUTE_UNUSED volatile char t = *page;
#else
- // Advise that mmap'd files should be loaded into memory.
- madvise((void *)buff.data(), buff.size(), MADV_WILLNEED);
+ if (madvise((void *)buff.data(), buff.size(), MADV_WILLNEED) < 0)
+#ifndef NDEBUG
+ llvm::errs() << "madvise() error " << strerror(errno)
+#endif
+ ;
#endif
};
+
#if LLVM_ENABLE_THREADS
{ // Create scope for waiting for the taskGroup
std::atomic_size_t index = 0;
@@ -380,14 +383,16 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) {
}
});
}
+#else
+ for (const auto &file : deferred)
+ preloadDeferredFile(file);
#endif
-#ifndef NDEBUG
+
auto dt = high_resolution_clock::now() - t0;
if (Process::GetEnv("LLD_MULTI_THREAD_PAGE"))
llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
<< numDeferedFilesAdvised << "/" << deferred.size() << "/"
<< duration_cast<milliseconds>(dt).count() / 1000. << "\n";
-#endif
}
static void multiThreadedPageIn(const DeferredFiles &deferred) {
>From 298380eb921bae4f2507aebf5b59611d76a41117 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github at johnholdsworth.com>
Date: Thu, 11 Sep 2025 08:53:49 +0200
Subject: [PATCH 3/7] Use LLVM_DEBUG.
---
lld/MachO/Driver.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 03f1b4038beeb..ee0c7ec80f993 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -41,6 +41,7 @@
#include "llvm/Object/Archive.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Parallel.h"
#include "llvm/Support/Path.h"
@@ -361,11 +362,10 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) {
page += pageSize)
LLVM_ATTRIBUTE_UNUSED volatile char t = *page;
#else
+#define DEBUG_TYPE "lld-madvise"
if (madvise((void *)buff.data(), buff.size(), MADV_WILLNEED) < 0)
-#ifndef NDEBUG
- llvm::errs() << "madvise() error " << strerror(errno)
-#endif
- ;
+ LLVM_DEBUG(llvm::dbgs() << "madvise() error: " << strerror(errno));
+#undef DEBUG_TYPE
#endif
};
>From e31984b250c3fdd7980efdfe5683c155b409d826 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github at johnholdsworth.com>
Date: Thu, 11 Sep 2025 17:19:19 +0200
Subject: [PATCH 4/7] Page align buffer pointer and address 2 second
performance regression introduced by:
https://github.com/llvm/llvm-project/commit/85cd3d98686c47d015dbcc17f1f7d0714b00e172
---
lld/MachO/Driver.cpp | 7 +++----
lld/MachO/InputFiles.cpp | 3 ++-
llvm/lib/Object/Archive.cpp | 3 ++-
3 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index ee0c7ec80f993..f652e195ffc5e 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -350,8 +350,6 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) {
const StringRef &buff = deferredFile.buffer.getBuffer();
if (buff.size() > largeArchive)
return;
- if (((uintptr_t)buff.data() & (pageSize - 1)))
- return; // Not mmap()'d (not page aligned).
totalBytes += buff.size();
numDeferedFilesAdvised += 1;
@@ -363,8 +361,9 @@ void multiThreadedPageInBackground(DeferredFiles &deferred) {
LLVM_ATTRIBUTE_UNUSED volatile char t = *page;
#else
#define DEBUG_TYPE "lld-madvise"
- if (madvise((void *)buff.data(), buff.size(), MADV_WILLNEED) < 0)
- LLVM_DEBUG(llvm::dbgs() << "madvise() error: " << strerror(errno));
+ auto aligned = llvm::alignAddr(buff.data(), Align(pageSize));
+ if (madvise((void *)aligned, buff.size(), MADV_WILLNEED) < 0)
+ LLVM_DEBUG(llvm::dbgs() << "madvise error: " << strerror(errno) << "\n");
#undef DEBUG_TYPE
#endif
};
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index 442fc608865d2..29a83eaeb0da5 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -217,7 +217,8 @@ std::optional<MemoryBufferRef> macho::readFile(StringRef path) {
if (entry != cachedReads.end())
return entry->second;
- ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr = MemoryBuffer::getFile(path);
+ ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr =
+ MemoryBuffer::getFile(path, false, /*RequiresNullTerminator*/ false);
if (std::error_code ec = mbOrErr.getError()) {
error("cannot open " + path + ": " + ec.message());
return std::nullopt;
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index 92f31c909efd4..445bd6b4609d8 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -584,7 +584,8 @@ Expected<StringRef> Archive::Child::getBuffer() const {
if (!FullNameOrErr)
return FullNameOrErr.takeError();
const std::string &FullName = *FullNameOrErr;
- ErrorOr<std::unique_ptr<MemoryBuffer>> Buf = MemoryBuffer::getFile(FullName);
+ ErrorOr<std::unique_ptr<MemoryBuffer>> Buf =
+ MemoryBuffer::getFile(FullName, false, /*RequiresNullTerminator*/ false);
if (std::error_code EC = Buf.getError())
return errorCodeToError(EC);
Parent->ThinBuffers.push_back(std::move(*Buf));
>From 884b771e18b305b6d15b789dbb40d8025798e7cc Mon Sep 17 00:00:00 2001
From: John Holdsworth <github at johnholdsworth.com>
Date: Thu, 11 Sep 2025 20:25:52 +0200
Subject: [PATCH 5/7] Alternative solution to performance regression.
---
lld/MachO/InputFiles.cpp | 3 +--
llvm/lib/Object/Archive.cpp | 3 +--
llvm/lib/Support/MemoryBuffer.cpp | 6 ++++--
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index 29a83eaeb0da5..442fc608865d2 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -217,8 +217,7 @@ std::optional<MemoryBufferRef> macho::readFile(StringRef path) {
if (entry != cachedReads.end())
return entry->second;
- ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr =
- MemoryBuffer::getFile(path, false, /*RequiresNullTerminator*/ false);
+ ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr = MemoryBuffer::getFile(path);
if (std::error_code ec = mbOrErr.getError()) {
error("cannot open " + path + ": " + ec.message());
return std::nullopt;
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index 445bd6b4609d8..92f31c909efd4 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -584,8 +584,7 @@ Expected<StringRef> Archive::Child::getBuffer() const {
if (!FullNameOrErr)
return FullNameOrErr.takeError();
const std::string &FullName = *FullNameOrErr;
- ErrorOr<std::unique_ptr<MemoryBuffer>> Buf =
- MemoryBuffer::getFile(FullName, false, /*RequiresNullTerminator*/ false);
+ ErrorOr<std::unique_ptr<MemoryBuffer>> Buf = MemoryBuffer::getFile(FullName);
if (std::error_code EC = Buf.getError())
return errorCodeToError(EC);
Parent->ThinBuffers.push_back(std::move(*Buf));
diff --git a/llvm/lib/Support/MemoryBuffer.cpp b/llvm/lib/Support/MemoryBuffer.cpp
index 1c4645ad83641..9c08a197a6987 100644
--- a/llvm/lib/Support/MemoryBuffer.cpp
+++ b/llvm/lib/Support/MemoryBuffer.cpp
@@ -505,8 +505,10 @@ getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize,
// On at least Linux, and possibly on other systems, mmap may return pages
// from the page cache that are not properly filled with trailing zeroes,
// if some prior user of the page wrote non-zero bytes. Detect this and
- // don't use mmap in that case.
- if (!RequiresNullTerminator || *Result->getBufferEnd() == '\0')
+ // don't use mmap in that case (unless it is object or archive file).
+ if (!RequiresNullTerminator || *Result->getBufferEnd() == '\0' ||
+ StringRef(Filename.str()).ends_with(".o") ||
+ StringRef(Filename.str()).ends_with(".a"))
return std::move(Result);
}
}
>From 1b9b13998ba6f8caccaf3fdd8bea41d59beb403f Mon Sep 17 00:00:00 2001
From: John Holdsworth <github at johnholdsworth.com>
Date: Thu, 11 Sep 2025 22:04:55 +0200
Subject: [PATCH 6/7] Revert "Alternative solution to performance regression."
This reverts commit 884b771e18b305b6d15b789dbb40d8025798e7cc.
---
lld/MachO/InputFiles.cpp | 3 ++-
llvm/lib/Object/Archive.cpp | 3 ++-
llvm/lib/Support/MemoryBuffer.cpp | 6 ++----
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index 442fc608865d2..29a83eaeb0da5 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -217,7 +217,8 @@ std::optional<MemoryBufferRef> macho::readFile(StringRef path) {
if (entry != cachedReads.end())
return entry->second;
- ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr = MemoryBuffer::getFile(path);
+ ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr =
+ MemoryBuffer::getFile(path, false, /*RequiresNullTerminator*/ false);
if (std::error_code ec = mbOrErr.getError()) {
error("cannot open " + path + ": " + ec.message());
return std::nullopt;
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index 92f31c909efd4..445bd6b4609d8 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -584,7 +584,8 @@ Expected<StringRef> Archive::Child::getBuffer() const {
if (!FullNameOrErr)
return FullNameOrErr.takeError();
const std::string &FullName = *FullNameOrErr;
- ErrorOr<std::unique_ptr<MemoryBuffer>> Buf = MemoryBuffer::getFile(FullName);
+ ErrorOr<std::unique_ptr<MemoryBuffer>> Buf =
+ MemoryBuffer::getFile(FullName, false, /*RequiresNullTerminator*/ false);
if (std::error_code EC = Buf.getError())
return errorCodeToError(EC);
Parent->ThinBuffers.push_back(std::move(*Buf));
diff --git a/llvm/lib/Support/MemoryBuffer.cpp b/llvm/lib/Support/MemoryBuffer.cpp
index 9c08a197a6987..1c4645ad83641 100644
--- a/llvm/lib/Support/MemoryBuffer.cpp
+++ b/llvm/lib/Support/MemoryBuffer.cpp
@@ -505,10 +505,8 @@ getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize,
// On at least Linux, and possibly on other systems, mmap may return pages
// from the page cache that are not properly filled with trailing zeroes,
// if some prior user of the page wrote non-zero bytes. Detect this and
- // don't use mmap in that case (unless it is object or archive file).
- if (!RequiresNullTerminator || *Result->getBufferEnd() == '\0' ||
- StringRef(Filename.str()).ends_with(".o") ||
- StringRef(Filename.str()).ends_with(".a"))
+ // don't use mmap in that case.
+ if (!RequiresNullTerminator || *Result->getBufferEnd() == '\0')
return std::move(Result);
}
}
>From f5fef9b15cd7f2224c004459ec06aeb52132f3d4 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github at johnholdsworth.com>
Date: Fri, 12 Sep 2025 10:08:37 +0200
Subject: [PATCH 7/7] Format llvm/lib/Object/Archive.cpp
Co-authored-by: James Henderson <James.Henderson at sony.com>
---
llvm/lib/Object/Archive.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index 445bd6b4609d8..bedd021092081 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -585,7 +585,7 @@ Expected<StringRef> Archive::Child::getBuffer() const {
return FullNameOrErr.takeError();
const std::string &FullName = *FullNameOrErr;
ErrorOr<std::unique_ptr<MemoryBuffer>> Buf =
- MemoryBuffer::getFile(FullName, false, /*RequiresNullTerminator*/ false);
+ MemoryBuffer::getFile(FullName, false, /*RequiresNullTerminator=*/false);
if (std::error_code EC = Buf.getError())
return errorCodeToError(EC);
Parent->ThinBuffers.push_back(std::move(*Buf));
More information about the llvm-commits
mailing list