[llvm] [MCParser] .incbin: Don't retain the buffer, don't require NUL termination (PR #196696)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Sat May 9 13:39:17 PDT 2026
https://github.com/MaskRay updated https://github.com/llvm/llvm-project/pull/196696
>From 0f1db05a69a0cf2fdcbf55601a8d541bb9315bb8 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Fri, 8 May 2026 00:07:53 -0700
Subject: [PATCH 1/3] [MCParser] .incbin: Don't retain the buffer, don't
require NUL termination
processIncbinFile uses SourceMgr::AddIncludeFile, which
* sets `RequiresNullTerminator=true` and disable `mmap` when the file size is a multiple of the page size,
* and unnecessarily retains the throwaway buffer in `Buffers`.
Switch to OpenIncludeFile so the buffer is freed when processIncbinFile
returns, and pass RequiresNullTerminator=false. The buffer is consumed
only by emitBytes; the lexer never scans it, so it does not need a
trailing '\0'. Without that requirement, MemoryBuffer mmaps the file and
RSS tracks only the touched pages.
Stress test (1000 .incbin "blob.bin", 0, 16 against a 1 MiB blob):
```
Maximum RSS
Before 1042944 KiB
After 15360 KiB
```
This matches the result of #194254 without introducing the
SourceMgr::OpenSliceIncludeFile / vfs::File::getSliceBuffer /
FileSystem::getSliceBufferForFile API surface.
Fix #62339
---
llvm/include/llvm/Support/SourceMgr.h | 3 ++-
llvm/lib/MC/MCParser/AsmParser.cpp | 11 +++++++----
llvm/lib/Support/SourceMgr.cpp | 10 +++++++---
3 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 02e694cad8697..1bfbd1e7eaae7 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -200,7 +200,8 @@ class SourceMgr {
/// buffer of the stacked file. The full path to the included file can be
/// found in \p IncludedFile.
LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
- OpenIncludeFile(const std::string &Filename, std::string &IncludedFile);
+ OpenIncludeFile(const std::string &Filename, std::string &IncludedFile,
+ bool RequiresNullTerminator = true);
/// Return the ID of the buffer containing the specified location.
///
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 0d517fbd57472..482abd5bbbd55 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -860,14 +860,17 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
if (SymbolScanningMode)
return false;
+ // The buffer is consumed only by emitBytes. Skip the NUL termination to
+ // enable mmap in more cases, reading only the touched pages instead of the
+ // whole file.
std::string IncludedFile;
- unsigned NewBuf =
- SrcMgr.AddIncludeFile(Filename, Lexer.getLoc(), IncludedFile);
- if (!NewBuf)
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr = SrcMgr.OpenIncludeFile(
+ Filename, IncludedFile, /*RequiresNullTerminator=*/false);
+ if (!BufOrErr)
return true;
// Pick up the bytes from the file and emit them.
- StringRef Bytes = SrcMgr.getMemoryBuffer(NewBuf)->getBuffer();
+ StringRef Bytes = (*BufOrErr)->getBuffer();
Bytes = Bytes.drop_front(Skip);
if (Count) {
int64_t Res;
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 299615a6c8041..486537c7b0171 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -68,9 +68,13 @@ unsigned SourceMgr::AddIncludeFile(const std::string &Filename,
ErrorOr<std::unique_ptr<MemoryBuffer>>
SourceMgr::OpenIncludeFile(const std::string &Filename,
- std::string &IncludedFile) {
- auto GetFile = [this](StringRef Path) {
- return FS ? FS->getBufferForFile(Path) : MemoryBuffer::getFile(Path);
+ std::string &IncludedFile,
+ bool RequiresNullTerminator) {
+ auto GetFile = [this, RequiresNullTerminator](StringRef Path) {
+ return FS ? FS->getBufferForFile(Path, /*FileSize=*/-1,
+ RequiresNullTerminator)
+ : MemoryBuffer::getFile(Path, /*IsText=*/false,
+ RequiresNullTerminator);
};
ErrorOr<std::unique_ptr<MemoryBuffer>> NewBufOrErr = GetFile(Filename);
>From d82d9015751faa9319e565794fe47efb97d81d11 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sat, 9 May 2026 09:33:18 -0700
Subject: [PATCH 2/3] comment
---
llvm/include/llvm/Support/SourceMgr.h | 3 +--
llvm/lib/MC/MCParser/AsmParser.cpp | 10 +++-------
llvm/lib/Support/SourceMgr.cpp | 11 ++++++-----
3 files changed, 10 insertions(+), 14 deletions(-)
diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 1bfbd1e7eaae7..02e694cad8697 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -200,8 +200,7 @@ class SourceMgr {
/// buffer of the stacked file. The full path to the included file can be
/// found in \p IncludedFile.
LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
- OpenIncludeFile(const std::string &Filename, std::string &IncludedFile,
- bool RequiresNullTerminator = true);
+ OpenIncludeFile(const std::string &Filename, std::string &IncludedFile);
/// Return the ID of the buffer containing the specified location.
///
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 482abd5bbbd55..c78cf40b3a53b 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -860,18 +860,14 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
if (SymbolScanningMode)
return false;
- // The buffer is consumed only by emitBytes. Skip the NUL termination to
- // enable mmap in more cases, reading only the touched pages instead of the
- // whole file.
std::string IncludedFile;
- ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr = SrcMgr.OpenIncludeFile(
- Filename, IncludedFile, /*RequiresNullTerminator=*/false);
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+ SrcMgr.OpenIncludeFile(Filename, IncludedFile);
if (!BufOrErr)
return true;
// Pick up the bytes from the file and emit them.
- StringRef Bytes = (*BufOrErr)->getBuffer();
- Bytes = Bytes.drop_front(Skip);
+ StringRef Bytes = (*BufOrErr)->getBuffer().drop_front(Skip);
if (Count) {
int64_t Res;
if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 486537c7b0171..1a1b535f5dd78 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -68,13 +68,14 @@ unsigned SourceMgr::AddIncludeFile(const std::string &Filename,
ErrorOr<std::unique_ptr<MemoryBuffer>>
SourceMgr::OpenIncludeFile(const std::string &Filename,
- std::string &IncludedFile,
- bool RequiresNullTerminator) {
- auto GetFile = [this, RequiresNullTerminator](StringRef Path) {
+ std::string &IncludedFile) {
+ // Skip the NUL termination to enable mmap in more cases, reading only the
+ // touched pages instead of the whole file.
+ auto GetFile = [this](StringRef Path) {
return FS ? FS->getBufferForFile(Path, /*FileSize=*/-1,
- RequiresNullTerminator)
+ /*RequiresNullTerminator=*/false)
: MemoryBuffer::getFile(Path, /*IsText=*/false,
- RequiresNullTerminator);
+ /*RequiresNullTerminator=*/false);
};
ErrorOr<std::unique_ptr<MemoryBuffer>> NewBufOrErr = GetFile(Filename);
>From 3a8c4b3ef329b9da8b45294394f61ab3091e3cc6 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sat, 9 May 2026 13:39:07 -0700
Subject: [PATCH 3/3] restore
---
llvm/include/llvm/Support/SourceMgr.h | 3 ++-
llvm/lib/MC/MCParser/AsmParser.cpp | 10 +++++++---
llvm/lib/Support/SourceMgr.cpp | 11 +++++------
3 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 02e694cad8697..1bfbd1e7eaae7 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -200,7 +200,8 @@ class SourceMgr {
/// buffer of the stacked file. The full path to the included file can be
/// found in \p IncludedFile.
LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
- OpenIncludeFile(const std::string &Filename, std::string &IncludedFile);
+ OpenIncludeFile(const std::string &Filename, std::string &IncludedFile,
+ bool RequiresNullTerminator = true);
/// Return the ID of the buffer containing the specified location.
///
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index c78cf40b3a53b..482abd5bbbd55 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -860,14 +860,18 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
if (SymbolScanningMode)
return false;
+ // The buffer is consumed only by emitBytes. Skip the NUL termination to
+ // enable mmap in more cases, reading only the touched pages instead of the
+ // whole file.
std::string IncludedFile;
- ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
- SrcMgr.OpenIncludeFile(Filename, IncludedFile);
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr = SrcMgr.OpenIncludeFile(
+ Filename, IncludedFile, /*RequiresNullTerminator=*/false);
if (!BufOrErr)
return true;
// Pick up the bytes from the file and emit them.
- StringRef Bytes = (*BufOrErr)->getBuffer().drop_front(Skip);
+ StringRef Bytes = (*BufOrErr)->getBuffer();
+ Bytes = Bytes.drop_front(Skip);
if (Count) {
int64_t Res;
if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 1a1b535f5dd78..486537c7b0171 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -68,14 +68,13 @@ unsigned SourceMgr::AddIncludeFile(const std::string &Filename,
ErrorOr<std::unique_ptr<MemoryBuffer>>
SourceMgr::OpenIncludeFile(const std::string &Filename,
- std::string &IncludedFile) {
- // Skip the NUL termination to enable mmap in more cases, reading only the
- // touched pages instead of the whole file.
- auto GetFile = [this](StringRef Path) {
+ std::string &IncludedFile,
+ bool RequiresNullTerminator) {
+ auto GetFile = [this, RequiresNullTerminator](StringRef Path) {
return FS ? FS->getBufferForFile(Path, /*FileSize=*/-1,
- /*RequiresNullTerminator=*/false)
+ RequiresNullTerminator)
: MemoryBuffer::getFile(Path, /*IsText=*/false,
- /*RequiresNullTerminator=*/false);
+ RequiresNullTerminator);
};
ErrorOr<std::unique_ptr<MemoryBuffer>> NewBufOrErr = GetFile(Filename);
More information about the llvm-commits
mailing list