[llvm] [MC][AsmParser]: Don't use SourceMgr::AddIncludeFile when opening .incbin files (PR #194254)
via llvm-commits
llvm-commits at lists.llvm.org
Sun May 3 11:40:31 PDT 2026
https://github.com/SergeantCooper updated https://github.com/llvm/llvm-project/pull/194254
>From 111bdff49cb9f7fbcbfdbf33ff67b58c90a99192 Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 26 Apr 2026 22:24:49 +0530
Subject: [PATCH 1/4] [MC][AsmParser]: Don't use SrcMgr::AddIncludeFile when
opening .incbin files
Each .incbin directive previously called SourceMgr::AddIncludeFile, which
retains the buffer in memory for long time, thus increasing the memory usage,
rather use SrcMgr::OpenIncludeFile to read as the allocated buffer gets cleared
when the processIncbinFile function moves out of scope.
Measured on a stress test of 1000 .incbin directives against a 1 MB blob:
Max RSS
Before ~1.04 GB
After ~14.5 MB
---
llvm/lib/MC/MCParser/AsmParser.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 4e95bf47bb7ee..74279359d9487 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -861,13 +861,13 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
return false;
std::string IncludedFile;
- unsigned NewBuf =
- SrcMgr.AddIncludeFile(Filename, Lexer.getLoc(), IncludedFile);
- if (!NewBuf)
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
+ SrcMgr.OpenIncludeFile(Filename, IncludedFile);
+ if (!BuffOrErr)
return true;
// Pick up the bytes from the file and emit them.
- StringRef Bytes = SrcMgr.getMemoryBuffer(NewBuf)->getBuffer();
+ StringRef Bytes = (*BuffOrErr)->getBuffer();
Bytes = Bytes.drop_front(Skip);
if (Count) {
int64_t Res;
>From daf9be49125e8711ee5ca5eddab7bf2cd3a2341b Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 26 Apr 2026 22:24:49 +0530
Subject: [PATCH 2/4] [MC][AsmParser]: Read only required data of incbin files.
Introduced SourceMgr::OpenSliceIncludeFile and related helper functions
which only allocates required amount of data for the incbin file and
gets cleared when the processIncbinFile function moves out of scope.
Measured on a stress test of 1000 .incbin directives against a 1 MB blob:
Max RSS
Before ~14.5 MB
After ~13.5 MB
---
llvm/include/llvm/Support/SourceMgr.h | 4 ++
llvm/include/llvm/Support/VirtualFileSystem.h | 9 +++++
llvm/lib/MC/MCParser/AsmParser.cpp | 23 ++++++-----
llvm/lib/Support/SourceMgr.cpp | 25 ++++++++++++
llvm/lib/Support/VirtualFileSystem.cpp | 38 +++++++++++++++++++
5 files changed, 89 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 02e694cad8697..54f1c699b7ad5 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -202,6 +202,10 @@ class SourceMgr {
LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
OpenIncludeFile(const std::string &Filename, std::string &IncludedFile);
+ LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
+ OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
+ int64_t Count, std::string &IncludedFile);
+
/// Return the ID of the buffer containing the specified location.
///
/// 0 is returned if the buffer is not found.
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 5b8871b8f3db5..695c3175b570e 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -133,6 +133,10 @@ class LLVM_ABI File {
getBuffer(const Twine &Name, int64_t FileSize = -1,
bool RequiresNullTerminator = true, bool IsVolatile = false) = 0;
+ virtual ErrorOr<std::unique_ptr<MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset = 0, int64_t Count = -1,
+ bool IsVolatile = false);
+
/// Closes the file.
virtual std::error_code close() = 0;
@@ -296,6 +300,11 @@ class LLVM_ABI FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem>,
bool RequiresNullTerminator = true, bool IsVolatile = false,
bool IsText = true);
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getSliceBufferForFile(const Twine &Name, int64_t Offset = 0,
+ int64_t Count = -1, bool IsVolatile = false,
+ bool IsText = true);
+
/// Get a directory_iterator for \p Dir.
/// \note The 'end' iterator is directory_iterator().
virtual directory_iterator dir_begin(const Twine &Dir,
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 74279359d9487..0524d29862028 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -860,23 +860,26 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
if (SymbolScanningMode)
return false;
+ int64_t BytesToRead = -1; // -1 initially for reading the whole file
+ if (Count) {
+ int64_t Res;
+ if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
+ return Error(Loc, "expected absolute expression");
+ if (Res < 0)
+ return Warning(Loc,
+ "negative count has no effect"); // don't read if assembly
+ // has negative count
+ BytesToRead = Res;
+ }
+
std::string IncludedFile;
ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
- SrcMgr.OpenIncludeFile(Filename, IncludedFile);
+ SrcMgr.OpenSliceIncludeFile(Filename, Skip, BytesToRead, IncludedFile);
if (!BuffOrErr)
return true;
// Pick up the bytes from the file and emit them.
StringRef Bytes = (*BuffOrErr)->getBuffer();
- Bytes = Bytes.drop_front(Skip);
- if (Count) {
- int64_t Res;
- if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
- return Error(Loc, "expected absolute expression");
- if (Res < 0)
- return Warning(Loc, "negative count has no effect");
- Bytes = Bytes.take_front(Res);
- }
getStreamer().emitBytes(Bytes);
return false;
}
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 299615a6c8041..e1f7e8e5043a3 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -90,6 +90,31 @@ SourceMgr::OpenIncludeFile(const std::string &Filename,
return NewBufOrErr;
}
+ErrorOr<std::unique_ptr<MemoryBuffer>>
+SourceMgr::OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
+ int64_t Count, std::string &IncludedFile) {
+ auto GetFileSlice = [this, Offset, Count](StringRef Path) {
+ return FS ? FS->getSliceBufferForFile(Path, Offset, Count)
+ : MemoryBuffer::getFileSlice(Path, Count, Offset);
+ };
+
+ ErrorOr<std::unique_ptr<MemoryBuffer>> NewBufOrErr = GetFileSlice(Filename);
+
+ SmallString<64> Buffer(Filename);
+ // If the file didn't exist directly, see if it's in an include path.
+ for (unsigned i = 0, e = IncludeDirectories.size(); i != e && !NewBufOrErr;
+ ++i) {
+ Buffer = IncludeDirectories[i];
+ sys::path::append(Buffer, Filename);
+ NewBufOrErr = GetFileSlice(Buffer);
+ }
+
+ if (NewBufOrErr)
+ IncludedFile = static_cast<std::string>(Buffer);
+
+ return NewBufOrErr;
+}
+
unsigned SourceMgr::FindBufferContainingLoc(SMLoc Loc) const {
for (unsigned i = 0, e = Buffers.size(); i != e; ++i)
if (Loc.getPointer() >= Buffers[i].Buffer->getBufferStart() &&
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 69f3bb8582b87..8b32ca4287dbb 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -126,6 +126,16 @@ FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
}
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+FileSystem::getSliceBufferForFile(const Twine &Name, int64_t Offset,
+ int64_t Count, bool IsVolatile, bool IsText) {
+ auto F = IsText ? openFileForRead(Name) : openFileForReadBinary(Name);
+ if (!F)
+ return F.getError();
+
+ return (*F)->getSliceBuffer(Name, Offset, Count, IsVolatile);
+}
+
std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
if (llvm::sys::path::is_absolute(Path))
return {};
@@ -211,6 +221,11 @@ class RealFile : public File {
int64_t FileSize,
bool RequiresNullTerminator,
bool IsVolatile) override;
+
+ ErrorOr<std::unique_ptr<MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override;
+
std::error_code close() override;
void setPath(const Twine &Path) override;
};
@@ -246,6 +261,15 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize,
IsVolatile);
}
+ErrorOr<std::unique_ptr<MemoryBuffer>>
+RealFile::getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) {
+ auto BypassSandbox = sys::sandbox::scopedDisable();
+
+ assert(FD != kInvalidFile && "cannot get buffer for closed file");
+ return MemoryBuffer::getOpenFileSlice(FD, Name, Count, Offset, IsVolatile);
+}
+
std::error_code RealFile::close() {
auto BypassSandbox = sys::sandbox::scopedDisable();
@@ -2578,6 +2602,20 @@ class FileWithFixedStatus : public File {
} // namespace
+ErrorOr<std::unique_ptr<MemoryBuffer>> File::getSliceBuffer(const Twine &Name,
+ int64_t Offset,
+ int64_t Count,
+ bool IsVolatile) {
+ auto BuffOrError = getBuffer(Name, -1, false, IsVolatile);
+ if (!BuffOrError)
+ return BuffOrError.getError();
+ assert(Offset >= 0 && "Negative Offset");
+ assert(Count >= 0 && "Negative Count");
+ StringRef Bytes = (*BuffOrError)->getBuffer();
+ Bytes = Bytes.drop_front(Offset);
+ Bytes = Bytes.take_front(Count);
+ return MemoryBuffer::getMemBufferCopy(Bytes, Name);
+}
ErrorOr<std::unique_ptr<File>>
File::getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P) {
// See \c getRedirectedFileStatus - don't update path if it's exposing an
>From d9b453ac8238546513bcf0d9be01900600e1bccc Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 3 May 2026 00:46:51 +0530
Subject: [PATCH 3/4] [Support] Use uint64_t for slice Offset/Count parameters
---
llvm/include/llvm/Support/SourceMgr.h | 4 +--
llvm/include/llvm/Support/VirtualFileSystem.h | 6 ++---
llvm/lib/MC/MCParser/AsmParser.cpp | 11 ++++----
llvm/lib/Support/SourceMgr.cpp | 4 +--
llvm/lib/Support/VirtualFileSystem.cpp | 26 +++++++++++++------
5 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 54f1c699b7ad5..0e6ff2f7f8279 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -203,8 +203,8 @@ class SourceMgr {
OpenIncludeFile(const std::string &Filename, std::string &IncludedFile);
LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
- OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
- int64_t Count, std::string &IncludedFile);
+ OpenSliceIncludeFile(const std::string &Filename, uint64_t Offset,
+ uint64_t Count, std::string &IncludedFile);
/// Return the ID of the buffer containing the specified location.
///
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 695c3175b570e..85c1af08caf82 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -134,7 +134,7 @@ class LLVM_ABI File {
bool RequiresNullTerminator = true, bool IsVolatile = false) = 0;
virtual ErrorOr<std::unique_ptr<MemoryBuffer>>
- getSliceBuffer(const Twine &Name, int64_t Offset = 0, int64_t Count = -1,
+ getSliceBuffer(const Twine &Name, uint64_t Offset = 0, uint64_t Count = -1,
bool IsVolatile = false);
/// Closes the file.
@@ -301,8 +301,8 @@ class LLVM_ABI FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem>,
bool IsText = true);
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
- getSliceBufferForFile(const Twine &Name, int64_t Offset = 0,
- int64_t Count = -1, bool IsVolatile = false,
+ getSliceBufferForFile(const Twine &Name, uint64_t Offset = 0,
+ uint64_t Count = -1, bool IsVolatile = false,
bool IsText = true);
/// Get a directory_iterator for \p Dir.
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 0524d29862028..402ca6d5e7fc9 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -860,16 +860,15 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
if (SymbolScanningMode)
return false;
- int64_t BytesToRead = -1; // -1 initially for reading the whole file
+ uint64_t BytesToRead =
+ uint64_t(-1); // -1 initially for reading the whole file
if (Count) {
int64_t Res;
if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
return Error(Loc, "expected absolute expression");
- if (Res < 0)
- return Warning(Loc,
- "negative count has no effect"); // don't read if assembly
- // has negative count
- BytesToRead = Res;
+ if (Res < 0) // don't read if the assembly has negative count
+ return Warning(Loc, "negative count has no effect");
+ BytesToRead = uint64_t(Res);
}
std::string IncludedFile;
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index e1f7e8e5043a3..0816c44c3d56c 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -91,8 +91,8 @@ SourceMgr::OpenIncludeFile(const std::string &Filename,
}
ErrorOr<std::unique_ptr<MemoryBuffer>>
-SourceMgr::OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
- int64_t Count, std::string &IncludedFile) {
+SourceMgr::OpenSliceIncludeFile(const std::string &Filename, uint64_t Offset,
+ uint64_t Count, std::string &IncludedFile) {
auto GetFileSlice = [this, Offset, Count](StringRef Path) {
return FS ? FS->getSliceBufferForFile(Path, Offset, Count)
: MemoryBuffer::getFileSlice(Path, Count, Offset);
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 8b32ca4287dbb..f73f311f70428 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -127,8 +127,9 @@ FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
}
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
-FileSystem::getSliceBufferForFile(const Twine &Name, int64_t Offset,
- int64_t Count, bool IsVolatile, bool IsText) {
+FileSystem::getSliceBufferForFile(const Twine &Name, uint64_t Offset,
+ uint64_t Count, bool IsVolatile,
+ bool IsText) {
auto F = IsText ? openFileForRead(Name) : openFileForReadBinary(Name);
if (!F)
return F.getError();
@@ -223,7 +224,7 @@ class RealFile : public File {
bool IsVolatile) override;
ErrorOr<std::unique_ptr<MemoryBuffer>>
- getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ getSliceBuffer(const Twine &Name, uint64_t Offset, uint64_t Count,
bool IsVolatile) override;
std::error_code close() override;
@@ -262,11 +263,22 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize,
}
ErrorOr<std::unique_ptr<MemoryBuffer>>
-RealFile::getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+RealFile::getSliceBuffer(const Twine &Name, uint64_t Offset, uint64_t Count,
bool IsVolatile) {
auto BypassSandbox = sys::sandbox::scopedDisable();
assert(FD != kInvalidFile && "cannot get buffer for closed file");
+ if (Count == uint64_t(-1)) {
+ sys::fs::file_status Status;
+ if (std::error_code EC = sys::fs::status(FD, Status))
+ return EC;
+ uint64_t FileSize = Status.getSize();
+ if (Offset < FileSize)
+ Count = FileSize - Offset;
+ else
+ Count = 0;
+ }
+
return MemoryBuffer::getOpenFileSlice(FD, Name, Count, Offset, IsVolatile);
}
@@ -2603,14 +2615,12 @@ class FileWithFixedStatus : public File {
} // namespace
ErrorOr<std::unique_ptr<MemoryBuffer>> File::getSliceBuffer(const Twine &Name,
- int64_t Offset,
- int64_t Count,
+ uint64_t Offset,
+ uint64_t Count,
bool IsVolatile) {
auto BuffOrError = getBuffer(Name, -1, false, IsVolatile);
if (!BuffOrError)
return BuffOrError.getError();
- assert(Offset >= 0 && "Negative Offset");
- assert(Count >= 0 && "Negative Count");
StringRef Bytes = (*BuffOrError)->getBuffer();
Bytes = Bytes.drop_front(Offset);
Bytes = Bytes.take_front(Count);
>From 9e5abc396c2494c76423940a28cb4f7709ad7db8 Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 3 May 2026 15:55:52 +0530
Subject: [PATCH 4/4] [Support][SourceMgr] Avoid getFileSlice past-EOF read in
OpenSliceIncludeFile
---
llvm/lib/Support/SourceMgr.cpp | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 0816c44c3d56c..d66cdbb423457 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -93,9 +93,22 @@ SourceMgr::OpenIncludeFile(const std::string &Filename,
ErrorOr<std::unique_ptr<MemoryBuffer>>
SourceMgr::OpenSliceIncludeFile(const std::string &Filename, uint64_t Offset,
uint64_t Count, std::string &IncludedFile) {
- auto GetFileSlice = [this, Offset, Count](StringRef Path) {
- return FS ? FS->getSliceBufferForFile(Path, Offset, Count)
- : MemoryBuffer::getFileSlice(Path, Count, Offset);
+ auto GetFileSlice =
+ [this, Offset,
+ Count](StringRef Path) -> ErrorOr<std::unique_ptr<MemoryBuffer>> {
+ if (FS)
+ return FS->getSliceBufferForFile(Path, Offset, Count);
+
+ if (Count != uint64_t(-1))
+ return MemoryBuffer::getFileSlice(Path, Count, Offset);
+
+ auto BufOrErr = MemoryBuffer::getFile(Path);
+ if (!BufOrErr)
+ return BufOrErr.getError();
+ std::unique_ptr<MemoryBuffer> Buff = std::move(*BufOrErr);
+ StringRef Bytes = Buff->getBuffer();
+ Bytes = Bytes.drop_front(Offset);
+ return MemoryBuffer::getMemBufferCopy(Bytes, Buff->getBufferIdentifier());
};
ErrorOr<std::unique_ptr<MemoryBuffer>> NewBufOrErr = GetFileSlice(Filename);
More information about the llvm-commits
mailing list