[llvm] [MC][AsmParser]: Don't use SourceMgr::AddIncludeFile when opening .incbin files (PR #194254)
via llvm-commits
llvm-commits at lists.llvm.org
Sat May 2 07:35:25 PDT 2026
https://github.com/SergeantCooper updated https://github.com/llvm/llvm-project/pull/194254
>From 111bdff49cb9f7fbcbfdbf33ff67b58c90a99192 Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 26 Apr 2026 22:24:49 +0530
Subject: [PATCH 1/2] [MC][AsmParser]: Don't use SrcMgr::AddIncludeFile when
opening .incbin files
Each .incbin directive previously called SourceMgr::AddIncludeFile, which
retains the buffer in memory for long time, thus increasing the memory usage,
rather use SrcMgr::OpenIncludeFile to read as the allocated buffer gets cleared
when the processIncbinFile function moves out of scope.
Measured on a stress test of 1000 .incbin directives against a 1 MB blob:
Max RSS
Before ~1.04 GB
After ~14.5 MB
---
llvm/lib/MC/MCParser/AsmParser.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 4e95bf47bb7ee..74279359d9487 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -861,13 +861,13 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
return false;
std::string IncludedFile;
- unsigned NewBuf =
- SrcMgr.AddIncludeFile(Filename, Lexer.getLoc(), IncludedFile);
- if (!NewBuf)
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
+ SrcMgr.OpenIncludeFile(Filename, IncludedFile);
+ if (!BuffOrErr)
return true;
// Pick up the bytes from the file and emit them.
- StringRef Bytes = SrcMgr.getMemoryBuffer(NewBuf)->getBuffer();
+ StringRef Bytes = (*BuffOrErr)->getBuffer();
Bytes = Bytes.drop_front(Skip);
if (Count) {
int64_t Res;
>From daf9be49125e8711ee5ca5eddab7bf2cd3a2341b Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 26 Apr 2026 22:24:49 +0530
Subject: [PATCH 2/2] [MC][AsmParser]: Read only required data of incbin files.
Introduced SourceMgr::OpenSliceIncludeFile and related helper functions
which only allocates required amount of data for the incbin file and
gets cleared when the processIncbinFile function moves out of scope.
Measured on a stress test of 1000 .incbin directives against a 1 MB blob:
Max RSS
Before ~14.5 MB
After ~13.5 MB
---
llvm/include/llvm/Support/SourceMgr.h | 4 ++
llvm/include/llvm/Support/VirtualFileSystem.h | 9 +++++
llvm/lib/MC/MCParser/AsmParser.cpp | 23 ++++++-----
llvm/lib/Support/SourceMgr.cpp | 25 ++++++++++++
llvm/lib/Support/VirtualFileSystem.cpp | 38 +++++++++++++++++++
5 files changed, 89 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 02e694cad8697..54f1c699b7ad5 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -202,6 +202,10 @@ class SourceMgr {
LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
OpenIncludeFile(const std::string &Filename, std::string &IncludedFile);
+ LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
+ OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
+ int64_t Count, std::string &IncludedFile);
+
/// Return the ID of the buffer containing the specified location.
///
/// 0 is returned if the buffer is not found.
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 5b8871b8f3db5..695c3175b570e 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -133,6 +133,10 @@ class LLVM_ABI File {
getBuffer(const Twine &Name, int64_t FileSize = -1,
bool RequiresNullTerminator = true, bool IsVolatile = false) = 0;
+ virtual ErrorOr<std::unique_ptr<MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset = 0, int64_t Count = -1,
+ bool IsVolatile = false);
+
/// Closes the file.
virtual std::error_code close() = 0;
@@ -296,6 +300,11 @@ class LLVM_ABI FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem>,
bool RequiresNullTerminator = true, bool IsVolatile = false,
bool IsText = true);
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getSliceBufferForFile(const Twine &Name, int64_t Offset = 0,
+ int64_t Count = -1, bool IsVolatile = false,
+ bool IsText = true);
+
/// Get a directory_iterator for \p Dir.
/// \note The 'end' iterator is directory_iterator().
virtual directory_iterator dir_begin(const Twine &Dir,
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 74279359d9487..0524d29862028 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -860,23 +860,26 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
if (SymbolScanningMode)
return false;
+ int64_t BytesToRead = -1; // -1 initially for reading the whole file
+ if (Count) {
+ int64_t Res;
+ if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
+ return Error(Loc, "expected absolute expression");
+ if (Res < 0)
+ return Warning(Loc,
+ "negative count has no effect"); // don't read if assembly
+ // has negative count
+ BytesToRead = Res;
+ }
+
std::string IncludedFile;
ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
- SrcMgr.OpenIncludeFile(Filename, IncludedFile);
+ SrcMgr.OpenSliceIncludeFile(Filename, Skip, BytesToRead, IncludedFile);
if (!BuffOrErr)
return true;
// Pick up the bytes from the file and emit them.
StringRef Bytes = (*BuffOrErr)->getBuffer();
- Bytes = Bytes.drop_front(Skip);
- if (Count) {
- int64_t Res;
- if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
- return Error(Loc, "expected absolute expression");
- if (Res < 0)
- return Warning(Loc, "negative count has no effect");
- Bytes = Bytes.take_front(Res);
- }
getStreamer().emitBytes(Bytes);
return false;
}
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 299615a6c8041..e1f7e8e5043a3 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -90,6 +90,31 @@ SourceMgr::OpenIncludeFile(const std::string &Filename,
return NewBufOrErr;
}
+ErrorOr<std::unique_ptr<MemoryBuffer>>
+SourceMgr::OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
+ int64_t Count, std::string &IncludedFile) {
+ auto GetFileSlice = [this, Offset, Count](StringRef Path) {
+ return FS ? FS->getSliceBufferForFile(Path, Offset, Count)
+ : MemoryBuffer::getFileSlice(Path, Count, Offset);
+ };
+
+ ErrorOr<std::unique_ptr<MemoryBuffer>> NewBufOrErr = GetFileSlice(Filename);
+
+ SmallString<64> Buffer(Filename);
+ // If the file didn't exist directly, see if it's in an include path.
+ for (unsigned i = 0, e = IncludeDirectories.size(); i != e && !NewBufOrErr;
+ ++i) {
+ Buffer = IncludeDirectories[i];
+ sys::path::append(Buffer, Filename);
+ NewBufOrErr = GetFileSlice(Buffer);
+ }
+
+ if (NewBufOrErr)
+ IncludedFile = static_cast<std::string>(Buffer);
+
+ return NewBufOrErr;
+}
+
unsigned SourceMgr::FindBufferContainingLoc(SMLoc Loc) const {
for (unsigned i = 0, e = Buffers.size(); i != e; ++i)
if (Loc.getPointer() >= Buffers[i].Buffer->getBufferStart() &&
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 69f3bb8582b87..8b32ca4287dbb 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -126,6 +126,16 @@ FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
}
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+FileSystem::getSliceBufferForFile(const Twine &Name, int64_t Offset,
+ int64_t Count, bool IsVolatile, bool IsText) {
+ auto F = IsText ? openFileForRead(Name) : openFileForReadBinary(Name);
+ if (!F)
+ return F.getError();
+
+ return (*F)->getSliceBuffer(Name, Offset, Count, IsVolatile);
+}
+
std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
if (llvm::sys::path::is_absolute(Path))
return {};
@@ -211,6 +221,11 @@ class RealFile : public File {
int64_t FileSize,
bool RequiresNullTerminator,
bool IsVolatile) override;
+
+ ErrorOr<std::unique_ptr<MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override;
+
std::error_code close() override;
void setPath(const Twine &Path) override;
};
@@ -246,6 +261,15 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize,
IsVolatile);
}
+ErrorOr<std::unique_ptr<MemoryBuffer>>
+RealFile::getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) {
+ auto BypassSandbox = sys::sandbox::scopedDisable();
+
+ assert(FD != kInvalidFile && "cannot get buffer for closed file");
+ return MemoryBuffer::getOpenFileSlice(FD, Name, Count, Offset, IsVolatile);
+}
+
std::error_code RealFile::close() {
auto BypassSandbox = sys::sandbox::scopedDisable();
@@ -2578,6 +2602,20 @@ class FileWithFixedStatus : public File {
} // namespace
+ErrorOr<std::unique_ptr<MemoryBuffer>> File::getSliceBuffer(const Twine &Name,
+ int64_t Offset,
+ int64_t Count,
+ bool IsVolatile) {
+ auto BuffOrError = getBuffer(Name, -1, false, IsVolatile);
+ if (!BuffOrError)
+ return BuffOrError.getError();
+ assert(Offset >= 0 && "Negative Offset");
+ assert(Count >= 0 && "Negative Count");
+ StringRef Bytes = (*BuffOrError)->getBuffer();
+ Bytes = Bytes.drop_front(Offset);
+ Bytes = Bytes.take_front(Count);
+ return MemoryBuffer::getMemBufferCopy(Bytes, Name);
+}
ErrorOr<std::unique_ptr<File>>
File::getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P) {
// See \c getRedirectedFileStatus - don't update path if it's exposing an
More information about the llvm-commits
mailing list