[clang] [clang-tools-extra] [lldb] [llvm] [MC][AsmParser]: Don't use SrcMgr.AddIncludeFile when opening .incbin files (PR #194254)
via llvm-commits
llvm-commits at lists.llvm.org
Sat May 2 05:23:49 PDT 2026
https://github.com/SergeantCooper updated https://github.com/llvm/llvm-project/pull/194254
>From 199ba6df56478f43db1e68760bc83309bd7b545a Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 26 Apr 2026 22:24:49 +0530
Subject: [PATCH] [MC][AsmParser]: Don't use SourceMgr::AddIncludeFile when
opening .incbin files
Each .incbin directive previously called SourceMgr::AddIncludeFile, which
retains the buffer in memory for long time, thus increasing the memory usage.
Introduced SourceMgr::OpenSliceIncludeFile and related helper functions
which only allocates required amount of data for the incbin file and
gets cleared when the processIncbinFile function moves out of scope.
Measured on a stress test of 1000 .incbin directives against a 1 MB blob:
Max RSS
Before ~1.04 GB
After ~13.5 MB
---
clang-tools-extra/clangd/Preamble.cpp | 5 +++
.../clangd/support/ThreadsafeFS.cpp | 6 +++
.../DependencyScanningFilesystem.cpp | 6 +++
lldb/unittests/Host/FileSystemTest.cpp | 6 +++
llvm/include/llvm/Support/SourceMgr.h | 4 ++
llvm/include/llvm/Support/VirtualFileSystem.h | 9 ++++
llvm/lib/MC/MCParser/AsmParser.cpp | 25 ++++++-----
llvm/lib/Support/SourceMgr.cpp | 25 +++++++++++
llvm/lib/Support/VirtualFileSystem.cpp | 43 +++++++++++++++++++
.../Support/VirtualFileSystemTest.cpp | 5 +++
10 files changed, 123 insertions(+), 11 deletions(-)
diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index 58da7edcf3b93..722a7c12ed716 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -443,6 +443,11 @@ class TimerFile : public llvm::vfs::File {
return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
IsVolatile);
}
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override {
+ llvm_unreachable("unimplemented");
+ }
std::error_code close() override {
WallTimerRegion T(Timer);
return InnerFile->close();
diff --git a/clang-tools-extra/clangd/support/ThreadsafeFS.cpp b/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
index 7398e4258527b..47a1036044495 100644
--- a/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
+++ b/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
@@ -59,6 +59,12 @@ class VolatileFileSystem : public llvm::vfs::ProxyFileSystem {
/*IsVolatile=*/true);
}
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getSliceBuffer(const llvm::Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override {
+ llvm_unreachable("unimplemented");
+ }
+
llvm::ErrorOr<llvm::vfs::Status> status() override {
return Wrapped->status();
}
diff --git a/clang/lib/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/DependencyScanning/DependencyScanningFilesystem.cpp
index 49dad3758cf57..dea56553eaceb 100644
--- a/clang/lib/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -362,6 +362,12 @@ class DepScanFile final : public llvm::vfs::File {
RequiresNullTerminator);
}
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override {
+ llvm_unreachable("unimplemented");
+ }
+
std::error_code close() override { return {}; }
private:
diff --git a/lldb/unittests/Host/FileSystemTest.cpp b/lldb/unittests/Host/FileSystemTest.cpp
index e3c2b5a5ab49e..4db9d0d3fe68e 100644
--- a/lldb/unittests/Host/FileSystemTest.cpp
+++ b/lldb/unittests/Host/FileSystemTest.cpp
@@ -29,6 +29,12 @@ struct DummyFile : public vfs::File {
bool IsVolatile) override {
llvm_unreachable("unimplemented");
}
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override {
+ llvm_unreachable("unimplemented");
+ }
+
std::error_code close() override { return std::error_code(); }
};
diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 02e694cad8697..54f1c699b7ad5 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -202,6 +202,10 @@ class SourceMgr {
LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
OpenIncludeFile(const std::string &Filename, std::string &IncludedFile);
+ LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
+ OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
+ int64_t Count, std::string &IncludedFile);
+
/// Return the ID of the buffer containing the specified location.
///
/// 0 is returned if the buffer is not found.
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 5b8871b8f3db5..d0cc4d01cbc88 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -133,6 +133,10 @@ class LLVM_ABI File {
getBuffer(const Twine &Name, int64_t FileSize = -1,
bool RequiresNullTerminator = true, bool IsVolatile = false) = 0;
+ virtual ErrorOr<std::unique_ptr<MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset = 0, int64_t Count = -1,
+ bool IsVolatile = false) = 0;
+
/// Closes the file.
virtual std::error_code close() = 0;
@@ -296,6 +300,11 @@ class LLVM_ABI FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem>,
bool RequiresNullTerminator = true, bool IsVolatile = false,
bool IsText = true);
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getSliceBufferForFile(const Twine &Name, int64_t Offset = 0,
+ int64_t Count = -1, bool IsVolatile = false,
+ bool IsText = true);
+
/// Get a directory_iterator for \p Dir.
/// \note The 'end' iterator is directory_iterator().
virtual directory_iterator dir_begin(const Twine &Dir,
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 4e95bf47bb7ee..6ff6e02d920b3 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -860,23 +860,26 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
if (SymbolScanningMode)
return false;
- std::string IncludedFile;
- unsigned NewBuf =
- SrcMgr.AddIncludeFile(Filename, Lexer.getLoc(), IncludedFile);
- if (!NewBuf)
- return true;
-
- // Pick up the bytes from the file and emit them.
- StringRef Bytes = SrcMgr.getMemoryBuffer(NewBuf)->getBuffer();
- Bytes = Bytes.drop_front(Skip);
+ int64_t BytesToRead = -1; // -1 initially for reading the whole file
if (Count) {
int64_t Res;
if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
return Error(Loc, "expected absolute expression");
if (Res < 0)
- return Warning(Loc, "negative count has no effect");
- Bytes = Bytes.take_front(Res);
+ return Warning(Loc,
+ "negative count has no effect"); // now, negative value
+ // denotes wrong assembly
+ BytesToRead = Res;
}
+
+ std::string IncludedFile;
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
+ SrcMgr.OpenSliceIncludeFile(Filename, Skip, BytesToRead, IncludedFile);
+ if (!BuffOrErr)
+ return true;
+
+ // Pick up the bytes from the file and emit them.
+ StringRef Bytes = (*BuffOrErr)->getBuffer();
getStreamer().emitBytes(Bytes);
return false;
}
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 299615a6c8041..e1f7e8e5043a3 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -90,6 +90,31 @@ SourceMgr::OpenIncludeFile(const std::string &Filename,
return NewBufOrErr;
}
+ErrorOr<std::unique_ptr<MemoryBuffer>>
+SourceMgr::OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
+ int64_t Count, std::string &IncludedFile) {
+ auto GetFileSlice = [this, Offset, Count](StringRef Path) {
+ return FS ? FS->getSliceBufferForFile(Path, Offset, Count)
+ : MemoryBuffer::getFileSlice(Path, Count, Offset);
+ };
+
+ ErrorOr<std::unique_ptr<MemoryBuffer>> NewBufOrErr = GetFileSlice(Filename);
+
+ SmallString<64> Buffer(Filename);
+ // If the file didn't exist directly, see if it's in an include path.
+ for (unsigned i = 0, e = IncludeDirectories.size(); i != e && !NewBufOrErr;
+ ++i) {
+ Buffer = IncludeDirectories[i];
+ sys::path::append(Buffer, Filename);
+ NewBufOrErr = GetFileSlice(Buffer);
+ }
+
+ if (NewBufOrErr)
+ IncludedFile = static_cast<std::string>(Buffer);
+
+ return NewBufOrErr;
+}
+
unsigned SourceMgr::FindBufferContainingLoc(SMLoc Loc) const {
for (unsigned i = 0, e = Buffers.size(); i != e; ++i)
if (Loc.getPointer() >= Buffers[i].Buffer->getBufferStart() &&
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 69f3bb8582b87..359942fb5c93d 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -126,6 +126,16 @@ FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
}
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+FileSystem::getSliceBufferForFile(const Twine &Name, int64_t Offset,
+ int64_t Count, bool IsVolatile, bool IsText) {
+ auto F = IsText ? openFileForRead(Name) : openFileForReadBinary(Name);
+ if (!F)
+ return F.getError();
+
+ return (*F)->getSliceBuffer(Name, Offset, Count, IsVolatile);
+}
+
std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
if (llvm::sys::path::is_absolute(Path))
return {};
@@ -211,6 +221,11 @@ class RealFile : public File {
int64_t FileSize,
bool RequiresNullTerminator,
bool IsVolatile) override;
+
+ ErrorOr<std::unique_ptr<MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override;
+
std::error_code close() override;
void setPath(const Twine &Path) override;
};
@@ -246,6 +261,22 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize,
IsVolatile);
}
+ErrorOr<std::unique_ptr<MemoryBuffer>>
+RealFile::getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) {
+ auto BypassSandBox = sys::sandbox::scopedDisable();
+ if (Count == -1) {
+ uint64_t FileSize;
+ std::error_code Error = sys::fs::file_size(Name, FileSize);
+ if (Error)
+ return Error;
+ Count = FileSize - Offset;
+ }
+
+ assert(FD != kInvalidFile && "cannot get buffer for closed file");
+ return MemoryBuffer::getOpenFileSlice(FD, Name, Count, Offset, IsVolatile);
+}
+
std::error_code RealFile::close() {
auto BypassSandbox = sys::sandbox::scopedDisable();
@@ -789,6 +820,12 @@ class InMemoryFileAdaptor : public File {
Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator);
}
+ ErrorOr<std::unique_ptr<MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override {
+ llvm_unreachable("unimplemented");
+ }
+
std::error_code close() override { return {}; }
void setPath(const Twine &Path) override { RequestedName = Path.str(); }
@@ -2571,6 +2608,12 @@ class FileWithFixedStatus : public File {
IsVolatile);
}
+ ErrorOr<std::unique_ptr<MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override {
+ llvm_unreachable("unimplemented");
+ }
+
std::error_code close() override { return InnerFile->close(); }
void setPath(const Twine &Path) override { S = S.copyWithNewName(S, Path); }
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index bb0172b28e373..30a9eecc7802e 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -42,6 +42,11 @@ struct DummyFile : public vfs::File {
bool IsVolatile) override {
llvm_unreachable("unimplemented");
}
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+ getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+ bool IsVolatile) override {
+ llvm_unreachable("unimplemented");
+ }
std::error_code close() override { return std::error_code(); }
};
More information about the llvm-commits
mailing list