[llvm] [MC][AsmParser]: Don't use SourceMgr::AddIncludeFile when opening .incbin files (PR #194254)

via llvm-commits llvm-commits at lists.llvm.org
Sat May 2 07:35:25 PDT 2026


https://github.com/SergeantCooper updated https://github.com/llvm/llvm-project/pull/194254

>From 111bdff49cb9f7fbcbfdbf33ff67b58c90a99192 Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 26 Apr 2026 22:24:49 +0530
Subject: [PATCH 1/2] [MC][AsmParser]: Don't use SrcMgr::AddIncludeFile when
 opening .incbin files

Each .incbin directive previously called SourceMgr::AddIncludeFile, which
retains the buffer in memory for long time, thus increasing the memory usage,
rather use SrcMgr::OpenIncludeFile to read as the allocated  buffer gets cleared
when the processIncbinFile function moves out of scope.

Measured on a stress test of 1000 .incbin directives against a 1 MB blob:

                  Max RSS
    Before       ~1.04 GB
    After        ~14.5 MB
---
 llvm/lib/MC/MCParser/AsmParser.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 4e95bf47bb7ee..74279359d9487 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -861,13 +861,13 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
     return false;
 
   std::string IncludedFile;
-  unsigned NewBuf =
-      SrcMgr.AddIncludeFile(Filename, Lexer.getLoc(), IncludedFile);
-  if (!NewBuf)
+  ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
+      SrcMgr.OpenIncludeFile(Filename, IncludedFile);
+  if (!BuffOrErr)
     return true;
 
   // Pick up the bytes from the file and emit them.
-  StringRef Bytes = SrcMgr.getMemoryBuffer(NewBuf)->getBuffer();
+  StringRef Bytes = (*BuffOrErr)->getBuffer();
   Bytes = Bytes.drop_front(Skip);
   if (Count) {
     int64_t Res;

>From daf9be49125e8711ee5ca5eddab7bf2cd3a2341b Mon Sep 17 00:00:00 2001
From: Ujjawal Kumar <ujjawalkumarchouhan7895 at gmail.com>
Date: Sun, 26 Apr 2026 22:24:49 +0530
Subject: [PATCH 2/2] [MC][AsmParser]: Read only required data of incbin files.

Introduced SourceMgr::OpenSliceIncludeFile and related helper functions
which only allocates required amount of data for the incbin file and
gets cleared when the processIncbinFile function moves out of scope.

Measured on a stress test of 1000 .incbin directives against a 1 MB blob:

                  Max RSS
    Before       ~14.5 MB
    After        ~13.5 MB
---
 llvm/include/llvm/Support/SourceMgr.h         |  4 ++
 llvm/include/llvm/Support/VirtualFileSystem.h |  9 +++++
 llvm/lib/MC/MCParser/AsmParser.cpp            | 23 ++++++-----
 llvm/lib/Support/SourceMgr.cpp                | 25 ++++++++++++
 llvm/lib/Support/VirtualFileSystem.cpp        | 38 +++++++++++++++++++
 5 files changed, 89 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h
index 02e694cad8697..54f1c699b7ad5 100644
--- a/llvm/include/llvm/Support/SourceMgr.h
+++ b/llvm/include/llvm/Support/SourceMgr.h
@@ -202,6 +202,10 @@ class SourceMgr {
   LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
   OpenIncludeFile(const std::string &Filename, std::string &IncludedFile);
 
+  LLVM_ABI ErrorOr<std::unique_ptr<MemoryBuffer>>
+  OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
+                       int64_t Count, std::string &IncludedFile);
+
   /// Return the ID of the buffer containing the specified location.
   ///
   /// 0 is returned if the buffer is not found.
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 5b8871b8f3db5..695c3175b570e 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -133,6 +133,10 @@ class LLVM_ABI File {
   getBuffer(const Twine &Name, int64_t FileSize = -1,
             bool RequiresNullTerminator = true, bool IsVolatile = false) = 0;
 
+  virtual ErrorOr<std::unique_ptr<MemoryBuffer>>
+  getSliceBuffer(const Twine &Name, int64_t Offset = 0, int64_t Count = -1,
+                 bool IsVolatile = false);
+
   /// Closes the file.
   virtual std::error_code close() = 0;
 
@@ -296,6 +300,11 @@ class LLVM_ABI FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem>,
                    bool RequiresNullTerminator = true, bool IsVolatile = false,
                    bool IsText = true);
 
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  getSliceBufferForFile(const Twine &Name, int64_t Offset = 0,
+                        int64_t Count = -1, bool IsVolatile = false,
+                        bool IsText = true);
+
   /// Get a directory_iterator for \p Dir.
   /// \note The 'end' iterator is directory_iterator().
   virtual directory_iterator dir_begin(const Twine &Dir,
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 74279359d9487..0524d29862028 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -860,23 +860,26 @@ bool AsmParser::processIncbinFile(const std::string &Filename, int64_t Skip,
   if (SymbolScanningMode)
     return false;
 
+  int64_t BytesToRead = -1; // -1 initially for reading the whole file
+  if (Count) {
+    int64_t Res;
+    if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
+      return Error(Loc, "expected absolute expression");
+    if (Res < 0)
+      return Warning(Loc,
+                     "negative count has no effect"); // don't read if assembly
+    // has negative count
+    BytesToRead = Res;
+  }
+
   std::string IncludedFile;
   ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
-      SrcMgr.OpenIncludeFile(Filename, IncludedFile);
+      SrcMgr.OpenSliceIncludeFile(Filename, Skip, BytesToRead, IncludedFile);
   if (!BuffOrErr)
     return true;
 
   // Pick up the bytes from the file and emit them.
   StringRef Bytes = (*BuffOrErr)->getBuffer();
-  Bytes = Bytes.drop_front(Skip);
-  if (Count) {
-    int64_t Res;
-    if (!Count->evaluateAsAbsolute(Res, getStreamer().getAssemblerPtr()))
-      return Error(Loc, "expected absolute expression");
-    if (Res < 0)
-      return Warning(Loc, "negative count has no effect");
-    Bytes = Bytes.take_front(Res);
-  }
   getStreamer().emitBytes(Bytes);
   return false;
 }
diff --git a/llvm/lib/Support/SourceMgr.cpp b/llvm/lib/Support/SourceMgr.cpp
index 299615a6c8041..e1f7e8e5043a3 100644
--- a/llvm/lib/Support/SourceMgr.cpp
+++ b/llvm/lib/Support/SourceMgr.cpp
@@ -90,6 +90,31 @@ SourceMgr::OpenIncludeFile(const std::string &Filename,
   return NewBufOrErr;
 }
 
+ErrorOr<std::unique_ptr<MemoryBuffer>>
+SourceMgr::OpenSliceIncludeFile(const std::string &Filename, int64_t Offset,
+                                int64_t Count, std::string &IncludedFile) {
+  auto GetFileSlice = [this, Offset, Count](StringRef Path) {
+    return FS ? FS->getSliceBufferForFile(Path, Offset, Count)
+              : MemoryBuffer::getFileSlice(Path, Count, Offset);
+  };
+
+  ErrorOr<std::unique_ptr<MemoryBuffer>> NewBufOrErr = GetFileSlice(Filename);
+
+  SmallString<64> Buffer(Filename);
+  // If the file didn't exist directly, see if it's in an include path.
+  for (unsigned i = 0, e = IncludeDirectories.size(); i != e && !NewBufOrErr;
+       ++i) {
+    Buffer = IncludeDirectories[i];
+    sys::path::append(Buffer, Filename);
+    NewBufOrErr = GetFileSlice(Buffer);
+  }
+
+  if (NewBufOrErr)
+    IncludedFile = static_cast<std::string>(Buffer);
+
+  return NewBufOrErr;
+}
+
 unsigned SourceMgr::FindBufferContainingLoc(SMLoc Loc) const {
   for (unsigned i = 0, e = Buffers.size(); i != e; ++i)
     if (Loc.getPointer() >= Buffers[i].Buffer->getBufferStart() &&
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 69f3bb8582b87..8b32ca4287dbb 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -126,6 +126,16 @@ FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
   return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
 }
 
+llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+FileSystem::getSliceBufferForFile(const Twine &Name, int64_t Offset,
+                                  int64_t Count, bool IsVolatile, bool IsText) {
+  auto F = IsText ? openFileForRead(Name) : openFileForReadBinary(Name);
+  if (!F)
+    return F.getError();
+
+  return (*F)->getSliceBuffer(Name, Offset, Count, IsVolatile);
+}
+
 std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
   if (llvm::sys::path::is_absolute(Path))
     return {};
@@ -211,6 +221,11 @@ class RealFile : public File {
                                                    int64_t FileSize,
                                                    bool RequiresNullTerminator,
                                                    bool IsVolatile) override;
+
+  ErrorOr<std::unique_ptr<MemoryBuffer>>
+  getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+                 bool IsVolatile) override;
+
   std::error_code close() override;
   void setPath(const Twine &Path) override;
 };
@@ -246,6 +261,15 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize,
                                    IsVolatile);
 }
 
+ErrorOr<std::unique_ptr<MemoryBuffer>>
+RealFile::getSliceBuffer(const Twine &Name, int64_t Offset, int64_t Count,
+                         bool IsVolatile) {
+  auto BypassSandbox = sys::sandbox::scopedDisable();
+
+  assert(FD != kInvalidFile && "cannot get buffer for closed file");
+  return MemoryBuffer::getOpenFileSlice(FD, Name, Count, Offset, IsVolatile);
+}
+
 std::error_code RealFile::close() {
   auto BypassSandbox = sys::sandbox::scopedDisable();
 
@@ -2578,6 +2602,20 @@ class FileWithFixedStatus : public File {
 
 } // namespace
 
+ErrorOr<std::unique_ptr<MemoryBuffer>> File::getSliceBuffer(const Twine &Name,
+                                                            int64_t Offset,
+                                                            int64_t Count,
+                                                            bool IsVolatile) {
+  auto BuffOrError = getBuffer(Name, -1, false, IsVolatile);
+  if (!BuffOrError)
+    return BuffOrError.getError();
+  assert(Offset >= 0 && "Negative Offset");
+  assert(Count >= 0 && "Negative Count");
+  StringRef Bytes = (*BuffOrError)->getBuffer();
+  Bytes = Bytes.drop_front(Offset);
+  Bytes = Bytes.take_front(Count);
+  return MemoryBuffer::getMemBufferCopy(Bytes, Name);
+}
 ErrorOr<std::unique_ptr<File>>
 File::getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P) {
   // See \c getRedirectedFileStatus - don't update path if it's exposing an



More information about the llvm-commits mailing list