[llvm] [llvm-strings] Use small buffer instead of reading whole file (PR #163073)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 23:35:08 PDT 2026
https://github.com/aokblast updated https://github.com/llvm/llvm-project/pull/163073
>From 42984b543dafa8731fbf2034331c8d81c8d7ed65 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Sun, 12 Oct 2025 22:01:13 +0800
Subject: [PATCH 1/6] [llvm-strings] Use small buffer instead of reading whole
file
To prevent large memory footprint, we use small buffer and a seperated
string to print. This can largely reduce the memory consumption when
processing.
In the following example, the test.file is 1G regular file, which is possible in the
daily workload.
cat ./test.file | llvm-strings --radix=o > out &&
ps -o command,vsz,rss | grep strings
Reading whole file:
llvm-strings 5129824 3999920
Small buffer implementation:
llvm-strings 17604 5476
Also, it does not affect too much runtime:
time llvm-strings --radix=o > out < ./test.file
Reading whole file:
________________________________________________________
Executed in 5.67 secs fish external
usr time 4.38 secs 2.40 millis 4.38 secs
sys time 1.28 secs 0.00 millis 1.28 secs
Small buffer implementation:
________________________________________________________
Executed in 5.87 secs fish external
usr time 5.65 secs 2.59 millis 5.65 secs
sys time 0.21 secs 0.00 millis 0.21 secs
---
llvm/tools/llvm-strings/llvm-strings.cpp | 74 ++++++++++++++++++------
1 file changed, 55 insertions(+), 19 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 9979b93de8427..425c9d9831b99 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -18,7 +18,9 @@
#include "llvm/Option/ArgList.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Errno.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/MemoryBuffer.h"
@@ -88,7 +90,9 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
}
}
-static void strings(raw_ostream &OS, StringRef FileName, StringRef Contents) {
+static void strings(raw_ostream &OS, StringRef FileName,
+ sys::fs::file_t FileHandle) {
+ SmallString<sys::fs::DefaultReadChunkSize> Buffer;
auto print = [&OS, FileName](unsigned Offset, StringRef L) {
if (L.size() < static_cast<size_t>(MinLength))
return;
@@ -110,19 +114,42 @@ static void strings(raw_ostream &OS, StringRef FileName, StringRef Contents) {
OS << L << '\n';
};
- const char *B = Contents.begin();
- const char *P = nullptr, *E = nullptr, *S = nullptr;
- for (P = Contents.begin(), E = Contents.end(); P < E; ++P) {
- if (isPrint(*P) || *P == '\t') {
- if (S == nullptr)
- S = P;
- } else if (S) {
- print(S - B, StringRef(S, P - S));
- S = nullptr;
+ unsigned Offset = 0, LocalOffset = 0, CurSize = 0;
+ Buffer.resize(sys::fs::DefaultReadChunkSize);
+ auto FillBuffer = [&Buffer, FileHandle, &FileName]() -> unsigned {
+ Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
+ FileHandle,
+ MutableArrayRef(Buffer.begin(), sys::fs::DefaultReadChunkSize));
+ if (!ReadBytesOrErr) {
+ errs() << FileName << ": "
+ << errorToErrorCode(ReadBytesOrErr.takeError()).message() << '\n';
+ return 0;
+ }
+ return *ReadBytesOrErr;
+ };
+ std::string StringBuffer;
+ while (true) {
+ if (LocalOffset == CurSize) {
+ CurSize = FillBuffer();
+ if (CurSize == 0)
+ break;
+ LocalOffset = 0;
+ }
+ char C = Buffer[LocalOffset++];
+ if (isPrint(C) || C == '\t') {
+ StringBuffer.push_back(C);
+ } else if (StringBuffer.size()) {
+ print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
+ Offset += StringBuffer.size();
+ StringBuffer.clear();
+ ++Offset;
+ } else {
+ ++Offset;
}
}
- if (S)
- print(S - B, StringRef(S, E - S));
+
+ if (StringBuffer.size())
+ print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
}
int main(int argc, char **argv) {
@@ -174,13 +201,22 @@ int main(int argc, char **argv) {
InputFileNames.push_back("-");
for (const auto &File : InputFileNames) {
- ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
- MemoryBuffer::getFileOrSTDIN(File, /*IsText=*/true);
- if (std::error_code EC = Buffer.getError())
- errs() << File << ": " << EC.message() << '\n';
- else
- strings(llvm::outs(), File == "-" ? "{standard input}" : File,
- Buffer.get()->getMemBufferRef().getBuffer());
+ sys::fs::file_t FD;
+ if (File == "-") {
+ FD = sys::fs::getStdinHandle();
+ } else {
+ Expected<sys::fs::file_t> FDOrErr =
+ sys::fs::openNativeFileForRead(File, sys::fs::OF_None);
+ ;
+ if (!FDOrErr) {
+ errs() << File << ": "
+ << errorToErrorCode(FDOrErr.takeError()).message() << '\n';
+ continue;
+ }
+ FD = *FDOrErr;
+ }
+
+ strings(llvm::outs(), File == "-" ? "{standard input}" : File, FD);
}
return EXIT_SUCCESS;
>From f2a884ed210f032ed9288b5de18d6dd211afbc15 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Sun, 26 Jul 2026 00:51:10 +0800
Subject: [PATCH 2/6] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 70 ++++++++++--------------
1 file changed, 28 insertions(+), 42 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 425c9d9831b99..888e8d8bdf955 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -27,7 +27,10 @@
#include "llvm/Support/Program.h"
#include "llvm/Support/WithColor.h"
#include <cctype>
+#include <fstream>
+#include <iostream>
#include <string>
+#include <vector>
using namespace llvm;
using namespace llvm::object;
@@ -90,8 +93,7 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
}
}
-static void strings(raw_ostream &OS, StringRef FileName,
- sys::fs::file_t FileHandle) {
+static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
auto print = [&OS, FileName](unsigned Offset, StringRef L) {
if (L.size() < static_cast<size_t>(MinLength))
@@ -114,38 +116,28 @@ static void strings(raw_ostream &OS, StringRef FileName,
OS << L << '\n';
};
- unsigned Offset = 0, LocalOffset = 0, CurSize = 0;
- Buffer.resize(sys::fs::DefaultReadChunkSize);
- auto FillBuffer = [&Buffer, FileHandle, &FileName]() -> unsigned {
- Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
- FileHandle,
- MutableArrayRef(Buffer.begin(), sys::fs::DefaultReadChunkSize));
- if (!ReadBytesOrErr) {
- errs() << FileName << ": "
- << errorToErrorCode(ReadBytesOrErr.takeError()).message() << '\n';
- return 0;
- }
- return *ReadBytesOrErr;
- };
std::string StringBuffer;
+ unsigned Offset = 0;
while (true) {
- if (LocalOffset == CurSize) {
- CurSize = FillBuffer();
- if (CurSize == 0)
- break;
- LocalOffset = 0;
- }
- char C = Buffer[LocalOffset++];
- if (isPrint(C) || C == '\t') {
- StringBuffer.push_back(C);
- } else if (StringBuffer.size()) {
- print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
- Offset += StringBuffer.size();
- StringBuffer.clear();
- ++Offset;
- } else {
- ++Offset;
+ IS.read(Buffer.data(), Buffer.size());
+ std::streamsize CurSize = IS.gcount();
+ if (CurSize <= 0)
+ break;
+ for (std::streamsize I = 0; I != CurSize; ++I) {
+ char C = Buffer[I];
+ if (isPrint(C) || C == '\t') {
+ StringBuffer.push_back(C);
+ } else if (StringBuffer.size()) {
+ print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
+ Offset += StringBuffer.size();
+ StringBuffer.clear();
+ ++Offset;
+ } else {
+ ++Offset;
+ }
}
+ if (!IS)
+ break;
}
if (StringBuffer.size())
@@ -201,22 +193,16 @@ int main(int argc, char **argv) {
InputFileNames.push_back("-");
for (const auto &File : InputFileNames) {
- sys::fs::file_t FD;
if (File == "-") {
- FD = sys::fs::getStdinHandle();
+ strings(llvm::outs(), "{standard input}", std::cin);
} else {
- Expected<sys::fs::file_t> FDOrErr =
- sys::fs::openNativeFileForRead(File, sys::fs::OF_None);
- ;
- if (!FDOrErr) {
- errs() << File << ": "
- << errorToErrorCode(FDOrErr.takeError()).message() << '\n';
+ std::ifstream IS(File, std::ios::in | std::ios::binary);
+ if (!IS) {
+ errs() << File << ": " << sys::StrError(errno) << '\n';
continue;
}
- FD = *FDOrErr;
+ strings(llvm::outs(), File, IS);
}
-
- strings(llvm::outs(), File == "-" ? "{standard input}" : File, FD);
}
return EXIT_SUCCESS;
>From d63e61523bd40d2cd116c1efde793f1d16abfc20 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Sun, 26 Jul 2026 01:37:51 +0800
Subject: [PATCH 3/6] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 888e8d8bdf955..2dd1450eb50e9 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -95,6 +95,7 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
+ Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
auto print = [&OS, FileName](unsigned Offset, StringRef L) {
if (L.size() < static_cast<size_t>(MinLength))
return;
>From 20b9d55d766aac46f0ecbed287d8584c73491156 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Mon, 27 Jul 2026 19:27:58 +0800
Subject: [PATCH 4/6] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 33 ++++++++++++++----------
1 file changed, 19 insertions(+), 14 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 2dd1450eb50e9..58973797d3fb3 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "Opts.inc"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Object/Binary.h"
#include "llvm/Option/Arg.h"
@@ -28,9 +29,7 @@
#include "llvm/Support/WithColor.h"
#include <cctype>
#include <fstream>
-#include <iostream>
#include <string>
-#include <vector>
using namespace llvm;
using namespace llvm::object;
@@ -93,9 +92,9 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
}
}
-static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
+static void strings(raw_ostream &OS, StringRef FileName,
+ sys::fs::file_t Handle) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
- Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
auto print = [&OS, FileName](unsigned Offset, StringRef L) {
if (L.size() < static_cast<size_t>(MinLength))
return;
@@ -117,14 +116,21 @@ static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
OS << L << '\n';
};
+ Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
std::string StringBuffer;
unsigned Offset = 0;
while (true) {
- IS.read(Buffer.data(), Buffer.size());
- std::streamsize CurSize = IS.gcount();
+ Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
+ Handle, MutableArrayRef(Buffer.data(), Buffer.size()));
+ if (!ReadBytesOrErr) {
+ errs() << FileName << ": "
+ << errorToErrorCode(ReadBytesOrErr.takeError()).message() << '\n';
+ return;
+ }
+ std::size_t CurSize = *ReadBytesOrErr;
if (CurSize <= 0)
break;
- for (std::streamsize I = 0; I != CurSize; ++I) {
+ for (std::size_t I = 0; I != CurSize; ++I) {
char C = Buffer[I];
if (isPrint(C) || C == '\t') {
StringBuffer.push_back(C);
@@ -137,8 +143,6 @@ static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
++Offset;
}
}
- if (!IS)
- break;
}
if (StringBuffer.size())
@@ -195,14 +199,15 @@ int main(int argc, char **argv) {
for (const auto &File : InputFileNames) {
if (File == "-") {
- strings(llvm::outs(), "{standard input}", std::cin);
+ strings(llvm::outs(), "{standard input}", sys::fs::getStdinHandle());
} else {
- std::ifstream IS(File, std::ios::in | std::ios::binary);
- if (!IS) {
- errs() << File << ": " << sys::StrError(errno) << '\n';
+ Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForReadWrite(
+ File, sys::fs::CD_OpenExisting, sys::fs::OF_None);
+ if (!FDOrErr) {
+ errs() << File << ": " << FDOrErr.takeError() << '\n';
continue;
}
- strings(llvm::outs(), File, IS);
+ strings(llvm::outs(), File, *FDOrErr);
}
}
>From a539beb754b4b9c8ba386cfae54715be93ce59cc Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Mon, 27 Jul 2026 19:33:52 +0800
Subject: [PATCH 5/6] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 58973797d3fb3..763872281349d 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -12,14 +12,12 @@
//===----------------------------------------------------------------------===//
#include "Opts.inc"
-#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Object/Binary.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Errno.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Format.h"
@@ -28,7 +26,6 @@
#include "llvm/Support/Program.h"
#include "llvm/Support/WithColor.h"
#include <cctype>
-#include <fstream>
#include <string>
using namespace llvm;
>From 2d49b81bad90667262c72d5a5095ce756fecf3ab Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Thu, 13 Aug 2026 14:32:07 +0800
Subject: [PATCH 6/6] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 84 ++++++++++++++----------
1 file changed, 48 insertions(+), 36 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 763872281349d..2ce9b913dac7f 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -69,7 +69,8 @@ static StringRef ToolName;
static cl::list<std::string> InputFileNames(cl::Positional,
cl::desc("<input object files>"));
-static int MinLength = 4;
+static constexpr int DefaultMinLength = 4;
+static int MinLength = DefaultMinLength;
static bool PrintFileName;
enum radix { none, octal, hexadecimal, decimal };
@@ -92,31 +93,39 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
static void strings(raw_ostream &OS, StringRef FileName,
sys::fs::file_t Handle) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
- auto print = [&OS, FileName](unsigned Offset, StringRef L) {
- if (L.size() < static_cast<size_t>(MinLength))
- return;
- if (PrintFileName)
- OS << FileName << ": ";
- switch (Radix) {
- case none:
- break;
- case octal:
- OS << format("%7o ", Offset);
- break;
- case hexadecimal:
- OS << format("%7x ", Offset);
- break;
- case decimal:
- OS << format("%7u ", Offset);
- break;
+ SmallString<DefaultMinLength> Prefix;
+ auto print = [&OS, FileName, &Prefix](unsigned Offset, StringRef L) {
+ if (Prefix.size() + L.size() >= static_cast<size_t>(MinLength)) {
+ if (PrintFileName)
+ OS << FileName << ": ";
+ switch (Radix) {
+ case none:
+ break;
+ case octal:
+ OS << format("%7o ", Offset);
+ break;
+ case hexadecimal:
+ OS << format("%7x ", Offset);
+ break;
+ case decimal:
+ OS << format("%7u ", Offset);
+ break;
+ }
+ OS << Prefix << L << '\n';
}
- OS << L << '\n';
+ Prefix.clear();
};
Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
- std::string StringBuffer;
+ // Offset of the start of the current chunk within the file.
unsigned Offset = 0;
while (true) {
+ /*
+ * llvm-strings should be able to process a very large file on a
+ * memory-budgeted machine. To handle this, we read the file in chunk
+ * instead of allocate a very large memory and copy the whole file to the
+ * memory.
+ */
Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
Handle, MutableArrayRef(Buffer.data(), Buffer.size()));
if (!ReadBytesOrErr) {
@@ -125,25 +134,28 @@ static void strings(raw_ostream &OS, StringRef FileName,
return;
}
std::size_t CurSize = *ReadBytesOrErr;
- if (CurSize <= 0)
+ if (CurSize == 0)
break;
- for (std::size_t I = 0; I != CurSize; ++I) {
- char C = Buffer[I];
- if (isPrint(C) || C == '\t') {
- StringBuffer.push_back(C);
- } else if (StringBuffer.size()) {
- print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
- Offset += StringBuffer.size();
- StringBuffer.clear();
- ++Offset;
- } else {
- ++Offset;
+
+ const char *B = Buffer.data();
+ const char *E = B + CurSize;
+ const char *S = Prefix.empty() ? nullptr : B;
+ for (const char *P = B; P != E; ++P) {
+ if (isPrint(*P) || *P == '\t') {
+ if (!S)
+ S = P;
+ } else if (S) {
+ print(Offset + (S - B) - Prefix.size(), StringRef(S, P - S));
+ S = nullptr;
}
}
+ if (S)
+ Prefix.append(S, E);
+ Offset += CurSize;
}
- if (StringBuffer.size())
- print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
+ if (!Prefix.empty())
+ print(Offset - Prefix.size(), StringRef());
}
int main(int argc, char **argv) {
@@ -198,8 +210,8 @@ int main(int argc, char **argv) {
if (File == "-") {
strings(llvm::outs(), "{standard input}", sys::fs::getStdinHandle());
} else {
- Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForReadWrite(
- File, sys::fs::CD_OpenExisting, sys::fs::OF_None);
+ Expected<sys::fs::file_t> FDOrErr =
+ sys::fs::openNativeFileForRead(File, sys::fs::OF_TextWithCRLF);
if (!FDOrErr) {
errs() << File << ": " << FDOrErr.takeError() << '\n';
continue;
More information about the llvm-commits
mailing list