[llvm] [llvm-strings] Use small buffer instead of reading whole file (PR #163073)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 17 03:14:19 PDT 2026
https://github.com/aokblast updated https://github.com/llvm/llvm-project/pull/163073
>From 42984b543dafa8731fbf2034331c8d81c8d7ed65 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Sun, 12 Oct 2025 22:01:13 +0800
Subject: [PATCH 1/7] [llvm-strings] Use small buffer instead of reading whole
file
To prevent large memory footprint, we use small buffer and a seperated
string to print. This can largely reduce the memory consumption when
processing.
In the following example, the test.file is 1G regular file, which is possible in the
daily workload.
cat ./test.file | llvm-strings --radix=o > out &&
ps -o command,vsz,rss | grep strings
Reading whole file:
llvm-strings 5129824 3999920
Small buffer implementation:
llvm-strings 17604 5476
Also, it does not affect too much runtime:
time llvm-strings --radix=o > out < ./test.file
Reading whole file:
________________________________________________________
Executed in 5.67 secs fish external
usr time 4.38 secs 2.40 millis 4.38 secs
sys time 1.28 secs 0.00 millis 1.28 secs
Small buffer implementation:
________________________________________________________
Executed in 5.87 secs fish external
usr time 5.65 secs 2.59 millis 5.65 secs
sys time 0.21 secs 0.00 millis 0.21 secs
---
llvm/tools/llvm-strings/llvm-strings.cpp | 74 ++++++++++++++++++------
1 file changed, 55 insertions(+), 19 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 9979b93de8427..425c9d9831b99 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -18,7 +18,9 @@
#include "llvm/Option/ArgList.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Errno.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/MemoryBuffer.h"
@@ -88,7 +90,9 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
}
}
-static void strings(raw_ostream &OS, StringRef FileName, StringRef Contents) {
+static void strings(raw_ostream &OS, StringRef FileName,
+ sys::fs::file_t FileHandle) {
+ SmallString<sys::fs::DefaultReadChunkSize> Buffer;
auto print = [&OS, FileName](unsigned Offset, StringRef L) {
if (L.size() < static_cast<size_t>(MinLength))
return;
@@ -110,19 +114,42 @@ static void strings(raw_ostream &OS, StringRef FileName, StringRef Contents) {
OS << L << '\n';
};
- const char *B = Contents.begin();
- const char *P = nullptr, *E = nullptr, *S = nullptr;
- for (P = Contents.begin(), E = Contents.end(); P < E; ++P) {
- if (isPrint(*P) || *P == '\t') {
- if (S == nullptr)
- S = P;
- } else if (S) {
- print(S - B, StringRef(S, P - S));
- S = nullptr;
+ unsigned Offset = 0, LocalOffset = 0, CurSize = 0;
+ Buffer.resize(sys::fs::DefaultReadChunkSize);
+ auto FillBuffer = [&Buffer, FileHandle, &FileName]() -> unsigned {
+ Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
+ FileHandle,
+ MutableArrayRef(Buffer.begin(), sys::fs::DefaultReadChunkSize));
+ if (!ReadBytesOrErr) {
+ errs() << FileName << ": "
+ << errorToErrorCode(ReadBytesOrErr.takeError()).message() << '\n';
+ return 0;
+ }
+ return *ReadBytesOrErr;
+ };
+ std::string StringBuffer;
+ while (true) {
+ if (LocalOffset == CurSize) {
+ CurSize = FillBuffer();
+ if (CurSize == 0)
+ break;
+ LocalOffset = 0;
+ }
+ char C = Buffer[LocalOffset++];
+ if (isPrint(C) || C == '\t') {
+ StringBuffer.push_back(C);
+ } else if (StringBuffer.size()) {
+ print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
+ Offset += StringBuffer.size();
+ StringBuffer.clear();
+ ++Offset;
+ } else {
+ ++Offset;
}
}
- if (S)
- print(S - B, StringRef(S, E - S));
+
+ if (StringBuffer.size())
+ print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
}
int main(int argc, char **argv) {
@@ -174,13 +201,22 @@ int main(int argc, char **argv) {
InputFileNames.push_back("-");
for (const auto &File : InputFileNames) {
- ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
- MemoryBuffer::getFileOrSTDIN(File, /*IsText=*/true);
- if (std::error_code EC = Buffer.getError())
- errs() << File << ": " << EC.message() << '\n';
- else
- strings(llvm::outs(), File == "-" ? "{standard input}" : File,
- Buffer.get()->getMemBufferRef().getBuffer());
+ sys::fs::file_t FD;
+ if (File == "-") {
+ FD = sys::fs::getStdinHandle();
+ } else {
+ Expected<sys::fs::file_t> FDOrErr =
+ sys::fs::openNativeFileForRead(File, sys::fs::OF_None);
+ ;
+ if (!FDOrErr) {
+ errs() << File << ": "
+ << errorToErrorCode(FDOrErr.takeError()).message() << '\n';
+ continue;
+ }
+ FD = *FDOrErr;
+ }
+
+ strings(llvm::outs(), File == "-" ? "{standard input}" : File, FD);
}
return EXIT_SUCCESS;
>From f2a884ed210f032ed9288b5de18d6dd211afbc15 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Sun, 26 Jul 2026 00:51:10 +0800
Subject: [PATCH 2/7] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 70 ++++++++++--------------
1 file changed, 28 insertions(+), 42 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 425c9d9831b99..888e8d8bdf955 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -27,7 +27,10 @@
#include "llvm/Support/Program.h"
#include "llvm/Support/WithColor.h"
#include <cctype>
+#include <fstream>
+#include <iostream>
#include <string>
+#include <vector>
using namespace llvm;
using namespace llvm::object;
@@ -90,8 +93,7 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
}
}
-static void strings(raw_ostream &OS, StringRef FileName,
- sys::fs::file_t FileHandle) {
+static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
auto print = [&OS, FileName](unsigned Offset, StringRef L) {
if (L.size() < static_cast<size_t>(MinLength))
@@ -114,38 +116,28 @@ static void strings(raw_ostream &OS, StringRef FileName,
OS << L << '\n';
};
- unsigned Offset = 0, LocalOffset = 0, CurSize = 0;
- Buffer.resize(sys::fs::DefaultReadChunkSize);
- auto FillBuffer = [&Buffer, FileHandle, &FileName]() -> unsigned {
- Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
- FileHandle,
- MutableArrayRef(Buffer.begin(), sys::fs::DefaultReadChunkSize));
- if (!ReadBytesOrErr) {
- errs() << FileName << ": "
- << errorToErrorCode(ReadBytesOrErr.takeError()).message() << '\n';
- return 0;
- }
- return *ReadBytesOrErr;
- };
std::string StringBuffer;
+ unsigned Offset = 0;
while (true) {
- if (LocalOffset == CurSize) {
- CurSize = FillBuffer();
- if (CurSize == 0)
- break;
- LocalOffset = 0;
- }
- char C = Buffer[LocalOffset++];
- if (isPrint(C) || C == '\t') {
- StringBuffer.push_back(C);
- } else if (StringBuffer.size()) {
- print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
- Offset += StringBuffer.size();
- StringBuffer.clear();
- ++Offset;
- } else {
- ++Offset;
+ IS.read(Buffer.data(), Buffer.size());
+ std::streamsize CurSize = IS.gcount();
+ if (CurSize <= 0)
+ break;
+ for (std::streamsize I = 0; I != CurSize; ++I) {
+ char C = Buffer[I];
+ if (isPrint(C) || C == '\t') {
+ StringBuffer.push_back(C);
+ } else if (StringBuffer.size()) {
+ print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
+ Offset += StringBuffer.size();
+ StringBuffer.clear();
+ ++Offset;
+ } else {
+ ++Offset;
+ }
}
+ if (!IS)
+ break;
}
if (StringBuffer.size())
@@ -201,22 +193,16 @@ int main(int argc, char **argv) {
InputFileNames.push_back("-");
for (const auto &File : InputFileNames) {
- sys::fs::file_t FD;
if (File == "-") {
- FD = sys::fs::getStdinHandle();
+ strings(llvm::outs(), "{standard input}", std::cin);
} else {
- Expected<sys::fs::file_t> FDOrErr =
- sys::fs::openNativeFileForRead(File, sys::fs::OF_None);
- ;
- if (!FDOrErr) {
- errs() << File << ": "
- << errorToErrorCode(FDOrErr.takeError()).message() << '\n';
+ std::ifstream IS(File, std::ios::in | std::ios::binary);
+ if (!IS) {
+ errs() << File << ": " << sys::StrError(errno) << '\n';
continue;
}
- FD = *FDOrErr;
+ strings(llvm::outs(), File, IS);
}
-
- strings(llvm::outs(), File == "-" ? "{standard input}" : File, FD);
}
return EXIT_SUCCESS;
>From d63e61523bd40d2cd116c1efde793f1d16abfc20 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Sun, 26 Jul 2026 01:37:51 +0800
Subject: [PATCH 3/7] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 888e8d8bdf955..2dd1450eb50e9 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -95,6 +95,7 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
+ Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
auto print = [&OS, FileName](unsigned Offset, StringRef L) {
if (L.size() < static_cast<size_t>(MinLength))
return;
>From 20b9d55d766aac46f0ecbed287d8584c73491156 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Mon, 27 Jul 2026 19:27:58 +0800
Subject: [PATCH 4/7] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 33 ++++++++++++++----------
1 file changed, 19 insertions(+), 14 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 2dd1450eb50e9..58973797d3fb3 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "Opts.inc"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Object/Binary.h"
#include "llvm/Option/Arg.h"
@@ -28,9 +29,7 @@
#include "llvm/Support/WithColor.h"
#include <cctype>
#include <fstream>
-#include <iostream>
#include <string>
-#include <vector>
using namespace llvm;
using namespace llvm::object;
@@ -93,9 +92,9 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
}
}
-static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
+static void strings(raw_ostream &OS, StringRef FileName,
+ sys::fs::file_t Handle) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
- Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
auto print = [&OS, FileName](unsigned Offset, StringRef L) {
if (L.size() < static_cast<size_t>(MinLength))
return;
@@ -117,14 +116,21 @@ static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
OS << L << '\n';
};
+ Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
std::string StringBuffer;
unsigned Offset = 0;
while (true) {
- IS.read(Buffer.data(), Buffer.size());
- std::streamsize CurSize = IS.gcount();
+ Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
+ Handle, MutableArrayRef(Buffer.data(), Buffer.size()));
+ if (!ReadBytesOrErr) {
+ errs() << FileName << ": "
+ << errorToErrorCode(ReadBytesOrErr.takeError()).message() << '\n';
+ return;
+ }
+ std::size_t CurSize = *ReadBytesOrErr;
if (CurSize <= 0)
break;
- for (std::streamsize I = 0; I != CurSize; ++I) {
+ for (std::size_t I = 0; I != CurSize; ++I) {
char C = Buffer[I];
if (isPrint(C) || C == '\t') {
StringBuffer.push_back(C);
@@ -137,8 +143,6 @@ static void strings(raw_ostream &OS, StringRef FileName, std::istream &IS) {
++Offset;
}
}
- if (!IS)
- break;
}
if (StringBuffer.size())
@@ -195,14 +199,15 @@ int main(int argc, char **argv) {
for (const auto &File : InputFileNames) {
if (File == "-") {
- strings(llvm::outs(), "{standard input}", std::cin);
+ strings(llvm::outs(), "{standard input}", sys::fs::getStdinHandle());
} else {
- std::ifstream IS(File, std::ios::in | std::ios::binary);
- if (!IS) {
- errs() << File << ": " << sys::StrError(errno) << '\n';
+ Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForReadWrite(
+ File, sys::fs::CD_OpenExisting, sys::fs::OF_None);
+ if (!FDOrErr) {
+ errs() << File << ": " << FDOrErr.takeError() << '\n';
continue;
}
- strings(llvm::outs(), File, IS);
+ strings(llvm::outs(), File, *FDOrErr);
}
}
>From a539beb754b4b9c8ba386cfae54715be93ce59cc Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Mon, 27 Jul 2026 19:33:52 +0800
Subject: [PATCH 5/7] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 58973797d3fb3..763872281349d 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -12,14 +12,12 @@
//===----------------------------------------------------------------------===//
#include "Opts.inc"
-#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Object/Binary.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Errno.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Format.h"
@@ -28,7 +26,6 @@
#include "llvm/Support/Program.h"
#include "llvm/Support/WithColor.h"
#include <cctype>
-#include <fstream>
#include <string>
using namespace llvm;
>From 2d49b81bad90667262c72d5a5095ce756fecf3ab Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Thu, 13 Aug 2026 14:32:07 +0800
Subject: [PATCH 6/7] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
llvm/tools/llvm-strings/llvm-strings.cpp | 84 ++++++++++++++----------
1 file changed, 48 insertions(+), 36 deletions(-)
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 763872281349d..2ce9b913dac7f 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -69,7 +69,8 @@ static StringRef ToolName;
static cl::list<std::string> InputFileNames(cl::Positional,
cl::desc("<input object files>"));
-static int MinLength = 4;
+static constexpr int DefaultMinLength = 4;
+static int MinLength = DefaultMinLength;
static bool PrintFileName;
enum radix { none, octal, hexadecimal, decimal };
@@ -92,31 +93,39 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
static void strings(raw_ostream &OS, StringRef FileName,
sys::fs::file_t Handle) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
- auto print = [&OS, FileName](unsigned Offset, StringRef L) {
- if (L.size() < static_cast<size_t>(MinLength))
- return;
- if (PrintFileName)
- OS << FileName << ": ";
- switch (Radix) {
- case none:
- break;
- case octal:
- OS << format("%7o ", Offset);
- break;
- case hexadecimal:
- OS << format("%7x ", Offset);
- break;
- case decimal:
- OS << format("%7u ", Offset);
- break;
+ SmallString<DefaultMinLength> Prefix;
+ auto print = [&OS, FileName, &Prefix](unsigned Offset, StringRef L) {
+ if (Prefix.size() + L.size() >= static_cast<size_t>(MinLength)) {
+ if (PrintFileName)
+ OS << FileName << ": ";
+ switch (Radix) {
+ case none:
+ break;
+ case octal:
+ OS << format("%7o ", Offset);
+ break;
+ case hexadecimal:
+ OS << format("%7x ", Offset);
+ break;
+ case decimal:
+ OS << format("%7u ", Offset);
+ break;
+ }
+ OS << Prefix << L << '\n';
}
- OS << L << '\n';
+ Prefix.clear();
};
Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
- std::string StringBuffer;
+ // Offset of the start of the current chunk within the file.
unsigned Offset = 0;
while (true) {
+ /*
+ * llvm-strings should be able to process a very large file on a
+ * memory-budgeted machine. To handle this, we read the file in chunk
+ * instead of allocate a very large memory and copy the whole file to the
+ * memory.
+ */
Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
Handle, MutableArrayRef(Buffer.data(), Buffer.size()));
if (!ReadBytesOrErr) {
@@ -125,25 +134,28 @@ static void strings(raw_ostream &OS, StringRef FileName,
return;
}
std::size_t CurSize = *ReadBytesOrErr;
- if (CurSize <= 0)
+ if (CurSize == 0)
break;
- for (std::size_t I = 0; I != CurSize; ++I) {
- char C = Buffer[I];
- if (isPrint(C) || C == '\t') {
- StringBuffer.push_back(C);
- } else if (StringBuffer.size()) {
- print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
- Offset += StringBuffer.size();
- StringBuffer.clear();
- ++Offset;
- } else {
- ++Offset;
+
+ const char *B = Buffer.data();
+ const char *E = B + CurSize;
+ const char *S = Prefix.empty() ? nullptr : B;
+ for (const char *P = B; P != E; ++P) {
+ if (isPrint(*P) || *P == '\t') {
+ if (!S)
+ S = P;
+ } else if (S) {
+ print(Offset + (S - B) - Prefix.size(), StringRef(S, P - S));
+ S = nullptr;
}
}
+ if (S)
+ Prefix.append(S, E);
+ Offset += CurSize;
}
- if (StringBuffer.size())
- print(Offset, StringRef(StringBuffer.c_str(), StringBuffer.size()));
+ if (!Prefix.empty())
+ print(Offset - Prefix.size(), StringRef());
}
int main(int argc, char **argv) {
@@ -198,8 +210,8 @@ int main(int argc, char **argv) {
if (File == "-") {
strings(llvm::outs(), "{standard input}", sys::fs::getStdinHandle());
} else {
- Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForReadWrite(
- File, sys::fs::CD_OpenExisting, sys::fs::OF_None);
+ Expected<sys::fs::file_t> FDOrErr =
+ sys::fs::openNativeFileForRead(File, sys::fs::OF_TextWithCRLF);
if (!FDOrErr) {
errs() << File << ": " << FDOrErr.takeError() << '\n';
continue;
>From 83b62db4b0c774bf5047d4684248ccfda658e019 Mon Sep 17 00:00:00 2001
From: ShengYi Hung <aokblast at FreeBSD.org>
Date: Mon, 17 Aug 2026 03:01:07 -0700
Subject: [PATCH 7/7] fixup! [llvm-strings] Use small buffer instead of reading
whole file
---
.../tools/llvm-strings/chunk-boundary.test | 56 +++++++++++
llvm/test/tools/llvm-strings/errors.test | 9 ++
llvm/test/tools/llvm-strings/read-error.test | 13 +++
llvm/tools/llvm-strings/llvm-strings.cpp | 99 ++++++++++---------
4 files changed, 133 insertions(+), 44 deletions(-)
create mode 100644 llvm/test/tools/llvm-strings/chunk-boundary.test
create mode 100644 llvm/test/tools/llvm-strings/errors.test
create mode 100644 llvm/test/tools/llvm-strings/read-error.test
diff --git a/llvm/test/tools/llvm-strings/chunk-boundary.test b/llvm/test/tools/llvm-strings/chunk-boundary.test
new file mode 100644
index 0000000000000..9949bd76e5116
--- /dev/null
+++ b/llvm/test/tools/llvm-strings/chunk-boundary.test
@@ -0,0 +1,56 @@
+## Show that strings interacting with the read-chunk boundary are reported
+## correctly. The input files are crafted assuming the native read chunk size
+## (sys::fs::DefaultReadChunkSize) of 16384 bytes.
+
+## Case 1: at least min string size appears before the boundary, unprintable
+## byte as first byte of the next chunk. The string is printed on its own,
+## with the offset of its start (0x3ff8 = 16384 - 8).
+RUN: %python -c "import sys; sys.stdout.buffer.write(b'\0' * 16376 + b'ENDCHUNK' + b'\0' * 4)" > %t.1
+RUN: llvm-strings --radix=x %t.1 | FileCheck %s --check-prefix=CASE1 --strict-whitespace --implicit-check-not={{.}}
+
+CASE1:{{^}} 3ff8 ENDCHUNK{{$}}
+
+## Case 2: at least min string size appears before the boundary, printable
+## byte as first byte of the next chunk. The prefix is printed together with
+## the following characters, as a single string.
+RUN: %python -c "import sys; sys.stdout.buffer.write(b'\0' * 16378 + b'BEFORE' + b'AFTER!' + b'\0' * 4)" > %t.2
+RUN: llvm-strings --radix=x %t.2 | FileCheck %s --check-prefix=CASE2 --strict-whitespace --implicit-check-not={{.}}
+
+CASE2:{{^}} 3ffa BEFOREAFTER!{{$}}
+
+## Case 3: less than min string size appears before the boundary, unprintable
+## byte as the next byte. The prefix is not printed.
+RUN: %python -c "import sys; sys.stdout.buffer.write(b'\0' * 16382 + b'AB' + b'\0' * 4)" > %t.3
+RUN: llvm-strings %t.3 | count 0
+
+## Case 4: less than min string size appears before the boundary, printable
+## bytes as the next bytes, forming a min length string. The prefix is
+## printed together with the following characters, with the offset of its
+## true start (0x3ffd = 16384 - 3).
+RUN: %python -c "import sys; sys.stdout.buffer.write(b'\0' * 16381 + b'ABC' + b'DEF' + b'\0' * 4)" > %t.4
+RUN: llvm-strings --radix=x %t.4 | FileCheck %s --check-prefix=CASE4 --strict-whitespace --implicit-check-not={{.}}
+
+CASE4:{{^}} 3ffd ABCDEF{{$}}
+
+## Case 5: the prefix is empty at the start of a chunk that starts with a min
+## length string (0x4000 = 16384).
+RUN: %python -c "import sys; sys.stdout.buffer.write(b'\0' * 16384 + b'FRESH' + b'\0' * 4)" > %t.5
+RUN: llvm-strings --radix=x %t.5 | FileCheck %s --check-prefix=CASE5 --strict-whitespace --implicit-check-not={{.}}
+
+CASE5:{{^}} 4000 FRESH{{$}}
+
+## Case 6: a string that spans the entirety of one chunk, with one character
+## before the chunk start and one after its end. The string is printed
+## intact, once. The expected output is generated rather than written as a
+## CHECK line because it is a chunk long.
+RUN: %python -c "import sys; sys.stdout.buffer.write(b'\0' * 16383 + b'S' + b'M' * 16384 + b'E' + b'\0' * 4)" > %t.6
+RUN: %python -c "import sys; sys.stdout.write('S' + 'M' * 16384 + 'E\n')" > %t.6.expected
+RUN: llvm-strings %t.6 > %t.6.out
+RUN: diff %t.6.expected %t.6.out
+
+## A string terminated by the end of the file (no trailing unprintable byte)
+## must still be printed.
+RUN: %python -c "import sys; sys.stdout.buffer.write(b'\0' * 16380 + b'TRAILING')" > %t.7
+RUN: llvm-strings %t.7 | FileCheck %s --check-prefix=EOF --strict-whitespace --implicit-check-not={{.}}
+
+EOF:{{^}}TRAILING{{$}}
diff --git a/llvm/test/tools/llvm-strings/errors.test b/llvm/test/tools/llvm-strings/errors.test
new file mode 100644
index 0000000000000..d2433665bb57a
--- /dev/null
+++ b/llvm/test/tools/llvm-strings/errors.test
@@ -0,0 +1,9 @@
+## Show that a file that cannot be opened is reported on stderr, and that
+## processing continues with the remaining inputs.
+
+RUN: rm -rf %t && mkdir -p %t
+RUN: echo abcd > %t/good
+RUN: llvm-strings %t/does-not-exist %t/good 2>&1 | FileCheck %s -DFILE=%t/does-not-exist
+
+CHECK: [[FILE]]: {{[Nn]}}o such file or directory
+CHECK: abcd
diff --git a/llvm/test/tools/llvm-strings/read-error.test b/llvm/test/tools/llvm-strings/read-error.test
new file mode 100644
index 0000000000000..75e49399655e0
--- /dev/null
+++ b/llvm/test/tools/llvm-strings/read-error.test
@@ -0,0 +1,13 @@
+## Show that a file that opens but cannot be read is reported on stderr, and
+## that processing continues with the remaining inputs. A directory can be
+## opened for reading on POSIX systems, but reading from it fails with
+## EISDIR. On Windows opening the directory fails instead, so the read error
+## path is not reachable this way.
+# UNSUPPORTED: system-windows
+
+RUN: rm -rf %t && mkdir -p %t/dir
+RUN: echo abcd > %t/good
+RUN: llvm-strings %t/dir %t/good 2>&1 | FileCheck %s -DFILE=%t/dir
+
+CHECK: [[FILE]]: {{[Ii]}}s a directory
+CHECK: abcd
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 2ce9b913dac7f..b575601c9aca1 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -90,42 +90,38 @@ static void parseIntArg(const opt::InputArgList &Args, int ID, T &Value) {
}
}
+static bool isStringChar(char C) { return isPrint(C) || C == '\t'; }
+
static void strings(raw_ostream &OS, StringRef FileName,
sys::fs::file_t Handle) {
SmallString<sys::fs::DefaultReadChunkSize> Buffer;
- SmallString<DefaultMinLength> Prefix;
- auto print = [&OS, FileName, &Prefix](unsigned Offset, StringRef L) {
- if (Prefix.size() + L.size() >= static_cast<size_t>(MinLength)) {
- if (PrintFileName)
- OS << FileName << ": ";
- switch (Radix) {
- case none:
- break;
- case octal:
- OS << format("%7o ", Offset);
- break;
- case hexadecimal:
- OS << format("%7x ", Offset);
- break;
- case decimal:
- OS << format("%7u ", Offset);
- break;
- }
- OS << Prefix << L << '\n';
+ auto printHeader = [&OS, FileName](unsigned StringStart) {
+ if (PrintFileName)
+ OS << FileName << ": ";
+ switch (Radix) {
+ case none:
+ break;
+ case octal:
+ OS << format("%7o ", StringStart);
+ break;
+ case hexadecimal:
+ OS << format("%7x ", StringStart);
+ break;
+ case decimal:
+ OS << format("%7u ", StringStart);
+ break;
}
- Prefix.clear();
};
+ // llvm-strings should be able to process a very large file on a
+ // memory-budgeted machine, so the file is read in chunks. To handle this, we
+ // read the file in chunk instead of copying the whole file into memory.
+ SmallString<DefaultMinLength> Candidate;
+ bool InString = false;
+ unsigned StringStart = 0, Offset = 0;
+
Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
- // Offset of the start of the current chunk within the file.
- unsigned Offset = 0;
while (true) {
- /*
- * llvm-strings should be able to process a very large file on a
- * memory-budgeted machine. To handle this, we read the file in chunk
- * instead of allocate a very large memory and copy the whole file to the
- * memory.
- */
Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
Handle, MutableArrayRef(Buffer.data(), Buffer.size()));
if (!ReadBytesOrErr) {
@@ -137,25 +133,40 @@ static void strings(raw_ostream &OS, StringRef FileName,
if (CurSize == 0)
break;
- const char *B = Buffer.data();
- const char *E = B + CurSize;
- const char *S = Prefix.empty() ? nullptr : B;
- for (const char *P = B; P != E; ++P) {
- if (isPrint(*P) || *P == '\t') {
- if (!S)
- S = P;
- } else if (S) {
- print(Offset + (S - B) - Prefix.size(), StringRef(S, P - S));
- S = nullptr;
+ std::size_t I = 0;
+ while (I != CurSize) {
+ if (InString) {
+ std::size_t Start = I;
+ while (I != CurSize && isStringChar(Buffer[I]))
+ ++I;
+ OS << StringRef(Buffer.data() + Start, I - Start);
+ Offset += I - Start;
+ if (I != CurSize) {
+ OS << '\n';
+ InString = false;
+ }
+ } else if (isStringChar(Buffer[I])) {
+ if (Candidate.empty())
+ StringStart = Offset;
+ Candidate.push_back(Buffer[I]);
+ ++I;
+ ++Offset;
+ if (Candidate.size() >= static_cast<size_t>(MinLength)) {
+ printHeader(StringStart);
+ OS << Candidate;
+ Candidate.clear();
+ InString = true;
+ }
+ } else {
+ Candidate.clear();
+ ++I;
+ ++Offset;
}
}
- if (S)
- Prefix.append(S, E);
- Offset += CurSize;
}
- if (!Prefix.empty())
- print(Offset - Prefix.size(), StringRef());
+ if (InString)
+ OS << '\n';
}
int main(int argc, char **argv) {
@@ -213,7 +224,7 @@ int main(int argc, char **argv) {
Expected<sys::fs::file_t> FDOrErr =
sys::fs::openNativeFileForRead(File, sys::fs::OF_TextWithCRLF);
if (!FDOrErr) {
- errs() << File << ": " << FDOrErr.takeError() << '\n';
+ errs() << File << ": " << toString(FDOrErr.takeError()) << '\n';
continue;
}
strings(llvm::outs(), File, *FDOrErr);
More information about the llvm-commits
mailing list