[llvm-branch-commits] [llvm] [llvm-strings] Add extra --encoding options (PR #242891)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 15 14:14:20 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-binary-utilities
Author: Harald van Dijk (hvdijk)
<details>
<summary>Changes</summary>
This commit adds support for --encoding=l/b/L/B to allow processing strings encoded as UTF-16 or UTF-32.
---
Full diff: https://github.com/llvm/llvm-project/pull/242891.diff
2 Files Affected:
- (modified) llvm/test/tools/llvm-strings/encoding.test (+42)
- (modified) llvm/tools/llvm-strings/llvm-strings.cpp (+96-9)
``````````diff
diff --git a/llvm/test/tools/llvm-strings/encoding.test b/llvm/test/tools/llvm-strings/encoding.test
index a978499cc52fc..63d281b0cdaa7 100644
--- a/llvm/test/tools/llvm-strings/encoding.test
+++ b/llvm/test/tools/llvm-strings/encoding.test
@@ -56,6 +56,48 @@ CHECK-UTF8:cdef{{\xc2\xa0}}
CHECK-UTF8:defg
CHECK-UTF8:efgh
+# Check UTF-16 and UTF-32 separately.
+# UTF-16LE:
+RUN: echo -n -e 'a\0\n\0' > %t
+RUN: echo -n -e 'a\0b\0\n\0' > %t
+RUN: echo -n -e 'a\0b\0c\0\n\0' > %t
+RUN: echo -n -e 'a\0b\0c\0d\0\n\0' > %t
+RUN: echo -n -e 'b\0c\0d\0e\0\0240\0\n\0' >> %t
+RUN: echo -n -e 'c\0d\0e\0f\0\0240\0\0237\0\n\0' >> %t
+RUN: echo -n -e '\0\0330d\0e\0f\0g\0\0337\0337\n\0' >> %t
+RUN: echo -n -e '\0\0330\0\0330e\0f\0g\0h\0\0337\0337\n\0' >> %t
+RUN: llvm-strings -e l %t | FileCheck --check-prefixes CHECK,CHECK-UTF8 --strict-whitespace --match-full-lines %s
+# UTF-16BE:
+RUN: echo -n -e '\0a\0\n' > %t
+RUN: echo -n -e '\0a\0b\0\n' > %t
+RUN: echo -n -e '\0a\0b\0c\0\n' > %t
+RUN: echo -n -e '\0a\0b\0c\0d\0\n' > %t
+RUN: echo -n -e '\0b\0c\0d\0e\0\0240\0\n' >> %t
+RUN: echo -n -e '\0c\0d\0e\0f\0\0240\0\0237\0\n' >> %t
+RUN: echo -n -e '\0330\0\0d\0e\0f\0g\0337\0337\0\n' >> %t
+RUN: echo -n -e '\0330\0\0330\0\0e\0f\0g\0h\0337\0337\0\n' >> %t
+RUN: llvm-strings -e b %t | FileCheck --check-prefixes CHECK,CHECK-UTF8 --strict-whitespace --match-full-lines %s
+# UTF-32LE:
+RUN: echo -n -e 'a\0\0\0\n\0\0\0' > %t
+RUN: echo -n -e 'a\0\0\0b\0\0\0\n\0\0\0' > %t
+RUN: echo -n -e 'a\0\0\0b\0\0\0c\0\0\0\n\0\0\0' > %t
+RUN: echo -n -e 'a\0\0\0b\0\0\0c\0\0\0d\0\0\0\n\0\0\0' > %t
+RUN: echo -n -e 'b\0\0\0c\0\0\0d\0\0\0e\0\0\0\0240\0\0\0\n\0\0\0' >> %t
+RUN: echo -n -e 'c\0\0\0d\0\0\0e\0\0\0f\0\0\0\0240\0\0\0\0237\0\0\0\n\0\0\0' >> %t
+RUN: echo -n -e '\0\0330\0\0d\0\0\0e\0\0\0f\0\0\0g\0\0\0\0337\0337\0\0\n\0\0\0' >> %t
+RUN: echo -n -e '\0\0330\0\0\0\0337\0\0e\0\0\0f\0\0\0g\0\0\0h\0\0\0\0377\0377\0377\0377\n\0\0\0' >> %t
+RUN: llvm-strings -e L %t | FileCheck --check-prefixes CHECK,CHECK-UTF8 --strict-whitespace --match-full-lines %s
+# UTF-32BE:
+RUN: echo -n -e '\0\0\0a\0\0\0\n' > %t
+RUN: echo -n -e '\0\0\0a\0\0\0b\0\0\0\n' > %t
+RUN: echo -n -e '\0\0\0a\0\0\0b\0\0\0c\0\0\0\n' > %t
+RUN: echo -n -e '\0\0\0a\0\0\0b\0\0\0c\0\0\0d\0\0\0\n' > %t
+RUN: echo -n -e '\0\0\0b\0\0\0c\0\0\0d\0\0\0e\0\0\0\0240\0\0\0\n' >> %t
+RUN: echo -n -e '\0\0\0c\0\0\0d\0\0\0e\0\0\0f\0\0\0\0240\0\0\0\0237\0\0\0\n' >> %t
+RUN: echo -n -e '\0\0\0330\0\0\0\0d\0\0\0e\0\0\0f\0\0\0g\0\0\0337\0337\0\0\0\n' >> %t
+RUN: echo -n -e '\0\0\0330\0\0\0\0337\0\0\0\0e\0\0\0f\0\0\0g\0\0\0h\0377\0377\0377\0377\0\0\0\n' >> %t
+RUN: llvm-strings -e B %t | FileCheck --check-prefixes CHECK,CHECK-UTF8 --strict-whitespace --match-full-lines %s
+
# Check that invalid --encoding values are rejected.
RUN: not llvm-strings --encoding x 2>&1 %t | FileCheck --check-prefix CHECK-ERROR %s
RUN: not llvm-strings -e x 2>&1 %t | FileCheck --check-prefix CHECK-ERROR %s
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 1265555cb8085..b655be3c837f4 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -138,7 +138,7 @@ static constexpr int DefaultMinLength = 4;
static int MinLength = DefaultMinLength;
static bool PrintFileName;
-enum class Encoding { Ascii, Locale, Utf8 };
+enum class Encoding { Ascii, Locale, Utf8, Utf16LE, Utf16BE, Utf32LE, Utf32BE };
static Encoding Encoding;
enum class Radix { None, Octal, Hexadecimal, Decimal };
@@ -536,8 +536,8 @@ struct CharSet {
}
};
- // UTF-8 is stateless but with -eutf8 -Ulocale, we may convert it to a
- // stateful encoding.
+ // UTF encodings are stateless but with e.g. -eutf8 -Ulocale, we may convert
+ // to a stateful encoding.
struct Utf8 : MultiByteStateful {
static bool readChar(const char *&Cur, const char *End, bool AtEOF,
CharT &Ch) {
@@ -577,15 +577,90 @@ struct CharSet {
return;
}
- const UTF8 *Cur = reinterpret_cast<const UTF8 *>(Str.begin()),
- *End = reinterpret_cast<const UTF8 *>(Str.end());
- while (Cur != End) {
- UTF32 UCh;
- UTF32 *UTF32Next = &UCh;
- ConvertUTF8toUTF32(&Cur, End, &UTF32Next, &UCh + 1, strictConversion);
+ const char *Cur = Str.begin(), *End = Str.end();
+ CharT UCh;
+ while (readChar(Cur, End, /*AtEOF=*/true, UCh))
+ writeChar(OS, UCh);
+ }
+ };
+
+ struct Utf16 : MultiByteStateful {
+ endianness Endianness = getEndianness();
+ static endianness getEndianness() {
+ switch (Encoding) {
+ case Encoding::Utf16LE:
+ return endianness::little;
+ case Encoding::Utf16BE:
+ return endianness::big;
+ default:
+ llvm_unreachable("Utf16 is only used with --encoding=l/b");
+ }
+ }
+
+ bool readChar(const char *&Cur, const char *End, bool AtEOF, CharT &Ch) {
+ if (size_t(End - Cur) < sizeof(UTF16))
+ return false;
+
+ UTF16 UTF16Buf[2];
+ const UTF16 *UTF16Cur = UTF16Buf;
+ UTF16 *UTF16End = UTF16Buf;
+ *UTF16End++ = support::endian::read<UTF16>(Cur, Endianness);
+ if (size_t(End - Cur) >= 2 * sizeof(UTF16))
+ *UTF16End++ =
+ support::endian::read<UTF16>(Cur + sizeof(UTF16), Endianness);
+ UTF32 *UTF32Next = &Ch;
+ const auto Res = ConvertUTF16toUTF32(&UTF16Cur, UTF16End, &UTF32Next,
+ &Ch + 1, strictConversion);
+ if (UTF16Cur != UTF16Buf) {
+ assert(UTF32Next != &Ch);
+ Cur += (UTF16Cur - UTF16Buf) * sizeof(UTF16);
+ return true;
+ } else if (Res == sourceExhausted && !AtEOF) {
+ return false;
+ } else {
+ assert(UTF32Next == &Ch);
+ Cur += sizeof(UTF16);
+ Ch = 0;
+ return true;
+ }
+ }
+
+ void writeString(raw_ostream &OS, StringRef Str) {
+ const char *Cur = Str.begin(), *End = Str.end();
+ CharT UCh;
+ while (readChar(Cur, End, /*AtEOF=*/true, UCh))
writeChar(OS, UCh);
+ }
+ };
+
+ struct Utf32 : MultiByteStateful {
+ endianness Endianness = getEndianness();
+ static endianness getEndianness() {
+ switch (Encoding) {
+ case Encoding::Utf32LE:
+ return endianness::little;
+ case Encoding::Utf32BE:
+ return endianness::big;
+ default:
+ llvm_unreachable("Utf32 is only used with --encoding=L/B");
}
}
+
+ bool readChar(const char *&Cur, const char *End, bool AtEOF, CharT &Ch) {
+ if (size_t(End - Cur) < sizeof(UTF32))
+ return false;
+
+ Ch = std::min<UTF32>(support::endian::readNext<UTF32>(Cur, Endianness),
+ 0x10FFFF);
+ return true;
+ }
+
+ void writeString(raw_ostream &OS, StringRef Str) {
+ const char *Cur = Str.begin(), *End = Str.end();
+ CharT UCh;
+ while (readChar(Cur, End, /*AtEOF=*/true, UCh))
+ writeChar(OS, UCh);
+ }
};
struct SingleByte : Base<unsigned char> {
@@ -867,6 +942,10 @@ int main(int argc, char **argv) {
.Case("s", Encoding::Ascii)
.Case("S", Encoding::Locale)
.Case("utf8", Encoding::Utf8)
+ .Case("l", Encoding::Utf16LE)
+ .Case("b", Encoding::Utf16BE)
+ .Case("L", Encoding::Utf32LE)
+ .Case("B", Encoding::Utf32BE)
.Default(std::nullopt);
if (!EncodingVal)
invalidArgValue(EncodingArg);
@@ -961,6 +1040,14 @@ int main(int argc, char **argv) {
return strings<CharSet::Ascii>;
else
return strings<CharSet::Utf8>;
+
+ case Encoding::Utf16LE:
+ case Encoding::Utf16BE:
+ return strings<CharSet::Utf16>;
+
+ case Encoding::Utf32LE:
+ case Encoding::Utf32BE:
+ return strings<CharSet::Utf32>;
}
llvm_unreachable("unhandled encoding");
``````````
</details>
https://github.com/llvm/llvm-project/pull/242891
More information about the llvm-branch-commits
mailing list