[llvm-branch-commits] [llvm] [llvm-strings] Add extra --encoding options (PR #242891)

Harald van Dijk via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sun Nov 15 14:13:41 PST 2026


https://github.com/hvdijk created https://github.com/llvm/llvm-project/pull/242891

This commit adds support for --encoding=l/b/L/B to allow processing strings encoded as UTF-16 or UTF-32.

>From 26d62e81ef85b125cb8fe2016c3cd47959424fcd Mon Sep 17 00:00:00 2001
From: Harald van Dijk <hdijk at accesssoftek.com>
Date: Sun, 15 Nov 2026 22:11:20 +0000
Subject: [PATCH] [llvm-strings] Add extra --encoding options

This commit adds support for --encoding=l/b/L/B to allow processing
strings encoded as UTF-16 or UTF-32.
---
 llvm/test/tools/llvm-strings/encoding.test |  42 +++++++++
 llvm/tools/llvm-strings/llvm-strings.cpp   | 105 +++++++++++++++++++--
 2 files changed, 138 insertions(+), 9 deletions(-)

diff --git a/llvm/test/tools/llvm-strings/encoding.test b/llvm/test/tools/llvm-strings/encoding.test
index a978499cc52fcd..63d281b0cdaa71 100644
--- a/llvm/test/tools/llvm-strings/encoding.test
+++ b/llvm/test/tools/llvm-strings/encoding.test
@@ -56,6 +56,48 @@ CHECK-UTF8:cdef{{\xc2\xa0}}
 CHECK-UTF8:defg
 CHECK-UTF8:efgh
 
+# Check UTF-16 and UTF-32 separately.
+# UTF-16LE:
+RUN: echo -n -e 'a\0\n\0' > %t
+RUN: echo -n -e 'a\0b\0\n\0' > %t
+RUN: echo -n -e 'a\0b\0c\0\n\0' > %t
+RUN: echo -n -e 'a\0b\0c\0d\0\n\0' > %t
+RUN: echo -n -e 'b\0c\0d\0e\0\0240\0\n\0' >> %t
+RUN: echo -n -e 'c\0d\0e\0f\0\0240\0\0237\0\n\0' >> %t
+RUN: echo -n -e '\0\0330d\0e\0f\0g\0\0337\0337\n\0' >> %t
+RUN: echo -n -e '\0\0330\0\0330e\0f\0g\0h\0\0337\0337\n\0' >> %t
+RUN: llvm-strings -e l %t | FileCheck --check-prefixes CHECK,CHECK-UTF8 --strict-whitespace --match-full-lines %s
+# UTF-16BE:
+RUN: echo -n -e '\0a\0\n' > %t
+RUN: echo -n -e '\0a\0b\0\n' > %t
+RUN: echo -n -e '\0a\0b\0c\0\n' > %t
+RUN: echo -n -e '\0a\0b\0c\0d\0\n' > %t
+RUN: echo -n -e '\0b\0c\0d\0e\0\0240\0\n' >> %t
+RUN: echo -n -e '\0c\0d\0e\0f\0\0240\0\0237\0\n' >> %t
+RUN: echo -n -e '\0330\0\0d\0e\0f\0g\0337\0337\0\n' >> %t
+RUN: echo -n -e '\0330\0\0330\0\0e\0f\0g\0h\0337\0337\0\n' >> %t
+RUN: llvm-strings -e b %t | FileCheck --check-prefixes CHECK,CHECK-UTF8 --strict-whitespace --match-full-lines %s
+# UTF-32LE:
+RUN: echo -n -e 'a\0\0\0\n\0\0\0' > %t
+RUN: echo -n -e 'a\0\0\0b\0\0\0\n\0\0\0' > %t
+RUN: echo -n -e 'a\0\0\0b\0\0\0c\0\0\0\n\0\0\0' > %t
+RUN: echo -n -e 'a\0\0\0b\0\0\0c\0\0\0d\0\0\0\n\0\0\0' > %t
+RUN: echo -n -e 'b\0\0\0c\0\0\0d\0\0\0e\0\0\0\0240\0\0\0\n\0\0\0' >> %t
+RUN: echo -n -e 'c\0\0\0d\0\0\0e\0\0\0f\0\0\0\0240\0\0\0\0237\0\0\0\n\0\0\0' >> %t
+RUN: echo -n -e '\0\0330\0\0d\0\0\0e\0\0\0f\0\0\0g\0\0\0\0337\0337\0\0\n\0\0\0' >> %t
+RUN: echo -n -e '\0\0330\0\0\0\0337\0\0e\0\0\0f\0\0\0g\0\0\0h\0\0\0\0377\0377\0377\0377\n\0\0\0' >> %t
+RUN: llvm-strings -e L %t | FileCheck --check-prefixes CHECK,CHECK-UTF8 --strict-whitespace --match-full-lines %s
+# UTF-32BE:
+RUN: echo -n -e '\0\0\0a\0\0\0\n' > %t
+RUN: echo -n -e '\0\0\0a\0\0\0b\0\0\0\n' > %t
+RUN: echo -n -e '\0\0\0a\0\0\0b\0\0\0c\0\0\0\n' > %t
+RUN: echo -n -e '\0\0\0a\0\0\0b\0\0\0c\0\0\0d\0\0\0\n' > %t
+RUN: echo -n -e '\0\0\0b\0\0\0c\0\0\0d\0\0\0e\0\0\0\0240\0\0\0\n' >> %t
+RUN: echo -n -e '\0\0\0c\0\0\0d\0\0\0e\0\0\0f\0\0\0\0240\0\0\0\0237\0\0\0\n' >> %t
+RUN: echo -n -e '\0\0\0330\0\0\0\0d\0\0\0e\0\0\0f\0\0\0g\0\0\0337\0337\0\0\0\n' >> %t
+RUN: echo -n -e '\0\0\0330\0\0\0\0337\0\0\0\0e\0\0\0f\0\0\0g\0\0\0h\0377\0377\0377\0377\0\0\0\n' >> %t
+RUN: llvm-strings -e B %t | FileCheck --check-prefixes CHECK,CHECK-UTF8 --strict-whitespace --match-full-lines %s
+
 # Check that invalid --encoding values are rejected.
 RUN: not llvm-strings --encoding x 2>&1 %t | FileCheck --check-prefix CHECK-ERROR %s
 RUN: not llvm-strings -e x 2>&1 %t | FileCheck --check-prefix CHECK-ERROR %s
diff --git a/llvm/tools/llvm-strings/llvm-strings.cpp b/llvm/tools/llvm-strings/llvm-strings.cpp
index 1265555cb80858..b655be3c837f4b 100644
--- a/llvm/tools/llvm-strings/llvm-strings.cpp
+++ b/llvm/tools/llvm-strings/llvm-strings.cpp
@@ -138,7 +138,7 @@ static constexpr int DefaultMinLength = 4;
 static int MinLength = DefaultMinLength;
 static bool PrintFileName;
 
-enum class Encoding { Ascii, Locale, Utf8 };
+enum class Encoding { Ascii, Locale, Utf8, Utf16LE, Utf16BE, Utf32LE, Utf32BE };
 static Encoding Encoding;
 
 enum class Radix { None, Octal, Hexadecimal, Decimal };
@@ -536,8 +536,8 @@ struct CharSet {
     }
   };
 
-  // UTF-8 is stateless but with -eutf8 -Ulocale, we may convert it to a
-  // stateful encoding.
+  // UTF encodings are stateless but with e.g. -eutf8 -Ulocale, we may convert
+  // to a stateful encoding.
   struct Utf8 : MultiByteStateful {
     static bool readChar(const char *&Cur, const char *End, bool AtEOF,
                          CharT &Ch) {
@@ -577,15 +577,90 @@ struct CharSet {
         return;
       }
 
-      const UTF8 *Cur = reinterpret_cast<const UTF8 *>(Str.begin()),
-                 *End = reinterpret_cast<const UTF8 *>(Str.end());
-      while (Cur != End) {
-        UTF32 UCh;
-        UTF32 *UTF32Next = &UCh;
-        ConvertUTF8toUTF32(&Cur, End, &UTF32Next, &UCh + 1, strictConversion);
+      const char *Cur = Str.begin(), *End = Str.end();
+      CharT UCh;
+      while (readChar(Cur, End, /*AtEOF=*/true, UCh))
+        writeChar(OS, UCh);
+    }
+  };
+
+  struct Utf16 : MultiByteStateful {
+    endianness Endianness = getEndianness();
+    static endianness getEndianness() {
+      switch (Encoding) {
+      case Encoding::Utf16LE:
+        return endianness::little;
+      case Encoding::Utf16BE:
+        return endianness::big;
+      default:
+        llvm_unreachable("Utf16 is only used with --encoding=l/b");
+      }
+    }
+
+    bool readChar(const char *&Cur, const char *End, bool AtEOF, CharT &Ch) {
+      if (size_t(End - Cur) < sizeof(UTF16))
+        return false;
+
+      UTF16 UTF16Buf[2];
+      const UTF16 *UTF16Cur = UTF16Buf;
+      UTF16 *UTF16End = UTF16Buf;
+      *UTF16End++ = support::endian::read<UTF16>(Cur, Endianness);
+      if (size_t(End - Cur) >= 2 * sizeof(UTF16))
+        *UTF16End++ =
+            support::endian::read<UTF16>(Cur + sizeof(UTF16), Endianness);
+      UTF32 *UTF32Next = &Ch;
+      const auto Res = ConvertUTF16toUTF32(&UTF16Cur, UTF16End, &UTF32Next,
+                                           &Ch + 1, strictConversion);
+      if (UTF16Cur != UTF16Buf) {
+        assert(UTF32Next != &Ch);
+        Cur += (UTF16Cur - UTF16Buf) * sizeof(UTF16);
+        return true;
+      } else if (Res == sourceExhausted && !AtEOF) {
+        return false;
+      } else {
+        assert(UTF32Next == &Ch);
+        Cur += sizeof(UTF16);
+        Ch = 0;
+        return true;
+      }
+    }
+
+    void writeString(raw_ostream &OS, StringRef Str) {
+      const char *Cur = Str.begin(), *End = Str.end();
+      CharT UCh;
+      while (readChar(Cur, End, /*AtEOF=*/true, UCh))
         writeChar(OS, UCh);
+    }
+  };
+
+  struct Utf32 : MultiByteStateful {
+    endianness Endianness = getEndianness();
+    static endianness getEndianness() {
+      switch (Encoding) {
+      case Encoding::Utf32LE:
+        return endianness::little;
+      case Encoding::Utf32BE:
+        return endianness::big;
+      default:
+        llvm_unreachable("Utf32 is only used with --encoding=L/B");
       }
     }
+
+    bool readChar(const char *&Cur, const char *End, bool AtEOF, CharT &Ch) {
+      if (size_t(End - Cur) < sizeof(UTF32))
+        return false;
+
+      Ch = std::min<UTF32>(support::endian::readNext<UTF32>(Cur, Endianness),
+                           0x10FFFF);
+      return true;
+    }
+
+    void writeString(raw_ostream &OS, StringRef Str) {
+      const char *Cur = Str.begin(), *End = Str.end();
+      CharT UCh;
+      while (readChar(Cur, End, /*AtEOF=*/true, UCh))
+        writeChar(OS, UCh);
+    }
   };
 
   struct SingleByte : Base<unsigned char> {
@@ -867,6 +942,10 @@ int main(int argc, char **argv) {
                            .Case("s", Encoding::Ascii)
                            .Case("S", Encoding::Locale)
                            .Case("utf8", Encoding::Utf8)
+                           .Case("l", Encoding::Utf16LE)
+                           .Case("b", Encoding::Utf16BE)
+                           .Case("L", Encoding::Utf32LE)
+                           .Case("B", Encoding::Utf32BE)
                            .Default(std::nullopt);
     if (!EncodingVal)
       invalidArgValue(EncodingArg);
@@ -961,6 +1040,14 @@ int main(int argc, char **argv) {
         return strings<CharSet::Ascii>;
       else
         return strings<CharSet::Utf8>;
+
+    case Encoding::Utf16LE:
+    case Encoding::Utf16BE:
+      return strings<CharSet::Utf16>;
+
+    case Encoding::Utf32LE:
+    case Encoding::Utf32BE:
+      return strings<CharSet::Utf32>;
     }
 
     llvm_unreachable("unhandled encoding");



More information about the llvm-branch-commits mailing list