[llvm] [llvm-strings] Add --encoding={s,S,utf8} option (PR #221794)

[Harald van Dijk via llvm-commits]

================
@@ -112,31 +243,42 @@ static void strings(raw_ostream &OS, StringRef FileName,
   // of a chunk.
   const size_t Min = MinLength;
   SmallString<DefaultMinLength> Candidate;
+  size_t CandidateLength = 0;
   bool InString = false;
+  std::mbstate_t MBState{};
   // Offset of the start of the current chunk within the file.
   size_t ChunkOffset = 0;
 
-  Buffer.resize_for_overwrite(sys::fs::DefaultReadChunkSize);
-
   while (true) {
+    // Size of any partial character left over from the previous chunk.
+    size_t PartialCharSize = Buffer.size();
+    Buffer.resize_for_overwrite(PartialCharSize +
+                                sys::fs::DefaultReadChunkSize);
+
     Expected<size_t> ReadBytesOrErr = sys::fs::readNativeFile(
-        Handle, MutableArrayRef(Buffer.data(), Buffer.size()));
+        Handle,
+        MutableArrayRef(Buffer.begin() + PartialCharSize, Buffer.end()));
     if (!ReadBytesOrErr) {
       errs() << FileName << ": "
              << errorToErrorCode(ReadBytesOrErr.takeError()).message() << '\n';
       return;
     }
-    size_t ChunkSize = *ReadBytesOrErr;
+    const bool AtEOF = *ReadBytesOrErr == 0;
+    const size_t ChunkSize = PartialCharSize + *ReadBytesOrErr;
     if (ChunkSize == 0)
       break;
 
+    Buffer.resize_for_overwrite(ChunkSize);
+
     // To prevent performance regression under O0, access the raw pointer
     // instead of using methods provided by the standard library, which are not
     // inlined under O0.
-    const char *const Begin = Buffer.data();
-    const char *const End = Begin + ChunkSize;
+    const char *Begin = Buffer.data();
+    const char *End = Begin + ChunkSize;
     const char *Cur = Begin;
 
+    UTF32 Ch;
----------------
hvdijk wrote:

LLVM's API is somewhat confusingly designed, but as far as I can see there is no issue here, the implicit conversions from `unsigned int` to `int` to `uint32_t` (the latter conversion inside `isPrintable`) do not result in any changes of values that represent valid code points, and do not result in any invalid code points being misinterpreted as valid.

https://github.com/llvm/llvm-project/pull/221794