[llvm] 1c55313 - Treat `''` and `""` in Gnu-tokenized response files as an empty string argument. (#187566)

[via llvm-commits]

Author: Tony Allevato
Date: 2026-04-01T09:57:15-04:00
New Revision: 1c55313f344ff2189500f87c89cec2a3cf2eb5c9

URL: https://github.com/llvm/llvm-project/commit/1c55313f344ff2189500f87c89cec2a3cf2eb5c9
DIFF: https://github.com/llvm/llvm-project/commit/1c55313f344ff2189500f87c89cec2a3cf2eb5c9.diff

LOG: Treat `''` and `""` in Gnu-tokenized response files as an empty string argument. (#187566)

This matches the behavior of gcc and also fixes an inconsistency with
the way the same arguments would be parsed by the shell. For example,
this command line passed directly on the shell:

`command -option1 '' -option2`

has three arguments: `"-option1"`, `""`, and `"-option2"`. However, if
these are passed in a Gnu-tokenized response file today:

```
-option1
''
-option2
```

The `''` is discarded and the command incorrectly has two arguments:
`"-option1"` and `"-option2"`.

Added: 
    

Modified: 
    llvm/lib/Support/CommandLine.cpp
    llvm/unittests/Support/CommandLineTest.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index 98d80c62466c0..0c244b31724f1 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -839,9 +839,10 @@ void cl::TokenizeGNUCommandLine(StringRef Src, StringSaver &Saver,
                                 SmallVectorImpl<const char *> &NewArgv,
                                 bool MarkEOLs) {
   SmallString<128> Token;
+  bool InToken = false;
   for (size_t I = 0, E = Src.size(); I != E; ++I) {
     // Consume runs of whitespace.
-    if (Token.empty()) {
+    if (!InToken) {
       while (I != E && isWhitespace(Src[I])) {
         // Mark the end of lines in response files.
         if (MarkEOLs && Src[I] == '\n')
@@ -850,6 +851,7 @@ void cl::TokenizeGNUCommandLine(StringRef Src, StringSaver &Saver,
       }
       if (I == E)
         break;
+      InToken = true;
     }
 
     char C = Src[I];
@@ -878,12 +880,12 @@ void cl::TokenizeGNUCommandLine(StringRef Src, StringSaver &Saver,
 
     // End the token if this is whitespace.
     if (isWhitespace(C)) {
-      if (!Token.empty())
-        NewArgv.push_back(Saver.save(Token.str()).data());
+      NewArgv.push_back(Saver.save(Token.str()).data());
       // Mark the end of lines in response files.
       if (MarkEOLs && C == '\n')
         NewArgv.push_back(nullptr);
       Token.clear();
+      InToken = false;
       continue;
     }
 
@@ -892,7 +894,7 @@ void cl::TokenizeGNUCommandLine(StringRef Src, StringSaver &Saver,
   }
 
   // Append the last token after hitting EOF with no whitespace.
-  if (!Token.empty())
+  if (InToken)
     NewArgv.push_back(Saver.save(Token.str()).data());
 }
 

diff  --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp
index 9e8a165fe136f..fca2d298c460e 100644
--- a/llvm/unittests/Support/CommandLineTest.cpp
+++ b/llvm/unittests/Support/CommandLineTest.cpp
@@ -224,6 +224,71 @@ TEST(CommandLineTest, TokenizeGNUCommandLine) {
   testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input, Output);
 }
 
+TEST(CommandLineTest, TokenizeGNUCommandLineEmptyQuotes) {
+  // Explicit '' and "" should be treated as an empty string argument, as shells
+  // and gcc do.
+  const char Input1[] = R"(a b c "" d)";
+  const char *const Output1[] = {"a", "b", "c", "", "d"};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input1, Output1);
+
+  const char Input2[] = R"(a b c '' d)";
+  const char *const Output2[] = {"a", "b", "c", "", "d"};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input2, Output2);
+
+  // Check that empty arguments are preserved at the beginning/end of the
+  // input.
+  const char Input3[] = R"('' a b c d "")";
+  const char *const Output3[] = {"", "a", "b", "c", "d", ""};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input3, Output3);
+
+  // Check that an input containing only empty arguments is handled
+  // correctly.
+  const char Input4[] = R"("" '')";
+  const char *const Output4[] = {"", ""};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input4, Output4);
+
+  // Each sequence of juxtaposed empty string segments is still just one
+  // empty string.
+  const char Input5[] = R"('''' """" ''"")";
+  const char *const Output5[] = {"", "", ""};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input5, Output5);
+}
+
+TEST(CommandLineTest, TokenizeGNUCommandLineWhitespace) {
+  // Leading/trailing whitespace should be ignored.
+  const char Input1[] = R"(  a b c '' d  )";
+  const char *const Output1[] = {"a", "b", "c", "", "d"};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input1, Output1);
+}
+
+TEST(CommandLineTest, TokenizeGNUCommandLineJuxtaposedQuotedSegments) {
+  const char Input1[] = R"(a""a ""b c"" d''d ''e f'')";
+  const char *const Output1[] = {"aa", "b", "c", "dd", "e", "f"};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input1, Output1);
+
+  const char Input2[] = R"("'a'"'"b"')";
+  const char *const Output2[] = {R"('a'"b")"};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input2, Output2);
+}
+
+TEST(CommandLineTest, TokenizeGNUCommandLineUnterminatedQuotes) {
+  // Unterminated quotes are implicitly terminated at EOF.
+  const char Input1[] = R"(a b ')";
+  const char *const Output1[] = {"a", "b", ""};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input1, Output1);
+
+  const char Input2[] = R"(a b 'c d)";
+  const char *const Output2[] = {"a", "b", "c d"};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input2, Output2);
+}
+
+TEST(CommandLineTest, TokenizeGNUCommandLineNewlines) {
+  // Newlines are also treated literally inside quotes.
+  const char Input1[] = "a 'b\nc' d";
+  const char *const Output1[] = {"a", "b\nc", "d"};
+  testCommandLineTokenizer(cl::TokenizeGNUCommandLine, Input1, Output1);
+}
+
 TEST(CommandLineTest, TokenizeWindowsCommandLine1) {
   const char Input[] =
       R"(a\b c\\d e\\"f g" h\"i j\\\"k "lmn" o pqr "st \"u" \v)";