[clang] 9efabbb - [clang-format] Fix a bug in lexing C++ UDL ending in $ (#136476)

[via cfe-commits]

Author: Owen Pan
Date: 2025-04-22T21:08:09-07:00
New Revision: 9efabbbbe58bd8bc2141ba1c914f79376e09cbcf

URL: https://github.com/llvm/llvm-project/commit/9efabbbbe58bd8bc2141ba1c914f79376e09cbcf
DIFF: https://github.com/llvm/llvm-project/commit/9efabbbbe58bd8bc2141ba1c914f79376e09cbcf.diff

LOG: [clang-format] Fix a bug in lexing C++ UDL ending in $ (#136476)

Fix #61612

Added: 
    

Modified: 
    clang/lib/Format/FormatTokenLexer.cpp
    clang/lib/Format/FormatTokenLexer.h
    clang/unittests/Format/TokenAnnotatorTest.cpp

Removed: 
    


################################################################################
diff  --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 5c4e1f814d9b7..a4c94ac411fe0 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -128,6 +128,12 @@ void FormatTokenLexer::tryMergePreviousTokens() {
   if (Style.isCpp() && tryTransformTryUsageForC())
     return;
 
+  if ((Style.Language == FormatStyle::LK_Cpp ||
+       Style.Language == FormatStyle::LK_ObjC) &&
+      tryMergeUserDefinedLiteral()) {
+    return;
+  }
+
   if (Style.isJavaScript() || Style.isCSharp()) {
     static const tok::TokenKind NullishCoalescingOperator[] = {tok::question,
                                                                tok::question};
@@ -559,6 +565,29 @@ bool FormatTokenLexer::tryMergeGreaterGreater() {
   return true;
 }
 
+bool FormatTokenLexer::tryMergeUserDefinedLiteral() {
+  if (Tokens.size() < 2)
+    return false;
+
+  auto *First = Tokens.end() - 2;
+  auto &Suffix = First[1];
+  if (Suffix->hasWhitespaceBefore() || Suffix->TokenText != "$")
+    return false;
+
+  auto &Literal = First[0];
+  if (!Literal->Tok.isLiteral())
+    return false;
+
+  auto &Text = Literal->TokenText;
+  if (!Text.ends_with("_"))
+    return false;
+
+  Text = StringRef(Text.data(), Text.size() + 1);
+  ++Literal->ColumnWidth;
+  Tokens.erase(&Suffix);
+  return true;
+}
+
 bool FormatTokenLexer::tryMergeTokens(ArrayRef<tok::TokenKind> Kinds,
                                       TokenType NewType) {
   if (Tokens.size() < Kinds.size())

diff  --git a/clang/lib/Format/FormatTokenLexer.h b/clang/lib/Format/FormatTokenLexer.h
index 61474a3f9ada8..3f001bc69415d 100644
--- a/clang/lib/Format/FormatTokenLexer.h
+++ b/clang/lib/Format/FormatTokenLexer.h
@@ -48,6 +48,7 @@ class FormatTokenLexer {
 
   bool tryMergeLessLess();
   bool tryMergeGreaterGreater();
+  bool tryMergeUserDefinedLiteral();
   bool tryMergeNSStringLiteral();
   bool tryMergeJSPrivateIdentifier();
   bool tryMergeCSharpStringLiteral();

diff  --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 73858e87c832a..e540af85aff3a 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -3985,6 +3985,12 @@ TEST_F(TokenAnnotatorTest, IdentifierPackage) {
   EXPECT_FALSE(Tokens[0]->isObjCAccessSpecifier());
 }
 
+TEST_F(TokenAnnotatorTest, UserDefinedLiteral) {
+  auto Tokens = annotate("auto dollars = 2_$;");
+  ASSERT_EQ(Tokens.size(), 6u) << Tokens;
+  EXPECT_EQ(Tokens[3]->TokenText, "2_$");
+}
+
 } // namespace
 } // namespace format
 } // namespace clang