[clang] [clang][Lex][NFC] Share character decoding between lexer slow paths (PR #242169)

Chen Miao via cfe-commits cfe-commits at lists.llvm.org
Thu Nov 12 20:55:45 PST 2026


https://github.com/ChenMiaoi created https://github.com/llvm/llvm-project/pull/242169

Factor the common logic in `getCharAndSizeSlow` and `getCharAndSizeSlowNoWarn` into a shared implementation to avoid maintaining two copies of the trigraph and escaped-newline handling logic.

I’m not sure this is the best approach, though. My thinking was that if a change to either function also requires updating the other, why not factor out the shared logic?

>From 2f887889ba6683743d0e8af890ef9cf68dd685d8 Mon Sep 17 00:00:00 2001
From: Chen Miao <chenmiao.ku at gmail.com>
Date: Fri, 13 Nov 2026 12:38:58 +0800
Subject: [PATCH] [clang][Lex][NFC] Share character decoding between lexer slow
 paths

Factor the common logic in `getCharAndSizeSlow` and
`getCharAndSizeSlowNoWarn` into a shared implementation to avoid maintaining
two copies of the trigraph and escaped-newline handling logic.
---
 clang/lib/Lex/Lexer.cpp | 90 +++++++++++++----------------------------
 1 file changed, 27 insertions(+), 63 deletions(-)

diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index b659780bac593..ac765cec0b2af 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -1465,7 +1465,7 @@ SourceLocation Lexer::findLocationAfterToken(
   return TokenLoc.getLocWithOffset(Tok->getLength() + NumWhitespaceChars);
 }
 
-/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
+/// Peek a single 'character' from the specified buffer,
 /// get its size, and return it.  This is tricky in several cases:
 ///   1. If currently at the start of a trigraph, we warn about the trigraph,
 ///      then either return the trigraph (skipping 3 chars) or the '?',
@@ -1474,13 +1474,14 @@ SourceLocation Lexer::findLocationAfterToken(
 ///      the backslash and newline), implicitly skip the newline and return
 ///      the char after it.
 ///
-/// This handles the slow/uncommon case of the getCharAndSize method.  Here we
-/// know that we can accumulate into Size, and that we have already incremented
-/// Ptr by Size bytes.
-///
-/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
-/// be updated to match.
-Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
+/// Handle the slow/uncommon case of getCharAndSize and getCharAndSizeNoWarn.
+/// If Tok is non-null, set its NeedsCleaning flag as needed and use L to emit
+/// diagnostics outside raw mode.
+static Lexer::SizedChar getCharAndSizeSlowImpl(const char *Ptr,
+                                               const LangOptions &LangOpts,
+                                               Lexer *L, Token *Tok) {
+  assert((!Tok || L) && "Lexer is required when a token is provided");
+
   unsigned Size = 0;
   // If we have a slash, look for an escaped newline.
   if (Ptr[0] == '\\') {
@@ -1493,20 +1494,21 @@ Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
 
     // See if we have optional whitespace characters between the slash and
     // newline.
-    if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
+    if (unsigned EscapedNewLineSize = Lexer::getEscapedNewLineSize(Ptr)) {
       // Remember that this token needs to be cleaned.
-      if (Tok) Tok->setFlag(Token::NeedsCleaning);
+      if (Tok)
+        Tok->setFlag(Token::NeedsCleaning);
 
       // Warn if there was whitespace between the backslash and newline.
-      if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
-        Diag(Ptr, diag::backslash_newline_space);
+      if (Tok && Ptr[0] != '\n' && Ptr[0] != '\r' && !L->isLexingRawMode())
+        L->Diag(Ptr, diag::backslash_newline_space);
 
       // Found backslash<whitespace><newline>.  Parse the char after it.
       Size += EscapedNewLineSize;
       Ptr  += EscapedNewLineSize;
 
       // Use slow version to accumulate a correct size field.
-      auto CharAndSize = getCharAndSizeSlow(Ptr, Tok);
+      auto CharAndSize = getCharAndSizeSlowImpl(Ptr, LangOpts, L, Tok);
       CharAndSize.Size += Size;
       return CharAndSize;
     }
@@ -1516,13 +1518,14 @@ Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
   }
 
   // If this is a trigraph, process it.
-  if (Ptr[0] == '?' && Ptr[1] == '?') {
+  if ((Tok || LangOpts.Trigraphs) && Ptr[0] == '?' && Ptr[1] == '?') {
     // If this is actually a legal trigraph (not something like "??x"), emit
     // a trigraph warning.  If so, and if trigraphs are enabled, return it.
-    if (char C = DecodeTrigraphChar(Ptr + 2, Tok ? this : nullptr,
+    if (char C = DecodeTrigraphChar(Ptr + 2, Tok ? L : nullptr,
                                     LangOpts.Trigraphs)) {
       // Remember that this token needs to be cleaned.
-      if (Tok) Tok->setFlag(Token::NeedsCleaning);
+      if (Tok)
+        Tok->setFlag(Token::NeedsCleaning);
 
       Ptr += 3;
       Size += 3;
@@ -1535,55 +1538,16 @@ Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
   return {*Ptr, Size + 1u};
 }
 
-/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
-/// getCharAndSizeNoWarn method.  Here we know that we can accumulate into Size,
-/// and that we have already incremented Ptr by Size bytes.
-///
-/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
-/// be updated to match.
+/// Handle the slow/uncommon case of getCharAndSize.
+Lexer::SizedChar Lexer::getCharAndSizeSlow(const char *Ptr, Token *Tok) {
+  return getCharAndSizeSlowImpl(Ptr, LangOpts, this, Tok);
+}
+
+/// Handle the slow/uncommon case of getCharAndSizeNoWarn without diagnostics
+/// or token flag updates.
 Lexer::SizedChar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr,
                                                  const LangOptions &LangOpts) {
-
-  unsigned Size = 0;
-  // If we have a slash, look for an escaped newline.
-  if (Ptr[0] == '\\') {
-    ++Size;
-    ++Ptr;
-Slash:
-    // Common case, backslash-char where the char is not whitespace.
-    if (!isWhitespace(Ptr[0]))
-      return {'\\', Size};
-
-    // See if we have optional whitespace characters followed by a newline.
-    if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
-      // Found backslash<whitespace><newline>.  Parse the char after it.
-      Size += EscapedNewLineSize;
-      Ptr  += EscapedNewLineSize;
-
-      // Use slow version to accumulate a correct size field.
-      auto CharAndSize = getCharAndSizeSlowNoWarn(Ptr, LangOpts);
-      CharAndSize.Size += Size;
-      return CharAndSize;
-    }
-
-    // Otherwise, this is not an escaped newline, just return the slash.
-    return {'\\', Size};
-  }
-
-  // If this is a trigraph, process it.
-  if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
-    // If this is actually a legal trigraph (not something like "??x"), return
-    // it.
-    if (char C = GetTrigraphCharForLetter(Ptr[2])) {
-      Ptr += 3;
-      Size += 3;
-      if (C == '\\') goto Slash;
-      return {C, Size};
-    }
-  }
-
-  // If this is neither, return a single character.
-  return {*Ptr, Size + 1u};
+  return getCharAndSizeSlowImpl(Ptr, LangOpts, nullptr, nullptr);
 }
 
 //===----------------------------------------------------------------------===//



More information about the cfe-commits mailing list