[PATCH] D137960: [Lexer] Speedup LexTokenInternal

[serge via Phabricator via cfe-commits]

serge-sans-paille updated this revision to Diff 475699.
serge-sans-paille added a comment.

Add extra assert to ensure we start lexing in a clean state


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D137960/new/

https://reviews.llvm.org/D137960

Files:
  clang/include/clang/Lex/Token.h
  clang/lib/Lex/Lexer.cpp


Index: clang/lib/Lex/Lexer.cpp
===================================================================
--- clang/lib/Lex/Lexer.cpp
+++ clang/lib/Lex/Lexer.cpp
@@ -3516,10 +3516,9 @@
 /// token, not a normal token, as such, it is an internal interface.  It assumes
 /// that the Flags of result have been cleared before calling this.
 bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
-LexNextToken:
-  // New token, can't need cleaning yet.
-  Result.clearFlag(Token::NeedsCleaning);
-  Result.setIdentifierInfo(nullptr);
+LexStart:
+  assert(!Result.needsCleaning() && "Result doesn't need cleaning");
+  assert(!Result.hasPtrData() && "Result has been reset");
 
   // CurPtr - Cache BufferPtr in an automatic variable.
   const char *CurPtr = BufferPtr;
@@ -4141,7 +4140,7 @@
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else if (Char == '|') {
       // If this is '|||||||' and we're in a conflict marker, ignore it.
-      if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1))
+      if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr - 1))
         goto LexNextToken;
       Kind = tok::pipepipe;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
@@ -4170,7 +4169,7 @@
     Char = getCharAndSize(CurPtr, SizeTmp);
     if (Char == '=') {
       // If this is '====' and we're in a conflict marker, ignore it.
-      if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1))
+      if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr - 1))
         goto LexNextToken;
 
       Kind = tok::equalequal;
@@ -4301,6 +4300,10 @@
 
   // We parsed the directive; lex a token with the new state.
   return false;
+
+LexNextToken:
+  Result.clearFlag(Token::NeedsCleaning);
+  goto LexStart;
 }
 
 const char *Lexer::convertDependencyDirectiveToken(
Index: clang/include/clang/Lex/Token.h
===================================================================
--- clang/include/clang/Lex/Token.h
+++ clang/include/clang/Lex/Token.h
@@ -175,6 +175,8 @@
     Loc = SourceLocation().getRawEncoding();
   }
 
+  bool hasPtrData() const { return PtrData != nullptr; }
+
   IdentifierInfo *getIdentifierInfo() const {
     assert(isNot(tok::raw_identifier) &&
            "getIdentifierInfo() on a tok::raw_identifier token!");


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D137960.475699.patch
Type: text/x-patch
Size: 2281 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20221116/310b3ab5/attachment.bin>