[clang] cb3f8d5 - [Lexer] Speedup LexTokenInternal

[via cfe-commits]

Author: serge-sans-paille
Date: 2022-11-16T15:57:32+01:00
New Revision: cb3f8d53e6c35e6538ccbb54fdd848de2b3d0d1e

URL: https://github.com/llvm/llvm-project/commit/cb3f8d53e6c35e6538ccbb54fdd848de2b3d0d1e
DIFF: https://github.com/llvm/llvm-project/commit/cb3f8d53e6c35e6538ccbb54fdd848de2b3d0d1e.diff

LOG: [Lexer] Speedup LexTokenInternal

Only reset "NeedsCleaning" flag in case of re-entrant call.
Do not needlessly blank IdentifierInfo. This information will be set
once the token type is picked.

This yields a nice 1% speedup when pre-processing sqlite amalgamation
through:

valgrind --tool=callgrind ./bin/clang -E sqlite3.c -o/dev/null

Differential Revision: https://reviews.llvm.org/D137960

Added: 
    

Modified: 
    clang/include/clang/Lex/Token.h
    clang/lib/Lex/Lexer.cpp

Removed: 
    


################################################################################
diff  --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index 7115d68f0f269..f0c0794096778 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -175,6 +175,8 @@ class Token {
     Loc = SourceLocation().getRawEncoding();
   }
 
+  bool hasPtrData() const { return PtrData != nullptr; }
+
   IdentifierInfo *getIdentifierInfo() const {
     assert(isNot(tok::raw_identifier) &&
            "getIdentifierInfo() on a tok::raw_identifier token!");

diff  --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index a0a0802da1736..b6ffb85cd2fa6 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -3516,10 +3516,9 @@ bool Lexer::Lex(Token &Result) {
 /// token, not a normal token, as such, it is an internal interface.  It assumes
 /// that the Flags of result have been cleared before calling this.
 bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
-LexNextToken:
-  // New token, can't need cleaning yet.
-  Result.clearFlag(Token::NeedsCleaning);
-  Result.setIdentifierInfo(nullptr);
+LexStart:
+  assert(!Result.needsCleaning() && "Result doesn't need cleaning");
+  assert(!Result.hasPtrData() && "Result has been reset");
 
   // CurPtr - Cache BufferPtr in an automatic variable.
   const char *CurPtr = BufferPtr;
@@ -4301,6 +4300,10 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
 
   // We parsed the directive; lex a token with the new state.
   return false;
+
+LexNextToken:
+  Result.clearFlag(Token::NeedsCleaning);
+  goto LexStart;
 }
 
 const char *Lexer::convertDependencyDirectiveToken(