[cfe-commits] r62855 - in /cfe/trunk: include/clang/Basic/IdentifierTable.h lib/Lex/Lexer.cpp lib/Lex/PTHLexer.cpp lib/Lex/Preprocessor.cpp lib/Lex/TokenLexer.cpp

[Chris Lattner]

Author: lattner
Date: Fri Jan 23 12:35:48 2009
New Revision: 62855

URL: http://llvm.org/viewvc/llvm-project?rev=62855&view=rev
Log:
This is a follow-up to r62675:

Refactor how the preprocessor changes a token from being an tok::identifier to a 
keyword (e.g. tok::kw_for).  Instead of doing this in HandleIdentifier, hoist this
common case out into the caller, so that every keyword doesn't have to go through
HandleIdentifier.  This drops time in HandleIdentifier from 1.25ms to .62ms, and
speeds up clang -Eonly with PTH by about 1%.


Modified:
    cfe/trunk/include/clang/Basic/IdentifierTable.h
    cfe/trunk/lib/Lex/Lexer.cpp
    cfe/trunk/lib/Lex/PTHLexer.cpp
    cfe/trunk/lib/Lex/Preprocessor.cpp
    cfe/trunk/lib/Lex/TokenLexer.cpp

Modified: cfe/trunk/include/clang/Basic/IdentifierTable.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/IdentifierTable.h?rev=62855&r1=62854&r2=62855&view=diff

==============================================================================
--- cfe/trunk/include/clang/Basic/IdentifierTable.h (original)
+++ cfe/trunk/include/clang/Basic/IdentifierTable.h Fri Jan 23 12:35:48 2009
@@ -122,14 +122,7 @@
   /// can be used to cause the lexer to map identifiers to source-language
   /// tokens.
   tok::TokenKind getTokenID() const { return (tok::TokenKind)TokenID; }
-  void setTokenID(tok::TokenKind ID) {
-    TokenID = ID;
-  
-    if (ID != tok::identifier)
-      NeedsHandleIdentifier = 1;
-    else
-      RecomputeNeedsHandleIdentifier();
-  }
+  void setTokenID(tok::TokenKind ID) { TokenID = ID; }
   
   /// getPPKeywordID - Return the preprocessor keyword ID for this identifier.
   /// For example, "define" will return tok::pp_define.
@@ -225,7 +218,7 @@
   void RecomputeNeedsHandleIdentifier() {
     NeedsHandleIdentifier =
       (isPoisoned() | hasMacroDefinition() | isCPlusPlusOperatorKeyword() |
-       isExtensionToken()) || getTokenID() != tok::identifier;
+       isExtensionToken());
   }
 };
 

Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=62855&r1=62854&r2=62855&view=diff

==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Fri Jan 23 12:35:48 2009
@@ -558,6 +558,10 @@
     // identifier table.
     IdentifierInfo *II = PP->LookUpIdentifierInfo(Result, IdStart);
     
+    // Change the kind of this identifier to the appropriate token kind, e.g.
+    // turning "for" into a keyword.
+    Result.setKind(II->getTokenID());
+    
     // Finally, now that we know we have an identifier, pass this off to the
     // preprocessor, which may macro expand it or something.
     if (II->isHandleIdentifierCase())

Modified: cfe/trunk/lib/Lex/PTHLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PTHLexer.cpp?rev=62855&r1=62854&r2=62855&view=diff

==============================================================================
--- cfe/trunk/lib/Lex/PTHLexer.cpp (original)
+++ cfe/trunk/lib/Lex/PTHLexer.cpp Fri Jan 23 12:35:48 2009
@@ -101,7 +101,13 @@
   if (IdentifierID) {
     MIOpt.ReadToken();
     IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
+    
     Tok.setIdentifierInfo(II);
+    
+    // Change the kind of this identifier to the appropriate token kind, e.g.
+    // turning "for" into a keyword.
+    Tok.setKind(II->getTokenID());
+    
     if (II->isHandleIdentifierCase())
       PP->HandleIdentifier(Tok);
     return;

Modified: cfe/trunk/lib/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=62855&r1=62854&r2=62855&view=diff

==============================================================================
--- cfe/trunk/lib/Lex/Preprocessor.cpp (original)
+++ cfe/trunk/lib/Lex/Preprocessor.cpp Fri Jan 23 12:35:48 2009
@@ -759,10 +759,6 @@
   if (II.isCPlusPlusOperatorKeyword())
     Identifier.setIdentifierInfo(0);
 
-  // Change the kind of this identifier to the appropriate token kind, e.g.
-  // turning "for" into a keyword.
-  Identifier.setKind(II.getTokenID());
-    
   // If this is an extension token, diagnose its use.
   // We avoid diagnosing tokens that originate from macro definitions.
   if (II.isExtensionToken() && Features.C99 && !DisableMacroExpansion)

Modified: cfe/trunk/lib/Lex/TokenLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/TokenLexer.cpp?rev=62855&r1=62854&r2=62855&view=diff

==============================================================================
--- cfe/trunk/lib/Lex/TokenLexer.cpp (original)
+++ cfe/trunk/lib/Lex/TokenLexer.cpp Fri Jan 23 12:35:48 2009
@@ -326,9 +326,14 @@
   }
   
   // Handle recursive expansion!
-  if (Tok.getIdentifierInfo() && !DisableMacroExpansion &&
-      Tok.getIdentifierInfo()->isHandleIdentifierCase())
-    PP.HandleIdentifier(Tok);
+  if (IdentifierInfo *II = Tok.getIdentifierInfo()) {
+    // Change the kind of this identifier to the appropriate token kind, e.g.
+    // turning "for" into a keyword.
+    Tok.setKind(II->getTokenID());
+    
+    if (!DisableMacroExpansion && II->isHandleIdentifierCase())
+      PP.HandleIdentifier(Tok);
+  }
 
   // Otherwise, return a normal token.
 }