[cfe-commits] r60077 - /cfe/trunk/Driver/CacheTokens.cpp

[Daniel Dunbar]

Author: ddunbar
Date: Tue Nov 25 20:18:33 2008
New Revision: 60077

URL: http://llvm.org/viewvc/llvm-project?rev=60077&view=rev
Log:
Revert 60071, depends on uncommitted LLVM changes.

Modified:
    cfe/trunk/Driver/CacheTokens.cpp

Modified: cfe/trunk/Driver/CacheTokens.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Driver/CacheTokens.cpp?rev=60077&r1=60076&r2=60077&view=diff

==============================================================================
--- cfe/trunk/Driver/CacheTokens.cpp (original)
+++ cfe/trunk/Driver/CacheTokens.cpp Tue Nov 25 20:18:33 2008
@@ -28,19 +28,10 @@
 typedef llvm::DenseMap<const IdentifierInfo*,uint64_t> IDMap;
 
 static void Emit32(llvm::raw_ostream& Out, uint32_t V) {
-#if 0
   Out << (unsigned char)(V);
   Out << (unsigned char)(V >>  8);
   Out << (unsigned char)(V >> 16);
   Out << (unsigned char)(V >> 24);
-#else
-  Out << V;
-#endif
-}
-
-static void EmitOffset(llvm::raw_ostream& Out, uint64_t V) {
-  assert(((uint32_t) V) == V && "Offset exceeds 32 bits.");
-  Emit32(Out, (uint32_t) V);
 }
 
 static void Emit8(llvm::raw_ostream& Out, uint32_t V) {
@@ -84,11 +75,8 @@
   Emit32(Out, X);
 }
 
-static uint64_t EmitIdentifierTable(llvm::raw_fd_ostream& Out,
-                                    const IdentifierTable& T, const IDMap& IM) {
-
-  // Record the location within the PTH file.
-  uint64_t Off = Out.tell();
+static void EmitIdentifierTable(llvm::raw_ostream& Out,
+                                const IdentifierTable& T, const IDMap& IM) {
   
   for (IdentifierTable::const_iterator I=T.begin(), E=T.end(); I!=E; ++I) {
     const IdentifierInfo& II = I->getValue();
@@ -103,68 +91,10 @@
     unsigned len = I->getKeyLength();
     Emit32(Out, len);
     const char* buf = I->getKeyData();    
-    EmitBuf(Out, buf, buf+len);  
+    EmitBuf(Out, buf, buf+len);
   }
-  
-  return Off;
 }
 
-static uint64_t EmitFileTable(llvm::raw_fd_ostream& Out, SourceManager& SM,
-                              PCHMap& PM) {
-  
-  uint64_t off = Out.tell();
-  assert (0 && "Write out the table.");
-  return off;
-}
-
-static uint64_t LexTokens(llvm::raw_fd_ostream& Out, Lexer& L, Preprocessor& PP,
-                          uint32_t& idcount, IDMap& IM) {
-  
-  // Record the location within the token file.
-  uint64_t off = Out.tell();
-
-  Token Tok;
-  
-  do {
-    L.LexFromRawLexer(Tok);
-    
-    if (Tok.is(tok::identifier)) {
-      Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
-    }
-    else if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
-      // Special processing for #include.  Store the '#' token and lex
-      // the next token.
-      EmitToken(Out, Tok, idcount, IM);
-      L.LexFromRawLexer(Tok);
-      
-      // Did we see 'include'/'import'/'include_next'?
-      if (!Tok.is(tok::identifier))
-        continue;
-      
-      IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok);
-      Tok.setIdentifierInfo(II);
-      tok::PPKeywordKind K = II->getPPKeywordID();
-      
-      if (K == tok::pp_include || K == tok::pp_import || 
-          K == tok::pp_include_next) {
-        
-        // Save the 'include' token.
-        EmitToken(Out, Tok, idcount, IM);
-        
-        // Lex the next token as an include string.
-        L.setParsingPreprocessorDirective(true);
-        L.LexIncludeFilename(Tok); 
-        L.setParsingPreprocessorDirective(false);
-        
-        if (Tok.is(tok::identifier))
-          Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
-      }
-    }    
-  }
-  while (EmitToken(Out, Tok, idcount, IM), Tok.isNot(tok::eof));
-  
-  return off;
-}
 
 void clang::CacheTokens(Preprocessor& PP, const std::string& OutFile) {
   // Lex through the entire file.  This will populate SourceManager with
@@ -181,6 +111,7 @@
 
   PCHMap PM;
   IDMap  IM;
+  uint64_t tokIdx = 0;
   uint32_t idcount = 0;
   
   std::string ErrMsg;
@@ -195,30 +126,52 @@
        I!=E; ++I) {
     
     const SrcMgr::ContentCache* C = I.getFileIDInfo().getContentCache();
-    if (!C) continue;
-
-    const FileEntry* FE = C->Entry;    // Does this entry correspond to a file?    
-    if (!FE) continue;
 
-    PCHMap::iterator PI = PM.find(FE); // Have we already processed this file?
-    if (PI != PM.end()) continue;
+    if (!C)
+      continue;
+    
+    const FileEntry* FE = C->Entry;
+    
+    if (!FE)
+      continue;
+    
+    PCHMap::iterator PI = PM.find(FE);
+    if (PI != PM.end()) continue;    
+    PM[FE] = tokIdx;
     
-    const llvm::MemoryBuffer* B = C->Buffer;    
-    if (!B) continue;
+    // os << "Processing: " << FE->getName() << " : " << tokIdx << "\n";
     
+    const llvm::MemoryBuffer* B = C->Buffer;
+
+    if (!B)
+      continue;
+
+    // Create a raw lexer.
     Lexer L(SourceLocation::getFileLoc(I.getFileID(), 0), LOpts,
             B->getBufferStart(), B->getBufferEnd(), B);
+
+    // Ignore whitespace.
+    L.SetKeepWhitespaceMode(false);
+    L.SetCommentRetentionState(false);
     
-    PM[FE] = LexTokens(Out, L, PP, idcount, IM);
-  }
+    // Lex the file, populating our data structures.
+    Token Tok;
+    L.LexFromRawLexer(Tok);
 
-  // Write out the identifier table.
-  uint64_t IdTableOff = EmitIdentifierTable(Out, PP.getIdentifierTable(), IM);
-  
-  // Write out the file table.
-  uint64_t FileTableOff = EmitFileTable(Out, SM, PM);  
+    while (Tok.isNot(tok::eof)) {
+      ++tokIdx;
+
+      if (Tok.is(tok::identifier))
+        Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
+
+      // Write the token to disk.
+      EmitToken(Out, Tok, idcount, IM);
+
+      // Lex the next token.
+      L.LexFromRawLexer(Tok);
+    }
+  }
   
-  // Finally, write out the offset table at the end.
-  EmitOffset(Out, IdTableOff);
-  EmitOffset(Out, FileTableOff);
+  // Now, write out the identifier table.
+  EmitIdentifierTable(Out, PP.getIdentifierTable(), IM);
 }