[cfe-commits] r62414 - in /cfe/trunk: Driver/CacheTokens.cpp Driver/clang.cpp include/clang/Lex/Lexer.h lib/Lex/Lexer.cpp lib/Rewrite/HTMLRewrite.cpp lib/Rewrite/TokenRewriter.cpp

[Chris Lattner]

Author: lattner
Date: Sat Jan 17 01:35:14 2009
New Revision: 62414

URL: http://llvm.org/viewvc/llvm-project?rev=62414&view=rev
Log:
add a simplified lexer ctor that sets up the lexer to raw-lex an
entire file.

Modified:
    cfe/trunk/Driver/CacheTokens.cpp
    cfe/trunk/Driver/clang.cpp
    cfe/trunk/include/clang/Lex/Lexer.h
    cfe/trunk/lib/Lex/Lexer.cpp
    cfe/trunk/lib/Rewrite/HTMLRewrite.cpp
    cfe/trunk/lib/Rewrite/TokenRewriter.cpp

Modified: cfe/trunk/Driver/CacheTokens.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Driver/CacheTokens.cpp?rev=62414&r1=62413&r2=62414&view=diff

==============================================================================
--- cfe/trunk/Driver/CacheTokens.cpp (original)
+++ cfe/trunk/Driver/CacheTokens.cpp Sat Jan 17 01:35:14 2009
@@ -486,8 +486,7 @@
     if (!B) continue;
 
     FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
-    Lexer L(SM.getLocForStartOfFile(FID), LOpts,
-            B->getBufferStart(), B->getBufferEnd(), B);
+    Lexer L(FID, SM, LOpts);
     PM[FE] = LexTokens(L);
   }
 

Modified: cfe/trunk/Driver/clang.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Driver/clang.cpp?rev=62414&r1=62413&r2=62414&view=diff

==============================================================================
--- cfe/trunk/Driver/clang.cpp (original)
+++ cfe/trunk/Driver/clang.cpp Sat Jan 17 01:35:14 2009
@@ -1332,15 +1332,11 @@
       
   case DumpRawTokens: {
     SourceManager &SM = PP.getSourceManager();
-    std::pair<const char*,const char*> File =
-      SM.getBufferData(SM.getMainFileID());
     // Start lexing the specified input file.
-    Lexer RawLex(SM.getLocForStartOfFile(SM.getMainFileID()),
-                 PP.getLangOptions(), File.first, File.second);
+    Lexer RawLex(SM.getMainFileID(), SM, PP.getLangOptions());
     RawLex.SetKeepWhitespaceMode(true);
 
     Token RawTok;
-
     RawLex.LexFromRawLexer(RawTok);
     while (RawTok.isNot(tok::eof)) {
       PP.DumpToken(RawTok, true);

Modified: cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Lexer.h?rev=62414&r1=62413&r2=62414&view=diff

==============================================================================
--- cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/trunk/include/clang/Lex/Lexer.h Sat Jan 17 01:35:14 2009
@@ -87,6 +87,11 @@
         const char *BufStart, const char *BufEnd,
         const llvm::MemoryBuffer *FromFile = 0);
   
+  /// Lexer constructor - Create a new raw lexer object.  This object is only
+  /// suitable for calls to 'LexRawToken'.  This lexer assumes that the text
+  /// range will outlive it, so it doesn't take ownership of it.
+  Lexer(FileID FID, const SourceManager &SM, const LangOptions &Features);
+  
   /// getFeatures - Return the language features currently enabled.  NOTE: this
   /// lexer modifies features as a file is parsed!
   const LangOptions &getFeatures() const { return Features; }
@@ -166,6 +171,7 @@
     ExtendedTokenMode = Mode ? 1 : 0;
   }
   
+  const char *getBufferStart() const { return BufferStart; }
   
   /// ReadToEndOfLine - Read the rest of the current preprocessor line as an
   /// uninterpreted string.  This switches the lexer out of directive mode.

Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=62414&r1=62413&r2=62414&view=diff

==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Sat Jan 17 01:35:14 2009
@@ -127,7 +127,6 @@
              const char *BufPtr, const char *BufEnd,
              const llvm::MemoryBuffer *FromFile)
   : FileLoc(fileloc), Features(features) {
-      
 
   // If a MemoryBuffer was specified, use its start as BufferStart. This affects
   // the source location objects produced by this lexer.
@@ -140,6 +139,20 @@
   LexingRawMode = true;
 }
 
+/// Lexer constructor - Create a new raw lexer object.  This object is only
+/// suitable for calls to 'LexRawToken'.  This lexer assumes that the text
+/// range will outlive it, so it doesn't take ownership of it.
+Lexer::Lexer(FileID FID, const SourceManager &SM, const LangOptions &features)
+  : FileLoc(SM.getLocForStartOfFile(FID)), Features(features) {
+  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+
+  InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 
+            FromFile->getBufferEnd());
+  
+  // We *are* in raw mode.
+  LexingRawMode = true;
+}
+
 
 /// Stringify - Convert the specified string into a C string, with surrounding
 /// ""'s, and with escaped \ and " characters.

Modified: cfe/trunk/lib/Rewrite/HTMLRewrite.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Rewrite/HTMLRewrite.cpp?rev=62414&r1=62413&r2=62414&view=diff

==============================================================================
--- cfe/trunk/lib/Rewrite/HTMLRewrite.cpp (original)
+++ cfe/trunk/lib/Rewrite/HTMLRewrite.cpp Sat Jan 17 01:35:14 2009
@@ -344,11 +344,8 @@
   RewriteBuffer &RB = R.getEditBuffer(FID);
 
   const SourceManager &SourceMgr = PP.getSourceManager();
-  std::pair<const char*, const char*> File = SourceMgr.getBufferData(FID);
-  const char *BufferStart = File.first;
-  
-  Lexer L(SourceMgr.getLocForStartOfFile(FID),
-          PP.getLangOptions(), File.first, File.second);
+  Lexer L(FID, SourceMgr, PP.getLangOptions());
+  const char *BufferStart = L.getBufferStart();
   
   // Inform the preprocessor that we want to retain comments as tokens, so we 
   // can highlight them.

Modified: cfe/trunk/lib/Rewrite/TokenRewriter.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Rewrite/TokenRewriter.cpp?rev=62414&r1=62413&r2=62414&view=diff

==============================================================================
--- cfe/trunk/lib/Rewrite/TokenRewriter.cpp (original)
+++ cfe/trunk/lib/Rewrite/TokenRewriter.cpp Sat Jan 17 01:35:14 2009
@@ -22,11 +22,8 @@
                              const LangOptions &LangOpts) {
   ScratchBuf.reset(new ScratchBuffer(SM));
   
-  std::pair<const char*,const char*> File = SM.getBufferData(FID);
-  
   // Create a lexer to lex all the tokens of the main file in raw mode.
-  Lexer RawLex(SM.getLocForStartOfFile(FID),
-               LangOpts, File.first, File.second);
+  Lexer RawLex(FID, SM, LangOpts);
   
   // Return all comments and whitespace as tokens.
   RawLex.SetKeepWhitespaceMode(true);