[cfe-commits] r57405 - in /cfe/trunk: Driver/clang.cpp test/Preprocessor/dumptokens_phyloc.c

[Chris Lattner]

Author: lattner
Date: Sun Oct 12 00:03:36 2008
New Revision: 57405

URL: http://llvm.org/viewvc/llvm-project?rev=57405&view=rev
Log:
Add a new -dump-raw-tokens option, which allows us to see raw tokens.  
Rename -dumptokens to -dump-tokens.


Modified:
    cfe/trunk/Driver/clang.cpp
    cfe/trunk/test/Preprocessor/dumptokens_phyloc.c

Modified: cfe/trunk/Driver/clang.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Driver/clang.cpp?rev=57405&r1=57404&r2=57405&view=diff

==============================================================================
--- cfe/trunk/Driver/clang.cpp (original)
+++ cfe/trunk/Driver/clang.cpp Sun Oct 12 00:03:36 2008
@@ -80,7 +80,8 @@
   ParseNoop,                    // Parse with noop callbacks.
   RunPreprocessorOnly,          // Just lex, no output.
   PrintPreprocessedInput,       // -E mode.
-  DumpTokens,                   // Token dump mode.
+  DumpTokens,                   // Dump out preprocessed tokens.
+  DumpRawTokens,                // Dump out raw tokens.
   RunAnalysis                   // Run one or more source code analyses. 
 };
 
@@ -92,7 +93,9 @@
                         "Just run preprocessor, no output (for timings)"),
              clEnumValN(PrintPreprocessedInput, "E",
                         "Run preprocessor, emit preprocessed file"),
-             clEnumValN(DumpTokens, "dumptokens",
+             clEnumValN(DumpRawTokens, "dump-raw-tokens",
+                        "Lex file in raw mode and dump raw tokens"),
+             clEnumValN(DumpTokens, "dump-tokens",
                         "Run preprocessor, dump internal rep of tokens"),
              clEnumValN(ParseNoop, "parse-noop",
                         "Run parser with noop callbacks (for timings)"),
@@ -1104,9 +1107,29 @@
 
     break;
       
+  case DumpRawTokens: {
+    SourceManager &SM = PP.getSourceManager();
+    std::pair<const char*,const char*> File =
+      SM.getBufferData(SM.getMainFileID());
+    // Start lexing the specified input file.
+    Lexer RawLex(SourceLocation::getFileLoc(SM.getMainFileID(), 0),
+                 PP.getLangOptions(), File.first, File.second);
+    RawLex.SetKeepWhitespaceMode(true);
+
+    Token RawTok;
+
+    RawLex.LexFromRawLexer(RawTok);
+    while (RawTok.isNot(tok::eof)) {
+      PP.DumpToken(RawTok, true);
+      fprintf(stderr, "\n");
+      RawLex.LexFromRawLexer(RawTok);
+    }
+    ClearSourceMgr = true;
+    break;
+  }
   case DumpTokens: {                 // Token dump mode.
     Token Tok;
-    // Start parsing the specified input file.
+    // Start preprocessing the specified input file.
     PP.EnterMainSourceFile();
     do {
       PP.Lex(Tok);

Modified: cfe/trunk/test/Preprocessor/dumptokens_phyloc.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Preprocessor/dumptokens_phyloc.c?rev=57405&r1=57404&r2=57405&view=diff

==============================================================================
--- cfe/trunk/test/Preprocessor/dumptokens_phyloc.c (original)
+++ cfe/trunk/test/Preprocessor/dumptokens_phyloc.c Sun Oct 12 00:03:36 2008
@@ -1,4 +1,4 @@
-// RUN: clang -dumptokens %s 2>&1 | grep "PhysLoc=[-_.a-zA-Z/\\]*:3:20"
+// RUN: clang -dump-tokens %s 2>&1 | grep "PhysLoc=[-_.a-zA-Z/\\]*:3:20"
 
 #define TESTPHYLOC 10