[cfe-commits] r57405 - in /cfe/trunk: Driver/clang.cpp test/Preprocessor/dumptokens_phyloc.c
Chris Lattner
sabre at nondot.org
Sat Oct 11 22:03:38 PDT 2008
Author: lattner
Date: Sun Oct 12 00:03:36 2008
New Revision: 57405
URL: http://llvm.org/viewvc/llvm-project?rev=57405&view=rev
Log:
Add a new -dump-raw-tokens option, which allows us to see raw tokens.
Rename -dumptokens to -dump-tokens.
Modified:
cfe/trunk/Driver/clang.cpp
cfe/trunk/test/Preprocessor/dumptokens_phyloc.c
Modified: cfe/trunk/Driver/clang.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Driver/clang.cpp?rev=57405&r1=57404&r2=57405&view=diff
==============================================================================
--- cfe/trunk/Driver/clang.cpp (original)
+++ cfe/trunk/Driver/clang.cpp Sun Oct 12 00:03:36 2008
@@ -80,7 +80,8 @@
ParseNoop, // Parse with noop callbacks.
RunPreprocessorOnly, // Just lex, no output.
PrintPreprocessedInput, // -E mode.
- DumpTokens, // Token dump mode.
+ DumpTokens, // Dump out preprocessed tokens.
+ DumpRawTokens, // Dump out raw tokens.
RunAnalysis // Run one or more source code analyses.
};
@@ -92,7 +93,9 @@
"Just run preprocessor, no output (for timings)"),
clEnumValN(PrintPreprocessedInput, "E",
"Run preprocessor, emit preprocessed file"),
- clEnumValN(DumpTokens, "dumptokens",
+ clEnumValN(DumpRawTokens, "dump-raw-tokens",
+ "Lex file in raw mode and dump raw tokens"),
+ clEnumValN(DumpTokens, "dump-tokens",
"Run preprocessor, dump internal rep of tokens"),
clEnumValN(ParseNoop, "parse-noop",
"Run parser with noop callbacks (for timings)"),
@@ -1104,9 +1107,29 @@
break;
+ case DumpRawTokens: {
+ SourceManager &SM = PP.getSourceManager();
+ std::pair<const char*,const char*> File =
+ SM.getBufferData(SM.getMainFileID());
+ // Start lexing the specified input file.
+ Lexer RawLex(SourceLocation::getFileLoc(SM.getMainFileID(), 0),
+ PP.getLangOptions(), File.first, File.second);
+ RawLex.SetKeepWhitespaceMode(true);
+
+ Token RawTok;
+
+ RawLex.LexFromRawLexer(RawTok);
+ while (RawTok.isNot(tok::eof)) {
+ PP.DumpToken(RawTok, true);
+ fprintf(stderr, "\n");
+ RawLex.LexFromRawLexer(RawTok);
+ }
+ ClearSourceMgr = true;
+ break;
+ }
case DumpTokens: { // Token dump mode.
Token Tok;
- // Start parsing the specified input file.
+ // Start preprocessing the specified input file.
PP.EnterMainSourceFile();
do {
PP.Lex(Tok);
Modified: cfe/trunk/test/Preprocessor/dumptokens_phyloc.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Preprocessor/dumptokens_phyloc.c?rev=57405&r1=57404&r2=57405&view=diff
==============================================================================
--- cfe/trunk/test/Preprocessor/dumptokens_phyloc.c (original)
+++ cfe/trunk/test/Preprocessor/dumptokens_phyloc.c Sun Oct 12 00:03:36 2008
@@ -1,4 +1,4 @@
-// RUN: clang -dumptokens %s 2>&1 | grep "PhysLoc=[-_.a-zA-Z/\\]*:3:20"
+// RUN: clang -dump-tokens %s 2>&1 | grep "PhysLoc=[-_.a-zA-Z/\\]*:3:20"
#define TESTPHYLOC 10
More information about the cfe-commits
mailing list