[cfe-commits] r57393 - in /cfe/trunk: Driver/RewriteMacros.cpp include/clang/Lex/Lexer.h lib/Lex/Lexer.cpp lib/Lex/TokenLexer.cpp lib/Rewrite/HTMLRewrite.cpp
Chris Lattner
sabre at nondot.org
Sat Oct 11 18:15:57 PDT 2008
Author: lattner
Date: Sat Oct 11 20:15:46 2008
New Revision: 57393
URL: http://llvm.org/viewvc/llvm-project?rev=57393&view=rev
Log:
Change how raw lexers are handled: instead of creating them and then
using LexRawToken, create one and use LexFromRawLexer. This avoids
twiddling the RawLexer flag around and simplifies some code (even
speeding raw lexing up a tiny bit).
This change also improves the token paster to use a Lexer on the stack
instead of new/deleting it.
Modified:
cfe/trunk/Driver/RewriteMacros.cpp
cfe/trunk/include/clang/Lex/Lexer.h
cfe/trunk/lib/Lex/Lexer.cpp
cfe/trunk/lib/Lex/TokenLexer.cpp
cfe/trunk/lib/Rewrite/HTMLRewrite.cpp
Modified: cfe/trunk/Driver/RewriteMacros.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Driver/RewriteMacros.cpp?rev=57393&r1=57392&r2=57393&view=diff
==============================================================================
--- cfe/trunk/Driver/RewriteMacros.cpp (original)
+++ cfe/trunk/Driver/RewriteMacros.cpp Sat Oct 11 20:15:46 2008
@@ -73,7 +73,7 @@
Token RawTok;
do {
- RawLex.LexRawToken(RawTok);
+ RawLex.LexFromRawLexer(RawTok);
// If we have an identifier with no identifier info for our raw token, look
// up the indentifier info. This is important for equality comparison of
Modified: cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Lexer.h?rev=57393&r1=57392&r2=57393&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/trunk/include/clang/Lex/Lexer.h Sat Oct 11 20:15:46 2008
@@ -105,11 +105,11 @@
const char *BufStart = 0, const char *BufEnd = 0);
/// Lexer constructor - Create a new raw lexer object. This object is only
- /// suitable for calls to 'LexRawToken'. This lexer assumes that the
- /// associated file buffer will outlive it, so it doesn't take ownership of
- /// either of them.
+ /// suitable for calls to 'LexRawToken'. This lexer assumes that the text
+ /// range will outlive it, so it doesn't take ownership of it.
Lexer(SourceLocation FileLoc, const LangOptions &Features,
- const char *BufStart, const char *BufEnd);
+ const char *BufStart, const char *BufEnd,
+ const llvm::MemoryBuffer *FromFile = 0);
/// getFeatures - Return the language features currently enabled. NOTE: this
/// lexer modifies features as a file is parsed!
@@ -141,14 +141,12 @@
LexTokenInternal(Result);
}
- /// LexRawToken - Switch the lexer to raw mode, lex a token into Result and
- /// switch it back. Return true if the 'next character to read' pointer
- /// points and the end of the lexer buffer, false otherwise.
- bool LexRawToken(Token &Result) {
- assert(!(PP && LexingRawMode) && "Already in raw mode!");
- LexingRawMode = true;
+ /// LexFromRawLexer - Lex a token from a designated raw lexer (one with no
+ /// associated preprocessor object. Return true if the 'next character to
+ /// read' pointer points and the end of the lexer buffer, false otherwise.
+ bool LexFromRawLexer(Token &Result) {
+ assert(LexingRawMode && "Not already in raw mode!");
Lex(Result);
- LexingRawMode = PP == 0;
// Note that lexing to the end of the buffer doesn't implicitly delete the
// lexer when in raw mode.
return BufferPtr == BufferEnd;
Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=57393&r1=57392&r2=57393&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Sat Oct 11 20:15:46 2008
@@ -104,15 +104,16 @@
}
/// Lexer constructor - Create a new raw lexer object. This object is only
-/// suitable for calls to 'LexRawToken'. This lexer assumes that the
-/// associated file buffer will outlive it, so it doesn't take ownership of it.
+/// suitable for calls to 'LexRawToken'. This lexer assumes that the text
+/// range will outlive it, so it doesn't take ownership of it.
Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
- const char *BufStart, const char *BufEnd)
+ const char *BufStart, const char *BufEnd,
+ const llvm::MemoryBuffer *FromFile)
: FileLoc(fileloc), PP(0), Features(features) {
Is_PragmaLexer = false;
InitCharacterInfo();
- BufferStart = BufStart;
+ BufferStart = FromFile ? FromFile->getBufferStart() : BufStart;
BufferPtr = BufStart;
BufferEnd = BufEnd;
@@ -192,7 +193,7 @@
// Create a lexer starting at the beginning of this token.
Lexer TheLexer(Loc, LangOpts, StrData, BufEnd);
Token TheTok;
- TheLexer.LexRawToken(TheTok);
+ TheLexer.LexFromRawLexer(TheTok);
return TheTok.getLength();
}
Modified: cfe/trunk/lib/Lex/TokenLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/TokenLexer.cpp?rev=57393&r1=57392&r2=57393&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/TokenLexer.cpp (original)
+++ cfe/trunk/lib/Lex/TokenLexer.cpp Sat Oct 11 20:15:46 2008
@@ -396,23 +396,23 @@
SourceManager &SourceMgr = PP.getSourceManager();
const char *ResultStrData = SourceMgr.getCharacterData(ResultTokLoc);
+ const llvm::MemoryBuffer *Buffer =
+ SourceMgr.getBuffer(ResultTokLoc.getFileID());
+
// Make a lexer object so that we lex and expand the paste result.
- Lexer *TL = new Lexer(ResultTokLoc, PP, ResultStrData,
- ResultStrData+LHSLen+RHSLen /*don't include null*/);
+ Lexer TL(ResultTokLoc, PP.getLangOptions(), ResultStrData,
+ ResultStrData+LHSLen+RHSLen /*don't include null*/, Buffer);
// Lex a token in raw mode. This way it won't look up identifiers
// automatically, lexing off the end will return an eof token, and
// warnings are disabled. This returns true if the result token is the
// entire buffer.
- bool IsComplete = TL->LexRawToken(Result);
+ bool IsComplete = TL.LexFromRawLexer(Result);
// If we got an EOF token, we didn't form even ONE token. For example, we
// did "/ ## /" to get "//".
IsComplete &= Result.isNot(tok::eof);
isInvalid = !IsComplete;
-
- // We're now done with the temporary lexer.
- delete TL;
}
// If pasting the two tokens didn't form a full new token, this is an error.
Modified: cfe/trunk/lib/Rewrite/HTMLRewrite.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Rewrite/HTMLRewrite.cpp?rev=57393&r1=57392&r2=57393&view=diff
==============================================================================
--- cfe/trunk/lib/Rewrite/HTMLRewrite.cpp (original)
+++ cfe/trunk/lib/Rewrite/HTMLRewrite.cpp Sat Oct 11 20:15:46 2008
@@ -357,7 +357,7 @@
// Lex all the tokens in raw mode, to avoid entering #includes or expanding
// macros.
Token Tok;
- L.LexRawToken(Tok);
+ L.LexFromRawLexer(Tok);
while (Tok.isNot(tok::eof)) {
// Since we are lexing unexpanded tokens, all tokens are from the main
@@ -398,10 +398,10 @@
// Eat all of the tokens until we get to the next one at the start of
// line.
unsigned TokEnd = TokOffs+TokLen;
- L.LexRawToken(Tok);
+ L.LexFromRawLexer(Tok);
while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
TokEnd = SourceMgr.getFullFilePos(Tok.getLocation())+Tok.getLength();
- L.LexRawToken(Tok);
+ L.LexFromRawLexer(Tok);
}
// Find end of line. This is a hack.
@@ -413,7 +413,7 @@
}
}
- L.LexRawToken(Tok);
+ L.LexFromRawLexer(Tok);
}
}
More information about the cfe-commits
mailing list