[cfe-commits] r60136 - in /cfe/trunk: include/clang/Lex/PTHLexer.h lib/Lex/PPLexerChange.cpp lib/Lex/PTHLexer.cpp
Ted Kremenek
kremenek at apple.com
Wed Nov 26 16:38:24 PST 2008
Author: kremenek
Date: Wed Nov 26 18:38:24 2008
New Revision: 60136
URL: http://llvm.org/viewvc/llvm-project?rev=60136&view=rev
Log:
PTHLexer now owns the Token vector.
Modified:
cfe/trunk/include/clang/Lex/PTHLexer.h
cfe/trunk/lib/Lex/PPLexerChange.cpp
cfe/trunk/lib/Lex/PTHLexer.cpp
Modified: cfe/trunk/include/clang/Lex/PTHLexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PTHLexer.h?rev=60136&r1=60135&r2=60136&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/PTHLexer.h (original)
+++ cfe/trunk/include/clang/Lex/PTHLexer.h Wed Nov 26 18:38:24 2008
@@ -15,30 +15,24 @@
#define LLVM_CLANG_PTHLEXER_H
#include "clang/Lex/PreprocessorLexer.h"
+#include <vector>
namespace clang {
class PTHLexer : public PreprocessorLexer {
- /// Tokens - This is the pointer to an array of tokens that the macro is
- /// defined to, with arguments expanded for function-like macros. If this is
- /// a token stream, these are the tokens we are returning.
- const Token *Tokens;
-
- /// LastTokenIdx - The index of the last token in Tokens. This token
- /// will be an eof token.
- unsigned LastTokenIdx;
-
+ /// Tokens - Vector of raw tokens.
+ std::vector<Token> Tokens;
+
/// CurTokenIdx - This is the index of the next token that Lex will return.
unsigned CurTokenIdx;
PTHLexer(const PTHLexer&); // DO NOT IMPLEMENT
void operator=(const PTHLexer&); // DO NOT IMPLEMENT
-public:
+public:
/// Create a PTHLexer for the specified token stream.
- PTHLexer(Preprocessor& pp, SourceLocation fileloc,
- const Token *TokArray, unsigned NumToks);
+ PTHLexer(Preprocessor& pp, SourceLocation fileloc);
~PTHLexer() {}
/// Lex - Return the next token.
@@ -46,6 +40,8 @@
void setEOF(Token &Tok);
+ std::vector<Token>& getTokens() { return Tokens; }
+
/// DiscardToEndOfLine - Read the rest of the current preprocessor line as an
/// uninterpreted string. This switches the lexer out of directive mode.
void DiscardToEndOfLine();
@@ -64,7 +60,7 @@
private:
/// AtLastToken - Returns true if the PTHLexer is at the last token.
- bool AtLastToken() const { return CurTokenIdx == LastTokenIdx; }
+ bool AtLastToken() const { return CurTokenIdx+1 == Tokens.size(); }
/// GetToken - Returns the next token. This method does not advance the
/// PTHLexer to the next token.
Modified: cfe/trunk/lib/Lex/PPLexerChange.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPLexerChange.cpp?rev=60136&r1=60135&r2=60136&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPLexerChange.cpp (original)
+++ cfe/trunk/lib/Lex/PPLexerChange.cpp Wed Nov 26 18:38:24 2008
@@ -78,6 +78,16 @@
Lexer *TheLexer = new Lexer(SourceLocation::getFileLoc(FileID, 0), *this);
EnterSourceFileWithLexer(TheLexer, CurDir);
#else
+ if (CurPPLexer || CurTokenLexer)
+ PushIncludeMacroStack();
+
+ CurDirLookup = CurDir;
+ SourceLocation Loc = SourceLocation::getFileLoc(FileID, 0);
+ CurPTHLexer.reset(new PTHLexer(*this, Loc));
+ CurPPLexer = CurPTHLexer.get();
+
+ // Generate the tokens.
+
const llvm::MemoryBuffer* B = getSourceManager().getBuffer(FileID);
// Create a raw lexer.
@@ -89,7 +99,7 @@
L.SetCommentRetentionState(false);
// Lex the file, populating our data structures.
- std::vector<Token>* Tokens = new std::vector<Token>();
+ std::vector<Token>& Tokens = CurPTHLexer->getTokens();
Token Tok;
do {
@@ -101,7 +111,7 @@
else if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
// Special processing for #include. Store the '#' token and lex
// the next token.
- Tokens->push_back(Tok);
+ Tokens.push_back(Tok);
L.LexFromRawLexer(Tok);
// Did we see 'include'/'import'/'include_next'?
@@ -116,7 +126,7 @@
K == tok::pp_include_next) {
// Save the 'include' token.
- Tokens->push_back(Tok);
+ Tokens.push_back(Tok);
// Lex the next token as an include string.
L.ParsingPreprocessorDirective = true;
@@ -128,15 +138,7 @@
}
}
}
- while (Tokens->push_back(Tok), Tok.isNot(tok::eof));
-
- if (CurPPLexer || CurTokenLexer)
- PushIncludeMacroStack();
-
- CurDirLookup = CurDir;
- SourceLocation Loc = SourceLocation::getFileLoc(FileID, 0);
- CurPTHLexer.reset(new PTHLexer(*this, Loc, &(*Tokens)[0], Tokens->size()));
- CurPPLexer = CurPTHLexer.get();
+ while (Tokens.push_back(Tok), Tok.isNot(tok::eof));
// Notify the client, if desired, that we are in a new source file.
if (Callbacks) {
Modified: cfe/trunk/lib/Lex/PTHLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PTHLexer.cpp?rev=60136&r1=60135&r2=60136&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PTHLexer.cpp (original)
+++ cfe/trunk/lib/Lex/PTHLexer.cpp Wed Nov 26 18:38:24 2008
@@ -16,16 +16,8 @@
#include "clang/Basic/TokenKinds.h"
using namespace clang;
-PTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc,
- const Token *TokArray, unsigned NumTokens)
- : PreprocessorLexer(&pp, fileloc),
- Tokens(TokArray),
- LastTokenIdx(NumTokens - 1),
- CurTokenIdx(0) {
-
- assert(NumTokens >= 1);
- assert(Tokens[LastTokenIdx].is(tok::eof));
-}
+PTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc)
+ : PreprocessorLexer(&pp, fileloc), CurTokenIdx(0) {}
Token PTHLexer::GetToken() {
Token Tok = Tokens[CurTokenIdx];
@@ -104,7 +96,8 @@
}
void PTHLexer::setEOF(Token& Tok) {
- Tok = Tokens[LastTokenIdx];
+ assert(!Tokens.empty());
+ Tok = Tokens[Tokens.size()-1];
}
void PTHLexer::DiscardToEndOfLine() {
More information about the cfe-commits
mailing list