[cfe-commits] r48072 - in /cfe/trunk: Lex/MacroExpander.cpp Lex/TokenLexer.cpp clang.xcodeproj/project.pbxproj include/clang/Lex/MacroExpander.h include/clang/Lex/Preprocessor.h include/clang/Lex/TokenLexer.h
Chris Lattner
sabre at nondot.org
Sat Mar 8 18:22:57 PST 2008
Author: lattner
Date: Sat Mar 8 20:22:57 2008
New Revision: 48072
URL: http://llvm.org/viewvc/llvm-project?rev=48072&view=rev
Log:
Rename MacroExpander.cpp/h -> TokenLexer.cpp/h
Added:
cfe/trunk/Lex/TokenLexer.cpp
- copied, changed from r48071, cfe/trunk/Lex/MacroExpander.cpp
cfe/trunk/include/clang/Lex/TokenLexer.h
- copied, changed from r48071, cfe/trunk/include/clang/Lex/MacroExpander.h
Removed:
cfe/trunk/Lex/MacroExpander.cpp
cfe/trunk/include/clang/Lex/MacroExpander.h
Modified:
cfe/trunk/clang.xcodeproj/project.pbxproj
cfe/trunk/include/clang/Lex/Preprocessor.h
Removed: cfe/trunk/Lex/MacroExpander.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Lex/MacroExpander.cpp?rev=48071&view=auto
==============================================================================
--- cfe/trunk/Lex/MacroExpander.cpp (original)
+++ cfe/trunk/Lex/MacroExpander.cpp (removed)
@@ -1,695 +0,0 @@
-//===--- TokenLexer.cpp - Lex from a token stream -------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the TokenLexer interface.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Lex/MacroExpander.h"
-#include "clang/Lex/MacroInfo.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Basic/Diagnostic.h"
-#include "llvm/ADT/SmallVector.h"
-using namespace clang;
-
-//===----------------------------------------------------------------------===//
-// MacroArgs Implementation
-//===----------------------------------------------------------------------===//
-
-/// MacroArgs ctor function - This destroys the vector passed in.
-MacroArgs *MacroArgs::create(const MacroInfo *MI,
- const Token *UnexpArgTokens,
- unsigned NumToks, bool VarargsElided) {
- assert(MI->isFunctionLike() &&
- "Can't have args for an object-like macro!");
-
- // Allocate memory for the MacroArgs object with the lexer tokens at the end.
- MacroArgs *Result = (MacroArgs*)malloc(sizeof(MacroArgs) +
- NumToks*sizeof(Token));
- // Construct the macroargs object.
- new (Result) MacroArgs(NumToks, VarargsElided);
-
- // Copy the actual unexpanded tokens to immediately after the result ptr.
- if (NumToks)
- memcpy(const_cast<Token*>(Result->getUnexpArgument(0)),
- UnexpArgTokens, NumToks*sizeof(Token));
-
- return Result;
-}
-
-/// destroy - Destroy and deallocate the memory for this object.
-///
-void MacroArgs::destroy() {
- // Run the dtor to deallocate the vectors.
- this->~MacroArgs();
- // Release the memory for the object.
- free(this);
-}
-
-
-/// getArgLength - Given a pointer to an expanded or unexpanded argument,
-/// return the number of tokens, not counting the EOF, that make up the
-/// argument.
-unsigned MacroArgs::getArgLength(const Token *ArgPtr) {
- unsigned NumArgTokens = 0;
- for (; ArgPtr->isNot(tok::eof); ++ArgPtr)
- ++NumArgTokens;
- return NumArgTokens;
-}
-
-
-/// getUnexpArgument - Return the unexpanded tokens for the specified formal.
-///
-const Token *MacroArgs::getUnexpArgument(unsigned Arg) const {
- // The unexpanded argument tokens start immediately after the MacroArgs object
- // in memory.
- const Token *Start = (const Token *)(this+1);
- const Token *Result = Start;
- // Scan to find Arg.
- for (; Arg; ++Result) {
- assert(Result < Start+NumUnexpArgTokens && "Invalid arg #");
- if (Result->is(tok::eof))
- --Arg;
- }
- return Result;
-}
-
-
-/// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
-/// by pre-expansion, return false. Otherwise, conservatively return true.
-bool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok,
- Preprocessor &PP) const {
- // If there are no identifiers in the argument list, or if the identifiers are
- // known to not be macros, pre-expansion won't modify it.
- for (; ArgTok->isNot(tok::eof); ++ArgTok)
- if (IdentifierInfo *II = ArgTok->getIdentifierInfo()) {
- if (II->hasMacroDefinition() && PP.getMacroInfo(II)->isEnabled())
- // Return true even though the macro could be a function-like macro
- // without a following '(' token.
- return true;
- }
- return false;
-}
-
-/// getPreExpArgument - Return the pre-expanded form of the specified
-/// argument.
-const std::vector<Token> &
-MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
- assert(Arg < NumUnexpArgTokens && "Invalid argument number!");
-
- // If we have already computed this, return it.
- if (PreExpArgTokens.empty())
- PreExpArgTokens.resize(NumUnexpArgTokens);
-
- std::vector<Token> &Result = PreExpArgTokens[Arg];
- if (!Result.empty()) return Result;
-
- const Token *AT = getUnexpArgument(Arg);
- unsigned NumToks = getArgLength(AT)+1; // Include the EOF.
-
- // Otherwise, we have to pre-expand this argument, populating Result. To do
- // this, we set up a fake TokenLexer to lex from the unexpanded argument
- // list. With this installed, we lex expanded tokens until we hit the EOF
- // token at the end of the unexp list.
- PP.EnterTokenStream(AT, NumToks);
-
- // Lex all of the macro-expanded tokens into Result.
- do {
- Result.push_back(Token());
- PP.Lex(Result.back());
- } while (Result.back().isNot(tok::eof));
-
- // Pop the token stream off the top of the stack. We know that the internal
- // pointer inside of it is to the "end" of the token stream, but the stack
- // will not otherwise be popped until the next token is lexed. The problem is
- // that the token may be lexed sometime after the vector of tokens itself is
- // destroyed, which would be badness.
- PP.RemoveTopOfLexerStack();
- return Result;
-}
-
-
-/// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
-/// tokens into the literal string token that should be produced by the C #
-/// preprocessor operator.
-///
-static Token StringifyArgument(const Token *ArgToks,
- Preprocessor &PP, bool Charify = false) {
- Token Tok;
- Tok.startToken();
- Tok.setKind(tok::string_literal);
-
- const Token *ArgTokStart = ArgToks;
-
- // Stringify all the tokens.
- std::string Result = "\"";
- // FIXME: Optimize this loop to not use std::strings.
- bool isFirst = true;
- for (; ArgToks->isNot(tok::eof); ++ArgToks) {
- const Token &Tok = *ArgToks;
- if (!isFirst && (Tok.hasLeadingSpace() || Tok.isAtStartOfLine()))
- Result += ' ';
- isFirst = false;
-
- // If this is a string or character constant, escape the token as specified
- // by 6.10.3.2p2.
- if (Tok.is(tok::string_literal) || // "foo"
- Tok.is(tok::wide_string_literal) || // L"foo"
- Tok.is(tok::char_constant)) { // 'x' and L'x'.
- Result += Lexer::Stringify(PP.getSpelling(Tok));
- } else {
- // Otherwise, just append the token.
- Result += PP.getSpelling(Tok);
- }
- }
-
- // If the last character of the string is a \, and if it isn't escaped, this
- // is an invalid string literal, diagnose it as specified in C99.
- if (Result[Result.size()-1] == '\\') {
- // Count the number of consequtive \ characters. If even, then they are
- // just escaped backslashes, otherwise it's an error.
- unsigned FirstNonSlash = Result.size()-2;
- // Guaranteed to find the starting " if nothing else.
- while (Result[FirstNonSlash] == '\\')
- --FirstNonSlash;
- if ((Result.size()-1-FirstNonSlash) & 1) {
- // Diagnose errors for things like: #define F(X) #X / F(\)
- PP.Diag(ArgToks[-1], diag::pp_invalid_string_literal);
- Result.erase(Result.end()-1); // remove one of the \'s.
- }
- }
- Result += '"';
-
- // If this is the charify operation and the result is not a legal character
- // constant, diagnose it.
- if (Charify) {
- // First step, turn double quotes into single quotes:
- Result[0] = '\'';
- Result[Result.size()-1] = '\'';
-
- // Check for bogus character.
- bool isBad = false;
- if (Result.size() == 3) {
- isBad = Result[1] == '\''; // ''' is not legal. '\' already fixed above.
- } else {
- isBad = (Result.size() != 4 || Result[1] != '\\'); // Not '\x'
- }
-
- if (isBad) {
- PP.Diag(ArgTokStart[0], diag::err_invalid_character_to_charify);
- Result = "' '"; // Use something arbitrary, but legal.
- }
- }
-
- Tok.setLength(Result.size());
- Tok.setLocation(PP.CreateString(&Result[0], Result.size()));
- return Tok;
-}
-
-/// getStringifiedArgument - Compute, cache, and return the specified argument
-/// that has been 'stringified' as required by the # operator.
-const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
- Preprocessor &PP) {
- assert(ArgNo < NumUnexpArgTokens && "Invalid argument number!");
- if (StringifiedArgs.empty()) {
- StringifiedArgs.resize(getNumArguments());
- memset(&StringifiedArgs[0], 0,
- sizeof(StringifiedArgs[0])*getNumArguments());
- }
- if (StringifiedArgs[ArgNo].isNot(tok::string_literal))
- StringifiedArgs[ArgNo] = StringifyArgument(getUnexpArgument(ArgNo), PP);
- return StringifiedArgs[ArgNo];
-}
-
-//===----------------------------------------------------------------------===//
-// TokenLexer Implementation
-//===----------------------------------------------------------------------===//
-
-/// Create a TokenLexer for the specified macro with the specified actual
-/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
-void TokenLexer::Init(Token &Tok, MacroArgs *Actuals) {
- // If the client is reusing a TokenLexer, make sure to free any memory
- // associated with it.
- destroy();
-
- Macro = PP.getMacroInfo(Tok.getIdentifierInfo());
- ActualArgs = Actuals;
- CurToken = 0;
- InstantiateLoc = Tok.getLocation();
- AtStartOfLine = Tok.isAtStartOfLine();
- HasLeadingSpace = Tok.hasLeadingSpace();
- Tokens = &*Macro->tokens_begin();
- OwnsTokens = false;
- NumTokens = Macro->tokens_end()-Macro->tokens_begin();
-
- // If this is a function-like macro, expand the arguments and change
- // Tokens to point to the expanded tokens.
- if (Macro->isFunctionLike() && Macro->getNumArgs())
- ExpandFunctionArguments();
-
- // Mark the macro as currently disabled, so that it is not recursively
- // expanded. The macro must be disabled only after argument pre-expansion of
- // function-like macro arguments occurs.
- Macro->DisableMacro();
-}
-
-
-
-/// Create a TokenLexer for the specified token stream. This does not
-/// take ownership of the specified token vector.
-void TokenLexer::Init(const Token *TokArray, unsigned NumToks) {
- // If the client is reusing a TokenLexer, make sure to free any memory
- // associated with it.
- destroy();
-
- Macro = 0;
- ActualArgs = 0;
- Tokens = TokArray;
- OwnsTokens = false;
- NumTokens = NumToks;
- CurToken = 0;
- InstantiateLoc = SourceLocation();
- AtStartOfLine = false;
- HasLeadingSpace = false;
-
- // Set HasLeadingSpace/AtStartOfLine so that the first token will be
- // returned unmodified.
- if (NumToks != 0) {
- AtStartOfLine = TokArray[0].isAtStartOfLine();
- HasLeadingSpace = TokArray[0].hasLeadingSpace();
- }
-}
-
-
-void TokenLexer::destroy() {
- // If this was a function-like macro that actually uses its arguments, delete
- // the expanded tokens.
- if (OwnsTokens) {
- delete [] Tokens;
- Tokens = 0;
- }
-
- // TokenLexer owns its formal arguments.
- if (ActualArgs) ActualArgs->destroy();
-}
-
-/// Expand the arguments of a function-like macro so that we can quickly
-/// return preexpanded tokens from Tokens.
-void TokenLexer::ExpandFunctionArguments() {
- llvm::SmallVector<Token, 128> ResultToks;
-
- // Loop through 'Tokens', expanding them into ResultToks. Keep
- // track of whether we change anything. If not, no need to keep them. If so,
- // we install the newly expanded sequence as the new 'Tokens' list.
- bool MadeChange = false;
-
- // NextTokGetsSpace - When this is true, the next token appended to the
- // output list will get a leading space, regardless of whether it had one to
- // begin with or not. This is used for placemarker support.
- bool NextTokGetsSpace = false;
-
- for (unsigned i = 0, e = NumTokens; i != e; ++i) {
- // If we found the stringify operator, get the argument stringified. The
- // preprocessor already verified that the following token is a macro name
- // when the #define was parsed.
- const Token &CurTok = Tokens[i];
- if (CurTok.is(tok::hash) || CurTok.is(tok::hashat)) {
- int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo());
- assert(ArgNo != -1 && "Token following # is not an argument?");
-
- Token Res;
- if (CurTok.is(tok::hash)) // Stringify
- Res = ActualArgs->getStringifiedArgument(ArgNo, PP);
- else {
- // 'charify': don't bother caching these.
- Res = StringifyArgument(ActualArgs->getUnexpArgument(ArgNo), PP, true);
- }
-
- // The stringified/charified string leading space flag gets set to match
- // the #/#@ operator.
- if (CurTok.hasLeadingSpace() || NextTokGetsSpace)
- Res.setFlag(Token::LeadingSpace);
-
- ResultToks.push_back(Res);
- MadeChange = true;
- ++i; // Skip arg name.
- NextTokGetsSpace = false;
- continue;
- }
-
- // Otherwise, if this is not an argument token, just add the token to the
- // output buffer.
- IdentifierInfo *II = CurTok.getIdentifierInfo();
- int ArgNo = II ? Macro->getArgumentNum(II) : -1;
- if (ArgNo == -1) {
- // This isn't an argument, just add it.
- ResultToks.push_back(CurTok);
-
- if (NextTokGetsSpace) {
- ResultToks.back().setFlag(Token::LeadingSpace);
- NextTokGetsSpace = false;
- }
- continue;
- }
-
- // An argument is expanded somehow, the result is different than the
- // input.
- MadeChange = true;
-
- // Otherwise, this is a use of the argument. Find out if there is a paste
- // (##) operator before or after the argument.
- bool PasteBefore =
- !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
- bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash);
-
- // If it is not the LHS/RHS of a ## operator, we must pre-expand the
- // argument and substitute the expanded tokens into the result. This is
- // C99 6.10.3.1p1.
- if (!PasteBefore && !PasteAfter) {
- const Token *ResultArgToks;
-
- // Only preexpand the argument if it could possibly need it. This
- // avoids some work in common cases.
- const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
- if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
- ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
- else
- ResultArgToks = ArgTok; // Use non-preexpanded tokens.
-
- // If the arg token expanded into anything, append it.
- if (ResultArgToks->isNot(tok::eof)) {
- unsigned FirstResult = ResultToks.size();
- unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
- ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
-
- // If any tokens were substituted from the argument, the whitespace
- // before the first token should match the whitespace of the arg
- // identifier.
- ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
- CurTok.hasLeadingSpace() ||
- NextTokGetsSpace);
- NextTokGetsSpace = false;
- } else {
- // If this is an empty argument, and if there was whitespace before the
- // formal token, make sure the next token gets whitespace before it.
- NextTokGetsSpace = CurTok.hasLeadingSpace();
- }
- continue;
- }
-
- // Okay, we have a token that is either the LHS or RHS of a paste (##)
- // argument. It gets substituted as its non-pre-expanded tokens.
- const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
- unsigned NumToks = MacroArgs::getArgLength(ArgToks);
- if (NumToks) { // Not an empty argument?
- // If this is the GNU ", ## __VA_ARG__" extension, and we just learned
- // that __VA_ARG__ expands to multiple tokens, avoid a pasting error when
- // the expander trys to paste ',' with the first token of the __VA_ARG__
- // expansion.
- if (PasteBefore && ResultToks.size() >= 2 &&
- ResultToks[ResultToks.size()-2].is(tok::comma) &&
- (unsigned)ArgNo == Macro->getNumArgs()-1 &&
- Macro->isVariadic()) {
- // Remove the paste operator, report use of the extension.
- PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
- ResultToks.pop_back();
- }
-
- ResultToks.append(ArgToks, ArgToks+NumToks);
-
- // If the next token was supposed to get leading whitespace, ensure it has
- // it now.
- if (NextTokGetsSpace) {
- ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace);
- NextTokGetsSpace = false;
- }
- continue;
- }
-
- // If an empty argument is on the LHS or RHS of a paste, the standard (C99
- // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
- // implement this by eating ## operators when a LHS or RHS expands to
- // empty.
- NextTokGetsSpace |= CurTok.hasLeadingSpace();
- if (PasteAfter) {
- // Discard the argument token and skip (don't copy to the expansion
- // buffer) the paste operator after it.
- NextTokGetsSpace |= Tokens[i+1].hasLeadingSpace();
- ++i;
- continue;
- }
-
- // If this is on the RHS of a paste operator, we've already copied the
- // paste operator to the ResultToks list. Remove it.
- assert(PasteBefore && ResultToks.back().is(tok::hashhash));
- NextTokGetsSpace |= ResultToks.back().hasLeadingSpace();
- ResultToks.pop_back();
-
- // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
- // and if the macro had at least one real argument, and if the token before
- // the ## was a comma, remove the comma.
- if ((unsigned)ArgNo == Macro->getNumArgs()-1 && // is __VA_ARGS__
- ActualArgs->isVarargsElidedUse() && // Argument elided.
- !ResultToks.empty() && ResultToks.back().is(tok::comma)) {
- // Never add a space, even if the comma, ##, or arg had a space.
- NextTokGetsSpace = false;
- // Remove the paste operator, report use of the extension.
- PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
- ResultToks.pop_back();
- }
- continue;
- }
-
- // If anything changed, install this as the new Tokens list.
- if (MadeChange) {
- // This is deleted in the dtor.
- NumTokens = ResultToks.size();
- Token *Res = new Token[ResultToks.size()];
- if (NumTokens)
- memcpy(Res, &ResultToks[0], NumTokens*sizeof(Token));
- Tokens = Res;
- OwnsTokens = true;
- }
-}
-
-/// Lex - Lex and return a token from this macro stream.
-///
-void TokenLexer::Lex(Token &Tok) {
- // Lexing off the end of the macro, pop this macro off the expansion stack.
- if (isAtEnd()) {
- // If this is a macro (not a token stream), mark the macro enabled now
- // that it is no longer being expanded.
- if (Macro) Macro->EnableMacro();
-
- // Pop this context off the preprocessors lexer stack and get the next
- // token. This will delete "this" so remember the PP instance var.
- Preprocessor &PPCache = PP;
- if (PP.HandleEndOfMacro(Tok))
- return;
-
- // HandleEndOfMacro may not return a token. If it doesn't, lex whatever is
- // next.
- return PPCache.Lex(Tok);
- }
-
- // If this is the first token of the expanded result, we inherit spacing
- // properties later.
- bool isFirstToken = CurToken == 0;
-
- // Get the next token to return.
- Tok = Tokens[CurToken++];
-
- // If this token is followed by a token paste (##) operator, paste the tokens!
- if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash))
- if (PasteTokens(Tok)) {
- // When handling the microsoft /##/ extension, the final token is
- // returned by PasteTokens, not the pasted token.
- return;
- }
-
- // The token's current location indicate where the token was lexed from. We
- // need this information to compute the spelling of the token, but any
- // diagnostics for the expanded token should appear as if they came from
- // InstantiationLoc. Pull this information together into a new SourceLocation
- // that captures all of this.
- if (InstantiateLoc.isValid()) { // Don't do this for token streams.
- SourceManager &SrcMgr = PP.getSourceManager();
- Tok.setLocation(SrcMgr.getInstantiationLoc(Tok.getLocation(),
- InstantiateLoc));
- }
-
- // If this is the first token, set the lexical properties of the token to
- // match the lexical properties of the macro identifier.
- if (isFirstToken) {
- Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
- Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
- }
-
- // Handle recursive expansion!
- if (Tok.getIdentifierInfo())
- return PP.HandleIdentifier(Tok);
-
- // Otherwise, return a normal token.
-}
-
-/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ##
-/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
-/// are is another ## after it, chomp it iteratively. Return the result as Tok.
-/// If this returns true, the caller should immediately return the token.
-bool TokenLexer::PasteTokens(Token &Tok) {
- llvm::SmallVector<char, 128> Buffer;
- do {
- // Consume the ## operator.
- SourceLocation PasteOpLoc = Tokens[CurToken].getLocation();
- ++CurToken;
- assert(!isAtEnd() && "No token on the RHS of a paste operator!");
-
- // Get the RHS token.
- const Token &RHS = Tokens[CurToken];
-
- bool isInvalid = false;
-
- // Allocate space for the result token. This is guaranteed to be enough for
- // the two tokens and a null terminator.
- Buffer.resize(Tok.getLength() + RHS.getLength() + 1);
-
- // Get the spelling of the LHS token in Buffer.
- const char *BufPtr = &Buffer[0];
- unsigned LHSLen = PP.getSpelling(Tok, BufPtr);
- if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
- memcpy(&Buffer[0], BufPtr, LHSLen);
-
- BufPtr = &Buffer[LHSLen];
- unsigned RHSLen = PP.getSpelling(RHS, BufPtr);
- if (BufPtr != &Buffer[LHSLen]) // Really, we want the chars in Buffer!
- memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
-
- // Add null terminator.
- Buffer[LHSLen+RHSLen] = '\0';
-
- // Trim excess space.
- Buffer.resize(LHSLen+RHSLen+1);
-
- // Plop the pasted result (including the trailing newline and null) into a
- // scratch buffer where we can lex it.
- SourceLocation ResultTokLoc = PP.CreateString(&Buffer[0], Buffer.size());
-
- // Lex the resultant pasted token into Result.
- Token Result;
-
- // Avoid testing /*, as the lexer would think it is the start of a comment
- // and emit an error that it is unterminated.
- if (Tok.is(tok::slash) && RHS.is(tok::star)) {
- isInvalid = true;
- } else if (Tok.is(tok::identifier) && RHS.is(tok::identifier)) {
- // Common paste case: identifier+identifier = identifier. Avoid creating
- // a lexer and other overhead.
- PP.IncrementPasteCounter(true);
- Result.startToken();
- Result.setKind(tok::identifier);
- Result.setLocation(ResultTokLoc);
- Result.setLength(LHSLen+RHSLen);
- } else {
- PP.IncrementPasteCounter(false);
-
- // Make a lexer to lex this string from.
- SourceManager &SourceMgr = PP.getSourceManager();
- const char *ResultStrData = SourceMgr.getCharacterData(ResultTokLoc);
-
- // Make a lexer object so that we lex and expand the paste result.
- Lexer *TL = new Lexer(ResultTokLoc, PP, ResultStrData,
- ResultStrData+LHSLen+RHSLen /*don't include null*/);
-
- // Lex a token in raw mode. This way it won't look up identifiers
- // automatically, lexing off the end will return an eof token, and
- // warnings are disabled. This returns true if the result token is the
- // entire buffer.
- bool IsComplete = TL->LexRawToken(Result);
-
- // If we got an EOF token, we didn't form even ONE token. For example, we
- // did "/ ## /" to get "//".
- IsComplete &= Result.isNot(tok::eof);
- isInvalid = !IsComplete;
-
- // We're now done with the temporary lexer.
- delete TL;
- }
-
- // If pasting the two tokens didn't form a full new token, this is an error.
- // This occurs with "x ## +" and other stuff. Return with Tok unmodified
- // and with RHS as the next token to lex.
- if (isInvalid) {
- // Test for the Microsoft extension of /##/ turning into // here on the
- // error path.
- if (PP.getLangOptions().Microsoft && Tok.is(tok::slash) &&
- RHS.is(tok::slash)) {
- HandleMicrosoftCommentPaste(Tok);
- return true;
- } else {
- // TODO: If not in assembler language mode.
- PP.Diag(PasteOpLoc, diag::err_pp_bad_paste,
- std::string(Buffer.begin(), Buffer.end()-1));
- return false;
- }
- }
-
- // Turn ## into 'unknown' to avoid # ## # from looking like a paste
- // operator.
- if (Result.is(tok::hashhash))
- Result.setKind(tok::unknown);
- // FIXME: Turn __VA_ARGS__ into "not a token"?
-
- // Transfer properties of the LHS over the the Result.
- Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine());
- Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace());
-
- // Finally, replace LHS with the result, consume the RHS, and iterate.
- ++CurToken;
- Tok = Result;
- } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash));
-
- // Now that we got the result token, it will be subject to expansion. Since
- // token pasting re-lexes the result token in raw mode, identifier information
- // isn't looked up. As such, if the result is an identifier, look up id info.
- if (Tok.is(tok::identifier)) {
- // Look up the identifier info for the token. We disabled identifier lookup
- // by saying we're skipping contents, so we need to do this manually.
- Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
- }
- return false;
-}
-
-/// isNextTokenLParen - If the next token lexed will pop this macro off the
-/// expansion stack, return 2. If the next unexpanded token is a '(', return
-/// 1, otherwise return 0.
-unsigned TokenLexer::isNextTokenLParen() const {
- // Out of tokens?
- if (isAtEnd())
- return 2;
- return Tokens[CurToken].is(tok::l_paren);
-}
-
-
-/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
-/// together to form a comment that comments out everything in the current
-/// macro, other active macros, and anything left on the current physical
-/// source line of the instantiated buffer. Handle this by returning the
-/// first token on the next line.
-void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) {
- // We 'comment out' the rest of this macro by just ignoring the rest of the
- // tokens that have not been lexed yet, if any.
-
- // Since this must be a macro, mark the macro enabled now that it is no longer
- // being expanded.
- assert(Macro && "Token streams can't paste comments");
- Macro->EnableMacro();
-
- PP.HandleMicrosoftCommentPaste(Tok);
-}
Copied: cfe/trunk/Lex/TokenLexer.cpp (from r48071, cfe/trunk/Lex/MacroExpander.cpp)
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/Lex/TokenLexer.cpp?p2=cfe/trunk/Lex/TokenLexer.cpp&p1=cfe/trunk/Lex/MacroExpander.cpp&r1=48071&r2=48072&rev=48072&view=diff
==============================================================================
--- cfe/trunk/Lex/MacroExpander.cpp (original)
+++ cfe/trunk/Lex/TokenLexer.cpp Sat Mar 8 20:22:57 2008
@@ -11,7 +11,7 @@
//
//===----------------------------------------------------------------------===//
-#include "clang/Lex/MacroExpander.h"
+#include "clang/Lex/TokenLexer.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Basic/SourceManager.h"
Modified: cfe/trunk/clang.xcodeproj/project.pbxproj
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/clang.xcodeproj/project.pbxproj?rev=48072&r1=48071&r2=48072&view=diff
==============================================================================
--- cfe/trunk/clang.xcodeproj/project.pbxproj (original)
+++ cfe/trunk/clang.xcodeproj/project.pbxproj Sat Mar 8 20:22:57 2008
@@ -49,7 +49,6 @@
DE344AB80AE5DF6D00DBC861 /* HeaderSearch.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DE344AB70AE5DF6D00DBC861 /* HeaderSearch.h */; };
DE344B540AE5E46C00DBC861 /* HeaderSearch.cpp in Sources */ = {isa = PBXBuildFile; fileRef = DE344B530AE5E46C00DBC861 /* HeaderSearch.cpp */; };
DE3450D70AEB543100DBC861 /* DirectoryLookup.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DE3450D60AEB543100DBC861 /* DirectoryLookup.h */; };
- DE3451580AEC176100DBC861 /* MacroExpander.cpp in Sources */ = {isa = PBXBuildFile; fileRef = DE3451570AEC176100DBC861 /* MacroExpander.cpp */; };
DE3452410AEF1A2D00DBC861 /* Stmt.cpp in Sources */ = {isa = PBXBuildFile; fileRef = DE3452400AEF1A2D00DBC861 /* Stmt.cpp */; };
DE3452810AEF1B1800DBC861 /* Stmt.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DE3452800AEF1B1800DBC861 /* Stmt.h */; };
DE345C1A0AFC658B00DBC861 /* StmtVisitor.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DE345C190AFC658B00DBC861 /* StmtVisitor.h */; };
@@ -103,6 +102,7 @@
DE75ED290B044DC90020CF81 /* ASTContext.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DE75ED280B044DC90020CF81 /* ASTContext.h */; };
DE75EDF10B06880E0020CF81 /* Type.cpp in Sources */ = {isa = PBXBuildFile; fileRef = DE75EDF00B06880E0020CF81 /* Type.cpp */; };
DE85CD4B0D8378320070E26E /* Directives.cpp in Sources */ = {isa = PBXBuildFile; fileRef = DE85CD4A0D8378320070E26E /* Directives.cpp */; };
+ DE85CD810D8380B10070E26E /* TokenLexer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = DE85CD800D8380B10070E26E /* TokenLexer.cpp */; };
DE928B130C05659200231DA4 /* ModuleBuilder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = DE928B120C05659200231DA4 /* ModuleBuilder.cpp */; };
DE928B200C0565B000231DA4 /* ModuleBuilder.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DE928B1F0C0565B000231DA4 /* ModuleBuilder.h */; };
DE928B7D0C0A615100231DA4 /* CodeGenModule.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DE928B7C0C0A615100231DA4 /* CodeGenModule.h */; };
@@ -132,7 +132,6 @@
DED7D7470A524295003AD0FB /* TokenKinds.def in CopyFiles */ = {isa = PBXBuildFile; fileRef = DED7D7370A524295003AD0FB /* TokenKinds.def */; };
DED7D7480A524295003AD0FB /* TokenKinds.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DED7D7380A524295003AD0FB /* TokenKinds.h */; };
DED7D74A0A524295003AD0FB /* Lexer.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DED7D73B0A524295003AD0FB /* Lexer.h */; };
- DED7D74C0A524295003AD0FB /* MacroExpander.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DED7D73D0A524295003AD0FB /* MacroExpander.h */; };
DED7D74D0A524295003AD0FB /* MacroInfo.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DED7D73E0A524295003AD0FB /* MacroInfo.h */; };
DED7D74E0A524295003AD0FB /* Pragma.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DED7D73F0A524295003AD0FB /* Pragma.h */; };
DED7D74F0A524295003AD0FB /* Preprocessor.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = DED7D7400A524295003AD0FB /* Preprocessor.h */; };
@@ -178,7 +177,6 @@
DED7D7470A524295003AD0FB /* TokenKinds.def in CopyFiles */,
DED7D7480A524295003AD0FB /* TokenKinds.h in CopyFiles */,
DED7D74A0A524295003AD0FB /* Lexer.h in CopyFiles */,
- DED7D74C0A524295003AD0FB /* MacroExpander.h in CopyFiles */,
DED7D74D0A524295003AD0FB /* MacroInfo.h in CopyFiles */,
DED7D74E0A524295003AD0FB /* Pragma.h in CopyFiles */,
DED7D74F0A524295003AD0FB /* Preprocessor.h in CopyFiles */,
@@ -298,7 +296,6 @@
DE344AB70AE5DF6D00DBC861 /* HeaderSearch.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = HeaderSearch.h; sourceTree = "<group>"; };
DE344B530AE5E46C00DBC861 /* HeaderSearch.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; path = HeaderSearch.cpp; sourceTree = "<group>"; };
DE3450D60AEB543100DBC861 /* DirectoryLookup.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = DirectoryLookup.h; sourceTree = "<group>"; };
- DE3451570AEC176100DBC861 /* MacroExpander.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; path = MacroExpander.cpp; sourceTree = "<group>"; };
DE3452400AEF1A2D00DBC861 /* Stmt.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; name = Stmt.cpp; path = AST/Stmt.cpp; sourceTree = "<group>"; };
DE3452800AEF1B1800DBC861 /* Stmt.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; name = Stmt.h; path = clang/AST/Stmt.h; sourceTree = "<group>"; };
DE345C190AFC658B00DBC861 /* StmtVisitor.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; name = StmtVisitor.h; path = clang/AST/StmtVisitor.h; sourceTree = "<group>"; };
@@ -366,6 +363,8 @@
DE75ED280B044DC90020CF81 /* ASTContext.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; name = ASTContext.h; path = clang/AST/ASTContext.h; sourceTree = "<group>"; };
DE75EDF00B06880E0020CF81 /* Type.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; name = Type.cpp; path = AST/Type.cpp; sourceTree = "<group>"; };
DE85CD4A0D8378320070E26E /* Directives.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Directives.cpp; sourceTree = "<group>"; };
+ DE85CD800D8380B10070E26E /* TokenLexer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = TokenLexer.cpp; sourceTree = "<group>"; };
+ DE85CD840D8380F20070E26E /* TokenLexer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = TokenLexer.h; sourceTree = "<group>"; };
DE928B120C05659200231DA4 /* ModuleBuilder.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; name = ModuleBuilder.cpp; path = CodeGen/ModuleBuilder.cpp; sourceTree = "<group>"; };
DE928B1F0C0565B000231DA4 /* ModuleBuilder.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; name = ModuleBuilder.h; path = clang/CodeGen/ModuleBuilder.h; sourceTree = "<group>"; };
DE928B7C0C0A615100231DA4 /* CodeGenModule.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; name = CodeGenModule.h; path = CodeGen/CodeGenModule.h; sourceTree = "<group>"; };
@@ -395,7 +394,6 @@
DED7D7370A524295003AD0FB /* TokenKinds.def */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = text; path = TokenKinds.def; sourceTree = "<group>"; };
DED7D7380A524295003AD0FB /* TokenKinds.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = TokenKinds.h; sourceTree = "<group>"; };
DED7D73B0A524295003AD0FB /* Lexer.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = Lexer.h; sourceTree = "<group>"; };
- DED7D73D0A524295003AD0FB /* MacroExpander.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = MacroExpander.h; sourceTree = "<group>"; };
DED7D73E0A524295003AD0FB /* MacroInfo.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = MacroInfo.h; sourceTree = "<group>"; };
DED7D73F0A524295003AD0FB /* Pragma.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = Pragma.h; sourceTree = "<group>"; };
DED7D7400A524295003AD0FB /* Preprocessor.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = Preprocessor.h; sourceTree = "<group>"; };
@@ -781,7 +779,6 @@
DE344AB70AE5DF6D00DBC861 /* HeaderSearch.h */,
DED7D73B0A524295003AD0FB /* Lexer.h */,
1A869A6E0BA2164C008DA07A /* LiteralSupport.h */,
- DED7D73D0A524295003AD0FB /* MacroExpander.h */,
DED7D73E0A524295003AD0FB /* MacroInfo.h */,
DEAEE98A0A5A2B970045101B /* MultipleIncludeOpt.h */,
DE01DA480B12ADA300AC22CE /* PPCallbacks.h */,
@@ -789,6 +786,7 @@
DED7D7400A524295003AD0FB /* Preprocessor.h */,
DED7D9170A52518C003AD0FB /* ScratchBuffer.h */,
DE6954630C5121BD00A5826B /* Token.h */,
+ DE85CD840D8380F20070E26E /* TokenLexer.h */,
);
name = Lex;
path = clang/Lex;
@@ -818,12 +816,12 @@
DE344B530AE5E46C00DBC861 /* HeaderSearch.cpp */,
DED7D79E0A5242E6003AD0FB /* Lexer.cpp */,
1A869AA70BA21ABA008DA07A /* LiteralSupport.cpp */,
- DE3451570AEC176100DBC861 /* MacroExpander.cpp */,
DED7D7A00A5242E6003AD0FB /* MacroInfo.cpp */,
DED7D7A20A5242E6003AD0FB /* PPExpressions.cpp */,
DED7D7A30A5242E6003AD0FB /* Pragma.cpp */,
DED7D7A40A5242E6003AD0FB /* Preprocessor.cpp */,
DED7D9E40A5257F6003AD0FB /* ScratchBuffer.cpp */,
+ DE85CD800D8380B10070E26E /* TokenLexer.cpp */,
);
path = Lex;
sourceTree = "<group>";
@@ -906,7 +904,6 @@
DED626C90AE0C065001E80A4 /* TargetInfo.cpp in Sources */,
DED62ABB0AE2EDF1001E80A4 /* Decl.cpp in Sources */,
DE344B540AE5E46C00DBC861 /* HeaderSearch.cpp in Sources */,
- DE3451580AEC176100DBC861 /* MacroExpander.cpp in Sources */,
DE3452410AEF1A2D00DBC861 /* Stmt.cpp in Sources */,
DE3460000AFDCC1900DBC861 /* ParseObjc.cpp in Sources */,
DE3460050AFDCC6500DBC861 /* ParseInit.cpp in Sources */,
@@ -985,6 +982,7 @@
35D55B270D81D8C60092E734 /* BasicValueFactory.cpp in Sources */,
35D55B280D81D8C60092E734 /* CFRefCount.cpp in Sources */,
DE85CD4B0D8378320070E26E /* Directives.cpp in Sources */,
+ DE85CD810D8380B10070E26E /* TokenLexer.cpp in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Removed: cfe/trunk/include/clang/Lex/MacroExpander.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/MacroExpander.h?rev=48071&view=auto
==============================================================================
--- cfe/trunk/include/clang/Lex/MacroExpander.h (original)
+++ cfe/trunk/include/clang/Lex/MacroExpander.h (removed)
@@ -1,212 +0,0 @@
-//===--- TokenLexer.h - Lex from a token buffer -----------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the TokenLexer and MacroArgs interfaces.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_TOKENLEXER_H
-#define LLVM_CLANG_TOKENLEXER_H
-
-#include "clang/Basic/SourceLocation.h"
-#include <vector>
-
-namespace clang {
- class MacroInfo;
- class Preprocessor;
- class Token;
-
-/// MacroArgs - An instance of this class captures information about
-/// the formal arguments specified to a function-like macro invocation.
-class MacroArgs {
- /// NumUnexpArgTokens - The number of raw, unexpanded tokens for the
- /// arguments. All of the actual argument tokens are allocated immediately
- /// after the MacroArgs object in memory. This is all of the arguments
- /// concatenated together, with 'EOF' markers at the end of each argument.
- unsigned NumUnexpArgTokens;
-
- /// PreExpArgTokens - Pre-expanded tokens for arguments that need them. Empty
- /// if not yet computed. This includes the EOF marker at the end of the
- /// stream.
- std::vector<std::vector<Token> > PreExpArgTokens;
-
- /// StringifiedArgs - This contains arguments in 'stringified' form. If the
- /// stringified form of an argument has not yet been computed, this is empty.
- std::vector<Token> StringifiedArgs;
-
- /// VarargsElided - True if this is a C99 style varargs macro invocation and
- /// there was no argument specified for the "..." argument. If the argument
- /// was specified (even empty) or this isn't a C99 style varargs function, or
- /// if in strict mode and the C99 varargs macro had only a ... argument, this
- /// is false.
- bool VarargsElided;
-
- MacroArgs(unsigned NumToks, bool varargsElided)
- : NumUnexpArgTokens(NumToks), VarargsElided(varargsElided) {}
- ~MacroArgs() {}
-public:
- /// MacroArgs ctor function - Create a new MacroArgs object with the specified
- /// macro and argument info.
- static MacroArgs *create(const MacroInfo *MI,
- const Token *UnexpArgTokens,
- unsigned NumArgTokens, bool VarargsElided);
-
- /// destroy - Destroy and deallocate the memory for this object.
- ///
- void destroy();
-
- /// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
- /// by pre-expansion, return false. Otherwise, conservatively return true.
- bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const;
-
- /// getUnexpArgument - Return a pointer to the first token of the unexpanded
- /// token list for the specified formal.
- ///
- const Token *getUnexpArgument(unsigned Arg) const;
-
- /// getArgLength - Given a pointer to an expanded or unexpanded argument,
- /// return the number of tokens, not counting the EOF, that make up the
- /// argument.
- static unsigned getArgLength(const Token *ArgPtr);
-
- /// getPreExpArgument - Return the pre-expanded form of the specified
- /// argument.
- const std::vector<Token> &
- getPreExpArgument(unsigned Arg, Preprocessor &PP);
-
- /// getStringifiedArgument - Compute, cache, and return the specified argument
- /// that has been 'stringified' as required by the # operator.
- const Token &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP);
-
- /// getNumArguments - Return the number of arguments passed into this macro
- /// invocation.
- unsigned getNumArguments() const { return NumUnexpArgTokens; }
-
-
- /// isVarargsElidedUse - Return true if this is a C99 style varargs macro
- /// invocation and there was no argument specified for the "..." argument. If
- /// the argument was specified (even empty) or this isn't a C99 style varargs
- /// function, or if in strict mode and the C99 varargs macro had only a ...
- /// argument, this returns false.
- bool isVarargsElidedUse() const { return VarargsElided; }
-};
-
-
-/// TokenLexer - This implements a lexer that returns token from a macro body
-/// or token stream instead of lexing from a character buffer.
-///
-class TokenLexer {
- /// Macro - The macro we are expanding from. This is null if expanding a
- /// token stream.
- ///
- MacroInfo *Macro;
-
- /// ActualArgs - The actual arguments specified for a function-like macro, or
- /// null. The TokenLexer owns the pointed-to object.
- MacroArgs *ActualArgs;
-
- /// PP - The current preprocessor object we are expanding for.
- ///
- Preprocessor &PP;
-
- /// Tokens - This is the pointer to an array of tokens that the macro is
- /// defined to, with arguments expanded for function-like macros. If this is
- /// a token stream, these are the tokens we are returning.
- const Token *Tokens;
-
- /// NumTokens - This is the length of the Tokens array.
- ///
- unsigned NumTokens;
-
- /// CurToken - This is the next token that Lex will return.
- ///
- unsigned CurToken;
-
- /// InstantiateLoc - The source location where this macro was instantiated.
- ///
- SourceLocation InstantiateLoc;
-
- /// Lexical information about the expansion point of the macro: the identifier
- /// that the macro expanded from had these properties.
- bool AtStartOfLine : 1;
- bool HasLeadingSpace : 1;
-
- /// OwnsTokens - This is true if this TokenLexer allocated the Tokens
- /// array, and thus needs to free it when destroyed. For simple object-like
- /// macros (for example) we just point into the token buffer of the macro
- /// definition, we don't make a copy of it.
- bool OwnsTokens : 1;
-
- TokenLexer(const TokenLexer&); // DO NOT IMPLEMENT
- void operator=(const TokenLexer&); // DO NOT IMPLEMENT
-public:
- /// Create a TokenLexer for the specified macro with the specified actual
- /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
- TokenLexer(Token &Tok, MacroArgs *ActualArgs, Preprocessor &pp)
- : Macro(0), ActualArgs(0), PP(pp), OwnsTokens(false) {
- Init(Tok, ActualArgs);
- }
-
- /// Init - Initialize this TokenLexer to expand from the specified macro
- /// with the specified argument information. Note that this ctor takes
- /// ownership of the ActualArgs pointer.
- void Init(Token &Tok, MacroArgs *ActualArgs);
-
- /// Create a TokenLexer for the specified token stream. This does not
- /// take ownership of the specified token vector.
- TokenLexer(const Token *TokArray, unsigned NumToks, Preprocessor &pp)
- : Macro(0), ActualArgs(0), PP(pp), OwnsTokens(false) {
- Init(TokArray, NumToks);
- }
-
- /// Init - Initialize this TokenLexer with the specified token stream.
- /// This does not take ownership of the specified token vector.
- void Init(const Token *TokArray, unsigned NumToks);
-
- ~TokenLexer() { destroy(); }
-
- /// isNextTokenLParen - If the next token lexed will pop this macro off the
- /// expansion stack, return 2. If the next unexpanded token is a '(', return
- /// 1, otherwise return 0.
- unsigned isNextTokenLParen() const;
-
- /// Lex - Lex and return a token from this macro stream.
- void Lex(Token &Tok);
-
-private:
- void destroy();
-
- /// isAtEnd - Return true if the next lex call will pop this macro off the
- /// include stack.
- bool isAtEnd() const {
- return CurToken == NumTokens;
- }
-
- /// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ##
- /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
- /// are is another ## after it, chomp it iteratively. Return the result as
- /// Tok. If this returns true, the caller should immediately return the
- /// token.
- bool PasteTokens(Token &Tok);
-
- /// Expand the arguments of a function-like macro so that we can quickly
- /// return preexpanded tokens from Tokens.
- void ExpandFunctionArguments();
-
- /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
- /// together to form a comment that comments out everything in the current
- /// macro, other active macros, and anything left on the current physical
- /// source line of the instantiated buffer. Handle this by returning the
- /// first token on the next line.
- void HandleMicrosoftCommentPaste(Token &Tok);
-};
-
-} // end namespace clang
-
-#endif
Modified: cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=48072&r1=48071&r2=48072&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/trunk/include/clang/Lex/Preprocessor.h Sat Mar 8 20:22:57 2008
@@ -15,7 +15,7 @@
#define LLVM_CLANG_LEX_PREPROCESSOR_H
#include "clang/Lex/Lexer.h"
-#include "clang/Lex/MacroExpander.h"
+#include "clang/Lex/TokenLexer.h"
#include "clang/Basic/IdentifierTable.h"
#include "clang/Basic/SourceLocation.h"
#include "llvm/ADT/DenseMap.h"
Copied: cfe/trunk/include/clang/Lex/TokenLexer.h (from r48071, cfe/trunk/include/clang/Lex/MacroExpander.h)
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/TokenLexer.h?p2=cfe/trunk/include/clang/Lex/TokenLexer.h&p1=cfe/trunk/include/clang/Lex/MacroExpander.h&r1=48071&r2=48072&rev=48072&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/MacroExpander.h (original)
+++ cfe/trunk/include/clang/Lex/TokenLexer.h Sat Mar 8 20:22:57 2008
@@ -99,7 +99,8 @@
/// TokenLexer - This implements a lexer that returns token from a macro body
-/// or token stream instead of lexing from a character buffer.
+/// or token stream instead of lexing from a character buffer. This is used for
+/// macro expansion and _Pragma handling, for example.
///
class TokenLexer {
/// Macro - The macro we are expanding from. This is null if expanding a
More information about the cfe-commits
mailing list