[cfe-commits] r38699 - in /cfe/cfe/trunk: Lex/Lexer.cpp Lex/MacroExpander.cpp Lex/Preprocessor.cpp include/clang/Lex/Lexer.h include/clang/Lex/MacroExpander.h include/clang/Lex/Preprocessor.h
sabre at cs.uiuc.edu
sabre at cs.uiuc.edu
Wed Jul 11 09:23:55 PDT 2007
Author: sabre
Date: Wed Jul 11 11:23:55 2007
New Revision: 38699
URL: http://llvm.org/viewvc/llvm-project?rev=38699&view=rev
Log:
Implement "lparen scanning" for lexer buffers, by making "skipping lexing"
completely reversible. This implements tests 3/4 of
test/Preprocessor/macro_fn_lparen_scan.c
Modified:
cfe/cfe/trunk/Lex/Lexer.cpp
cfe/cfe/trunk/Lex/MacroExpander.cpp
cfe/cfe/trunk/Lex/Preprocessor.cpp
cfe/cfe/trunk/include/clang/Lex/Lexer.h
cfe/cfe/trunk/include/clang/Lex/MacroExpander.h
cfe/cfe/trunk/include/clang/Lex/Preprocessor.h
Modified: cfe/cfe/trunk/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/Lexer.cpp?rev=38699&r1=38698&r2=38699&view=diff
==============================================================================
--- cfe/cfe/trunk/Lex/Lexer.cpp (original)
+++ cfe/cfe/trunk/Lex/Lexer.cpp Wed Jul 11 11:23:55 2007
@@ -896,16 +896,21 @@
return;
}
- // If we are in a #if directive, emit an error.
- while (!ConditionalStack.empty()) {
- PP.Diag(ConditionalStack.back().IfLoc,
- diag::err_pp_unterminated_conditional);
- ConditionalStack.pop_back();
- }
+ // If we aren't skipping, issue diagnostics. If we are skipping, let the
+ // skipping code do this: there are multiple possible reasons for skipping,
+ // and not all want these diagnostics.
+ if (!PP.isSkipping()) {
+ // If we are in a #if directive, emit an error.
+ while (!ConditionalStack.empty()) {
+ PP.Diag(ConditionalStack.back().IfLoc,
+ diag::err_pp_unterminated_conditional);
+ ConditionalStack.pop_back();
+ }
- // If the file was empty or didn't end in a newline, issue a pedwarn.
- if (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
- Diag(BufferEnd, diag::ext_no_newline_eof);
+ // If the file was empty or didn't end in a newline, issue a pedwarn.
+ if (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
+ Diag(BufferEnd, diag::ext_no_newline_eof);
+ }
BufferPtr = CurPtr;
PP.HandleEndOfFile(Result);
Modified: cfe/cfe/trunk/Lex/MacroExpander.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/MacroExpander.cpp?rev=38699&r1=38698&r2=38699&view=diff
==============================================================================
--- cfe/cfe/trunk/Lex/MacroExpander.cpp (original)
+++ cfe/cfe/trunk/Lex/MacroExpander.cpp Wed Jul 11 11:23:55 2007
@@ -79,18 +79,13 @@
// Otherwise, return a normal token.
}
-/// NextTokenIsKnownNotLParen - If the next token lexed will pop this macro
-/// off the expansion stack, return false and set RanOffEnd to true.
-/// Otherwise, return true if we know for sure that the next token returned
-/// will not be a '(' token. Return false if it is a '(' token or if we are
-/// not sure. This is used when determining whether to expand a function-like
-/// macro.
-bool MacroExpander::NextTokenIsKnownNotLParen(bool &RanOffEnd) const {
+/// isNextTokenLParen - If the next token lexed will pop this macro off the
+/// expansion stack, return 2. If the next unexpanded token is a '(', return
+/// 1, otherwise return 0.
+unsigned MacroExpander::isNextTokenLParen() const {
// Out of tokens?
- if (CurToken == Macro.getNumTokens()) {
- RanOffEnd = true;
- return false;
- }
+ if (CurToken == Macro.getNumTokens())
+ return 2;
- return Macro.getReplacementToken(CurToken).getKind() != tok::l_paren;
+ return Macro.getReplacementToken(CurToken).getKind() == tok::l_paren;
}
Modified: cfe/cfe/trunk/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/Preprocessor.cpp?rev=38699&r1=38698&r2=38699&view=diff
==============================================================================
--- cfe/cfe/trunk/Lex/Preprocessor.cpp (original)
+++ cfe/cfe/trunk/Lex/Preprocessor.cpp Wed Jul 11 11:23:55 2007
@@ -505,52 +505,72 @@
if (*I == II)
return false; // Identifier is a macro argument.
return true;
-}
+}
+
+/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
+/// the specified lexer will return a tok::l_paren token, 0 if it is something
+/// else and 2 if there are no more tokens in the buffer controlled by the
+/// lexer.
+unsigned Preprocessor::isNextPPTokenLParen(Lexer *L) {
+ assert(!isSkipping() && "How can we expand a macro from a skipping buffer?");
+
+ // Set the lexer to 'skipping' mode. This will ensure that we can lex a token
+ // without emitting diagnostics, disables macro expansion, and will cause EOF
+ // to return an EOF token instead of popping the include stack.
+ SkippingContents = true;
+
+ // Save state that can be changed while lexing so that we can restore it.
+ const char *BufferPtr = L->BufferPtr;
+
+ LexerToken Tok;
+ Tok.StartToken();
+ L->LexTokenInternal(Tok);
+
+ // Restore state that may have changed.
+ L->BufferPtr = BufferPtr;
+
+ // Restore the lexer back to non-skipping mode.
+ SkippingContents = false;
+
+ if (Tok.getKind() == tok::eof)
+ return 2;
+ return Tok.getKind() == tok::l_paren;
+}
+
/// isNextPPTokenLParen - Determine whether the next preprocessor token to be
/// lexed is a '('. If so, consume the token and return true, if not, this
/// method should have no observable side-effect on the lexed tokens.
bool Preprocessor::isNextPPTokenLParen() {
- bool RanOffEnd = false;
// Do some quick tests for rejection cases.
- if (CurLexer) {
-#if 0
- if (!CurLexer->NextTokenIsKnownNotLParen(RanOffEnd))
- return false;
-#endif
- } else {
- assert(CurMacroExpander && "No token source?");
- if (CurMacroExpander->NextTokenIsKnownNotLParen(RanOffEnd))
- return false;
- }
-
- // If we ran off the end of the lexer or macro expander, walk the include
- // stack, looking for whatever will return the next token.
- for (unsigned i = IncludeMacroStack.size(); RanOffEnd && i != 0; --i) {
- IncludeStackInfo &Entry = IncludeMacroStack[i-1];
- RanOffEnd = false;
- if (Entry.TheLexer) {
-#if 0
- if (!Entry.TheLexer->NextTokenIsKnownNotLParen(RanOffEnd))
- return false;
-#endif
- } else if (Entry.TheMacroExpander->NextTokenIsKnownNotLParen(RanOffEnd))
- return false;
+ unsigned Val;
+ if (CurLexer)
+ Val = isNextPPTokenLParen(CurLexer);
+ else
+ Val = CurMacroExpander->isNextTokenLParen();
+
+ if (Val == 2) {
+ // If we ran off the end of the lexer or macro expander, walk the include
+ // stack, looking for whatever will return the next token.
+ for (unsigned i = IncludeMacroStack.size(); Val == 2 && i != 0; --i) {
+ IncludeStackInfo &Entry = IncludeMacroStack[i-1];
+ if (Entry.TheLexer)
+ Val = isNextPPTokenLParen(Entry.TheLexer);
+ else
+ Val = Entry.TheMacroExpander->isNextTokenLParen();
+ }
}
- // Okay, if we get here we either know that the next token definitely IS a '('
- // token, or we don't know what it is. In either case we will speculatively
- // read the next token. If it turns out that it isn't a '(', then we create a
- // new macro context with just that token on it so that the token gets
- // reprocessed.
+ // Okay, if we know that the token is a '(', lex it and return. Otherwise we
+ // have found something that isn't a '(' or we found the end of the
+ // translation unit. In either case, return false.
+ if (Val != 1)
+ return false;
LexerToken Tok;
LexUnexpandedToken(Tok);
- if (Tok.getKind() == tok::l_paren)
- return true;
-
- // FIXME: push a fake macro context, push Tok onto it.
- assert(0 && "FIXME: implement speculation failure code!");
+ assert(Tok.getKind() == tok::l_paren && "Error computing l-paren-ness?");
+ return true;
}
/// HandleMacroExpandedIdentifier - If an identifier token is read that is to be
@@ -984,8 +1004,9 @@
// If we are in a #if 0 block skipping tokens, and we see the end of the file,
// this is an error condition. Just return the EOF token up to
- // SkipExcludedConditionalBlock. The Lexer will have already have issued
- // errors for the unterminated #if's on the conditional stack.
+ // SkipExcludedConditionalBlock. The code that enabled skipping will issue
+ // errors for the unterminated #if's on the conditional stack if it is
+ // interested.
if (isSkipping()) {
Result.StartToken();
CurLexer->BufferPtr = CurLexer->BufferEnd;
@@ -1165,10 +1186,19 @@
while (1) {
CurLexer->Lex(Tok);
- // If this is the end of the buffer, we have an error. The lexer will have
- // already handled this error condition, so just return and let the caller
- // lex after this #include.
- if (Tok.getKind() == tok::eof) break;
+ // If this is the end of the buffer, we have an error.
+ if (Tok.getKind() == tok::eof) {
+ // Emit errors for each unterminated conditional on the stack, including
+ // the current one.
+ while (!CurLexer->ConditionalStack.empty()) {
+ Diag(CurLexer->ConditionalStack.back().IfLoc,
+ diag::err_pp_unterminated_conditional);
+ CurLexer->ConditionalStack.pop_back();
+ }
+
+ // Just return and let the caller lex after this #include.
+ break;
+ }
// If this token is not a preprocessor directive, just skip it.
if (Tok.getKind() != tok::hash || !Tok.isAtStartOfLine())
Modified: cfe/cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/Lexer.h?rev=38699&r1=38698&r2=38699&view=diff
==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/Lexer.h Wed Jul 11 11:23:55 2007
@@ -62,12 +62,14 @@
bool IsMainFile; // True if top-level file.
// Context-specific lexing flags.
- bool IsAtStartOfLine; // True if sitting at start of line.
bool ParsingPreprocessorDirective; // True if parsing #XXX
bool ParsingFilename; // True after #include: turn <xx> into string.
- // Context that changes as the file is lexed.
-
+ // Context that changes as the file is lexed. NOTE: any state that mutates as
+ // the file is lexed should be added to Preprocessor::isNextPPTokenLParen.
+
+ bool IsAtStartOfLine; // True if sitting at start of line.
+
/// MIOpt - This is a state machine that detects the #ifndef-wrapping a file
/// idiom for the multiple-include optimization.
MultipleIncludeOpt MIOpt;
Modified: cfe/cfe/trunk/include/clang/Lex/MacroExpander.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/MacroExpander.h?rev=38699&r1=38698&r2=38699&view=diff
==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/MacroExpander.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/MacroExpander.h Wed Jul 11 11:23:55 2007
@@ -82,13 +82,10 @@
delete FormalArgs;
}
- /// NextTokenIsKnownNotLParen - If the next token lexed will pop this macro
- /// off the expansion stack, return false and set RanOffEnd to true.
- /// Otherwise, return true if we know for sure that the next token returned
- /// will not be a '(' token. Return false if it is a '(' token or if we are
- /// not sure. This is used when determining whether to expand a function-like
- /// macro.
- bool NextTokenIsKnownNotLParen(bool &RanOffEnd) const;
+ /// isNextTokenLParen - If the next token lexed will pop this macro off the
+ /// expansion stack, return 2. If the next unexpanded token is a '(', return
+ /// 1, otherwise return 0.
+ unsigned isNextTokenLParen() const;
MacroInfo &getMacro() const { return Macro; }
Modified: cfe/cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=38699&r1=38698&r2=38699&view=diff
==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/Preprocessor.h Wed Jul 11 11:23:55 2007
@@ -467,6 +467,12 @@
/// the macro should not be expanded return true, otherwise return false.
bool HandleMacroExpandedIdentifier(LexerToken &Tok, MacroInfo *MI);
+ /// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
+ /// the specified lexer will return a tok::l_paren token, 0 if it is something
+ /// else and 2 if there are no more tokens in the buffer controlled by the
+ /// lexer.
+ unsigned isNextPPTokenLParen(Lexer *L);
+
/// isNextPPTokenLParen - Determine whether the next preprocessor token to be
/// lexed is a '('. If so, consume the token and return true, if not, this
/// method should have no observable side-effect on the lexed tokens.
More information about the cfe-commits
mailing list