[cfe-commits] r38724 - in /cfe/cfe/trunk: Lex/MacroExpander.cpp Lex/Preprocessor.cpp include/clang/Basic/SourceManager.h include/clang/Lex/MacroExpander.h include/clang/Lex/Preprocessor.h

sabre at cs.uiuc.edu sabre at cs.uiuc.edu
Wed Jul 11 09:24:07 PDT 2007


Author: sabre
Date: Wed Jul 11 11:24:06 2007
New Revision: 38724

URL: http://llvm.org/viewvc/llvm-project?rev=38724&view=rev
Log:
Implement support for lexing from a pre-constructed token stream.

Use this support to implement function-like macro argument preexpansion.

This implements test/Preprocessor/macro_fn_preexpand.c

Modified:
    cfe/cfe/trunk/Lex/MacroExpander.cpp
    cfe/cfe/trunk/Lex/Preprocessor.cpp
    cfe/cfe/trunk/include/clang/Basic/SourceManager.h
    cfe/cfe/trunk/include/clang/Lex/MacroExpander.h
    cfe/cfe/trunk/include/clang/Lex/Preprocessor.h

Modified: cfe/cfe/trunk/Lex/MacroExpander.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/MacroExpander.cpp?rev=38724&r1=38723&r2=38724&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/MacroExpander.cpp (original)
+++ cfe/cfe/trunk/Lex/MacroExpander.cpp Wed Jul 11 11:24:06 2007
@@ -70,6 +70,40 @@
   return false;
 }
 
+/// getPreExpArgument - Return the pre-expanded form of the specified
+/// argument.
+const std::vector<LexerToken> &
+MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
+  assert(Arg < UnexpArgTokens.size() && "Invalid argument number!");
+  
+  // If we have already computed this, return it.
+  if (PreExpArgTokens.empty())
+    PreExpArgTokens.resize(UnexpArgTokens.size());
+
+  std::vector<LexerToken> &Result = PreExpArgTokens[Arg];
+  if (!Result.empty()) return Result;
+
+  // Otherwise, we have to pre-expand this argument, populating Result.  To do
+  // this, we set up a fake MacroExpander to lex from the unexpanded argument
+  // list.  With this installed, we lex expanded tokens until we hit the EOF
+  // token at the end of the unexp list.
+  PP.EnterTokenStream(UnexpArgTokens[Arg]);
+
+  // Lex all of the macro-expanded tokens into Result.
+  do {
+    Result.push_back(LexerToken());
+    PP.Lex(Result.back());
+  } while (Result.back().getKind() != tok::eof);
+  
+  // Pop the token stream off the top of the stack.  We know that the internal
+  // pointer inside of it is to the "end" of the token stream, but the stack
+  // will not otherwise be popped until the next token is lexed.  The problem is
+  // that the token may be lexed sometime after the vector of tokens itself is
+  // destroyed, which would be badness.
+  PP.RemoveTopOfLexerStack();
+  return Result;
+}
+
 
 /// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
 /// tokens into the literal string token that should be produced by the C #
@@ -83,9 +117,9 @@
 
   // Stringify all the tokens.
   std::string Result = "\"";
+  // FIXME: Optimize this loop to not use std::strings.
   for (unsigned i = 0, e = Toks.size()-1 /*no eof*/; i != e; ++i) {
     const LexerToken &Tok = Toks[i];
-    // FIXME: Optimize this.
     if (i != 0 && Tok.hasLeadingSpace())
       Result += ' ';
     
@@ -163,25 +197,49 @@
 // MacroExpander Implementation
 //===----------------------------------------------------------------------===//
 
+/// Create a macro expander for the specified macro with the specified actual
+/// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
 MacroExpander::MacroExpander(LexerToken &Tok, MacroArgs *Actuals,
                              Preprocessor &pp)
-  : Macro(*Tok.getIdentifierInfo()->getMacroInfo()),
+  : Macro(Tok.getIdentifierInfo()->getMacroInfo()),
     ActualArgs(Actuals), PP(pp), CurToken(0),
     InstantiateLoc(Tok.getLocation()),
     AtStartOfLine(Tok.isAtStartOfLine()),
     HasLeadingSpace(Tok.hasLeadingSpace()) {
-  MacroTokens = &Macro.getReplacementTokens();
+  MacroTokens = &Macro->getReplacementTokens();
 
   // If this is a function-like macro, expand the arguments and change
   // MacroTokens to point to the expanded tokens.
-  if (Macro.isFunctionLike() && Macro.getNumArgs())
+  if (Macro->isFunctionLike() && Macro->getNumArgs())
     ExpandFunctionArguments();
+  
+  // Mark the macro as currently disabled, so that it is not recursively
+  // expanded.  The macro must be disabled only after argument pre-expansion of
+  // function-like macro arguments occurs.
+  Macro->DisableMacro();
+}
+
+/// Create a macro expander for the specified token stream.  This does not
+/// take ownership of the specified token vector.
+MacroExpander::MacroExpander(const std::vector<LexerToken> &TokStream, 
+                             Preprocessor &pp)
+  : Macro(0), ActualArgs(0), PP(pp), MacroTokens(&TokStream), CurToken(0),
+    InstantiateLoc(SourceLocation()), AtStartOfLine(false), 
+    HasLeadingSpace(false) {
+      
+  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
+  // returned unmodified.
+  if (!TokStream.empty()) {
+    AtStartOfLine   = TokStream[0].isAtStartOfLine();
+    HasLeadingSpace = TokStream[0].hasLeadingSpace();
+  }
 }
 
+
 MacroExpander::~MacroExpander() {
   // If this was a function-like macro that actually uses its arguments, delete
   // the expanded tokens.
-  if (MacroTokens != &Macro.getReplacementTokens())
+  if (Macro && MacroTokens != &Macro->getReplacementTokens())
     delete MacroTokens;
   
   // MacroExpander owns its formal arguments.
@@ -205,7 +263,7 @@
     // when the #define was parsed.
     const LexerToken &CurTok = (*MacroTokens)[i];
     if (CurTok.getKind() == tok::hash || CurTok.getKind() == tok::hashat) {
-      int ArgNo = Macro.getArgumentNum((*MacroTokens)[i+1].getIdentifierInfo());
+      int ArgNo =Macro->getArgumentNum((*MacroTokens)[i+1].getIdentifierInfo());
       assert(ArgNo != -1 && "Token following # is not an argument?");
       
       if (CurTok.getKind() == tok::hash)  // Stringify
@@ -227,7 +285,7 @@
       // Otherwise, if this is not an argument token, just add the token to the
       // output buffer.
       IdentifierInfo *II = CurTok.getIdentifierInfo();
-      int ArgNo = II ? Macro.getArgumentNum(II) : -1;
+      int ArgNo = II ? Macro->getArgumentNum(II) : -1;
       if (ArgNo == -1) {
         ResultToks.push_back(CurTok);
         continue;
@@ -251,14 +309,10 @@
         const std::vector<LexerToken> *ArgToks;
         // Only preexpand the argument if it could possibly need it.  This
         // avoids some work in common cases.
-        if (ActualArgs->ArgNeedsPreexpansion(ArgNo)) {
-          // FIXME: WRONG
-          ArgToks = &ActualArgs->getUnexpArgument(ArgNo);
-        } else {
-          // If we don't need to pre-expand the argument, just substitute in the
-          // unexpanded tokens.
+        if (ActualArgs->ArgNeedsPreexpansion(ArgNo))
+          ArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP);
+        else
           ArgToks = &ActualArgs->getUnexpArgument(ArgNo);
-        }
         
         unsigned FirstTok = ResultToks.size();
         ResultToks.insert(ResultToks.end(), ArgToks->begin(), ArgToks->end()-1);
@@ -272,7 +326,12 @@
         continue;
       }
       
-      // FIXME: handle pasted args.      
+      // Okay, we have a token that is either the LHS or RHS of a paste (##)
+      // argument.
+      
+      // FIXME: Handle comma swallowing GNU extension.
+      
+      // FIXME: handle pasted args.  Handle 'placemarker' stuff.
       ResultToks.push_back(CurTok);
     }
   }
@@ -290,8 +349,15 @@
 ///
 void MacroExpander::Lex(LexerToken &Tok) {
   // Lexing off the end of the macro, pop this macro off the expansion stack.
-  if (isAtEnd())
+  if (isAtEnd()) {
+    // If this is a macro (not a token stream), mark the macro enabled now
+    // that it is no longer being expanded.
+    if (Macro) Macro->EnableMacro();
+
+    // Pop this context off the preprocessors lexer stack and get the next
+    // token.
     return PP.HandleEndOfMacro(Tok);
+  }
   
   // Get the next token to return.
   Tok = (*MacroTokens)[CurToken++];
@@ -301,9 +367,15 @@
   // diagnostics for the expanded token should appear as if they came from
   // InstantiationLoc.  Pull this information together into a new SourceLocation
   // that captures all of this.
-  Tok.SetLocation(PP.getSourceManager().getInstantiationLoc(Tok.getLocation(),
-                                                            InstantiateLoc));
-
+  if (InstantiateLoc.isValid()) {   // Don't do this for token streams.
+    SourceManager &SrcMgr = PP.getSourceManager();
+    // The token could have come from a prior macro expansion.  In that case,
+    // ignore the macro expand part to get to the physloc.  This happens for
+    // stuff like:  #define A(X) X    A(A(X))    A(1)
+    SourceLocation PhysLoc = SrcMgr.getPhysicalLoc(Tok.getLocation());
+    Tok.SetLocation(SrcMgr.getInstantiationLoc(PhysLoc, InstantiateLoc));
+  }
+  
   // If this is the first token, set the lexical properties of the token to
   // match the lexical properties of the macro identifier.
   if (CurToken == 1) {

Modified: cfe/cfe/trunk/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/Preprocessor.cpp?rev=38724&r1=38723&r2=38724&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/Preprocessor.cpp (original)
+++ cfe/cfe/trunk/Lex/Preprocessor.cpp Wed Jul 11 11:24:06 2007
@@ -450,12 +450,38 @@
   CurLexer     = 0;
   CurDirLookup = 0;
   
-  // Mark the macro as currently disabled, so that it is not recursively
-  // expanded.
-  MI.DisableMacro();
   CurMacroExpander = new MacroExpander(Tok, Args, *this);
 }
 
+/// EnterTokenStream - Add a "macro" context to the top of the include stack,
+/// which will cause the lexer to start returning the specified tokens.  Note
+/// that these tokens will be re-macro-expanded when/if expansion is enabled.
+/// This method assumes that the specified stream of tokens has a permanent
+/// owner somewhere, so they do not need to be copied.
+void Preprocessor::EnterTokenStream(const std::vector<LexerToken> &Stream) {
+  // Save our current state.
+  IncludeMacroStack.push_back(IncludeStackInfo(CurLexer, CurDirLookup,
+                                               CurMacroExpander));
+  CurLexer     = 0;
+  CurDirLookup = 0;
+
+  // Create a macro expander to expand from the specified token stream.
+  CurMacroExpander = new MacroExpander(Stream, *this);
+}
+
+/// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
+/// lexer stack.  This should only be used in situations where the current
+/// state of the top-of-stack lexer is known.
+void Preprocessor::RemoveTopOfLexerStack() {
+  assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
+  delete CurLexer;
+  delete CurMacroExpander;
+  CurLexer         = IncludeMacroStack.back().TheLexer;
+  CurDirLookup     = IncludeMacroStack.back().TheDirLookup;
+  CurMacroExpander = IncludeMacroStack.back().TheMacroExpander;
+  IncludeMacroStack.pop_back();
+}
+
 //===----------------------------------------------------------------------===//
 // Macro Expansion Handling.
 //===----------------------------------------------------------------------===//
@@ -1011,11 +1037,7 @@
   // lexing the #includer file.
   if (!IncludeMacroStack.empty()) {
     // We're done with the #included file.
-    delete CurLexer;
-    CurLexer         = IncludeMacroStack.back().TheLexer;
-    CurDirLookup     = IncludeMacroStack.back().TheDirLookup;
-    CurMacroExpander = IncludeMacroStack.back().TheMacroExpander;
-    IncludeMacroStack.pop_back();
+    RemoveTopOfLexerStack();
 
     // Notify the client, if desired, that we are in a new source file.
     if (FileChangeHandler && !isEndOfMacro && CurLexer) {
@@ -1050,13 +1072,11 @@
 }
 
 /// HandleEndOfMacro - This callback is invoked when the lexer hits the end of
-/// the current macro expansion.
+/// the current macro expansion or token stream expansion.
 void Preprocessor::HandleEndOfMacro(LexerToken &Result) {
   assert(CurMacroExpander && !CurLexer &&
          "Ending a macro when currently in a #include file!");
 
-  // Mark macro not ignored now that it is no longer being expanded.
-  CurMacroExpander->getMacro().EnableMacro();
   delete CurMacroExpander;
 
   // Handle this like a #include file being popped off the stack.

Modified: cfe/cfe/trunk/include/clang/Basic/SourceManager.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Basic/SourceManager.h?rev=38724&r1=38723&r2=38724&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Basic/SourceManager.h (original)
+++ cfe/cfe/trunk/include/clang/Basic/SourceManager.h Wed Jul 11 11:24:06 2007
@@ -262,6 +262,19 @@
     return Loc;
   }
   
+  /// getPhysicalLoc - Given a SourceLocation object, return the physical
+  /// location referenced by the ID.
+  SourceLocation getPhysicalLoc(SourceLocation Loc) const {
+    if (Loc.getFileID() == 0) return Loc;
+    
+    // For Macros, the physical loc is specified by the MacroTokenFileID.
+    const SrcMgr::FileIDInfo *FIDInfo = getFIDInfo(Loc.getFileID());
+    if (FIDInfo->IDType == SrcMgr::FileIDInfo::MacroExpansion)
+      return SourceLocation(FIDInfo->u.MacroTokenFileID,
+                            Loc.getRawFilePos());
+    return Loc;
+  }
+  
   /// PrintStats - Print statistics to stderr.
   ///
   void PrintStats() const;

Modified: cfe/cfe/trunk/include/clang/Lex/MacroExpander.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/MacroExpander.h?rev=38724&r1=38723&r2=38724&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/MacroExpander.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/MacroExpander.h Wed Jul 11 11:24:06 2007
@@ -30,9 +30,10 @@
   /// an 'EOF' marker at the end of each argument.
   std::vector<std::vector<LexerToken> > UnexpArgTokens;
 
-  /// ExpArgTokens - Pre-expanded tokens for arguments that need them.  Empty if
-  /// not yet computed.  This includes the EOF marker at the end of the stream.
-  std::vector<std::vector<LexerToken> > ExpArgTokens;
+  /// PreExpArgTokens - Pre-expanded tokens for arguments that need them.  Empty
+  /// if not yet computed.  This includes the EOF marker at the end of the
+  /// stream.
+  std::vector<std::vector<LexerToken> > PreExpArgTokens;
 
   /// StringifiedArgs - This contains arguments in 'stringified' form.  If the
   /// stringified form of an argument has not yet been computed, this is empty.
@@ -57,6 +58,11 @@
     return UnexpArgTokens[Arg];
   }
   
+  /// getPreExpArgument - Return the pre-expanded form of the specified
+  /// argument.
+  const std::vector<LexerToken> &
+    getPreExpArgument(unsigned Arg, Preprocessor &PP);  
+  
   /// getStringifiedArgument - Compute, cache, and return the specified argument
   /// that has been 'stringified' as required by the # operator.
   const LexerToken &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP);
@@ -68,12 +74,13 @@
 
   
 /// MacroExpander - This implements a lexer that returns token from a macro body
-/// instead of lexing from a character buffer.
+/// or token stream instead of lexing from a character buffer.
 ///
 class MacroExpander {
-  /// Macro - The macro we are expanding from.
+  /// Macro - The macro we are expanding from.  This is null if expanding a
+  /// token stream.
   ///
-  MacroInfo &Macro;
+  MacroInfo *Macro;
 
   /// ActualArgs - The actual arguments specified for a function-like macro, or
   /// null.  The MacroExpander owns the pointed-to object.
@@ -84,7 +91,8 @@
   Preprocessor &PP;
 
   /// MacroTokens - This is the pointer to the list of tokens that the macro is
-  /// defined to, with arguments expanded for function-like macros.
+  /// defined to, with arguments expanded for function-like macros.  If this is
+  /// a token stream, this are the tokens we are returning.
   const std::vector<LexerToken> *MacroTokens;
   
   /// CurToken - This is the next token that Lex will return.
@@ -102,9 +110,13 @@
   MacroExpander(const MacroExpander&);  // DO NOT IMPLEMENT
   void operator=(const MacroExpander&); // DO NOT IMPLEMENT
 public:
-  /// Create a macro expander of the specified macro with the specified actual
+  /// Create a macro expander for the specified macro with the specified actual
   /// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
   MacroExpander(LexerToken &Tok, MacroArgs *ActualArgs, Preprocessor &PP);
+  
+  /// Create a macro expander for the specified token stream.  This does not
+  /// take ownership of the specified token vector.
+  MacroExpander(const std::vector<LexerToken> &TokStream, Preprocessor &PP);
   ~MacroExpander();
   
   /// isNextTokenLParen - If the next token lexed will pop this macro off the
@@ -112,8 +124,6 @@
   /// 1, otherwise return 0.
   unsigned isNextTokenLParen() const;
   
-  MacroInfo &getMacro() const { return Macro; }
-
   /// Lex - Lex and return a token from this macro stream.
   void Lex(LexerToken &Tok);
   

Modified: cfe/cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=38724&r1=38723&r2=38724&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/Preprocessor.h Wed Jul 11 11:24:06 2007
@@ -325,6 +325,17 @@
   /// tokens input to a function-like macro.
   void EnterMacro(LexerToken &Identifier, MacroArgs *Args);
   
+  /// EnterTokenStream - Add a "macro" context to the top of the include stack,
+  /// which will cause the lexer to start returning the specified tokens.  Note
+  /// that these tokens will be re-macro-expanded when/if expansion is enabled.
+  /// This method assumes that the specified stream of tokens has a permanent
+  /// owner somewhere, so they do not need to be copied.
+  void EnterTokenStream(const std::vector<LexerToken> &Stream);
+  
+  /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
+  /// lexer stack.  This should only be used in situations where the current
+  /// state of the top-of-stack lexer is known.
+  void RemoveTopOfLexerStack();
   
   /// Lex - To lex a token from the preprocessor, just pull a token from the
   /// current lexer or macro object.





More information about the cfe-commits mailing list