[cfe-commits] r38551 - in /cfe/cfe/trunk: Basic/SourceManager.cpp Driver/clang.cpp Lex/Lexer.cpp Lex/MacroExpander.cpp Lex/MacroInfo.cpp Lex/PPExpressions.cpp Lex/Preprocessor.cpp include/clang/Basic/SourceManager.h include/clang/Lex/Lexer.h include/clang/Lex/MacroExpander.h include/clang/Lex/MacroInfo.h include/clang/Lex/Preprocessor.h

sabre at cs.uiuc.edu sabre at cs.uiuc.edu
Wed Jul 11 09:22:30 PDT 2007


Author: sabre
Date: Wed Jul 11 11:22:30 2007
New Revision: 38551

URL: http://llvm.org/viewvc/llvm-project?rev=38551&view=rev
Log:
Make a fundamental change to the way we represent the location of LexerToken's.
Now, instead of keeping a pointer to the start of the token in memory, we keep the
start of the token as a SourceLocation node.  This means that each LexerToken knows
the full include stack it was created with, and means that the LexerToken isn't
reliant on a "CurLexer" member to be around (lexer tokens would previously go out of
scope when their lexers were deallocated).

This simplifies several things, and forces good cleanup elsewhere.  Now the
Preprocessor is the one that knows how to dump tokens/macros and is the one that
knows how to get the spelling of a token (it has all the context).

Modified:
    cfe/cfe/trunk/Basic/SourceManager.cpp
    cfe/cfe/trunk/Driver/clang.cpp
    cfe/cfe/trunk/Lex/Lexer.cpp
    cfe/cfe/trunk/Lex/MacroExpander.cpp
    cfe/cfe/trunk/Lex/MacroInfo.cpp
    cfe/cfe/trunk/Lex/PPExpressions.cpp
    cfe/cfe/trunk/Lex/Preprocessor.cpp
    cfe/cfe/trunk/include/clang/Basic/SourceManager.h
    cfe/cfe/trunk/include/clang/Lex/Lexer.h
    cfe/cfe/trunk/include/clang/Lex/MacroExpander.h
    cfe/cfe/trunk/include/clang/Lex/MacroInfo.h
    cfe/cfe/trunk/include/clang/Lex/Preprocessor.h

Modified: cfe/cfe/trunk/Basic/SourceManager.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Basic/SourceManager.cpp?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/Basic/SourceManager.cpp (original)
+++ cfe/cfe/trunk/Basic/SourceManager.cpp Wed Jul 11 11:22:30 2007
@@ -109,6 +109,16 @@
   return Result;
 }
 
+/// getCharacterData - Return a pointer to the start of the specified location
+/// in the appropriate SourceBuffer.  This returns null if it cannot be
+/// computed (e.g. invalid SourceLocation).
+const char *SourceManager::getCharacterData(SourceLocation SL) const {
+  if (unsigned FileID = SL.getFileID())
+    return getFileInfo(FileID)->Buffer->getBufferStart() + getFilePos(SL);
+  return 0;
+}
+
+
 /// getColumnNumber - Return the column # for the specified include position.
 /// this is significantly cheaper to compute than the line number.  This returns
 /// zero if the column number isn't known.

Modified: cfe/cfe/trunk/Driver/clang.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Driver/clang.cpp?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/Driver/clang.cpp (original)
+++ cfe/cfe/trunk/Driver/clang.cpp Wed Jul 11 11:22:30 2007
@@ -627,12 +627,11 @@
     isFirstToken = false;    
     
     if (Tok.getLength() < 256) {
-      unsigned Len = Lexer::getSpelling(Tok, Buffer, PP.getLangOptions());
+      unsigned Len = PP.getSpelling(Tok, Buffer);
       Buffer[Len] = 0;
       std::cout << Buffer;
     } else {
-      std::string S = Lexer::getSpelling(Tok, PP.getLangOptions());
-      std::cout << S;
+      std::cout << PP.getSpelling(Tok);
     }
   } while (Tok.getKind() != tok::eof);
   std::cout << "\n";
@@ -645,8 +644,6 @@
 static cl::opt<std::string>
 InputFilename(cl::Positional, cl::desc("<input file>"), cl::init("-"));
 
-void PrintIdentStats();
-
 int main(int argc, char **argv) {
   cl::ParseCommandLineOptions(argc, argv, " llvm cfe\n");
   sys::PrintStackTraceOnErrorSignal();
@@ -751,7 +748,7 @@
     LexerToken Tok;
     do {
       PP.Lex(Tok);
-      Tok.dump(Options, true);
+      PP.DumpToken(Tok, true);
       std::cerr << "\n";
     } while (Tok.getKind() != tok::eof);
     break;

Modified: cfe/cfe/trunk/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/Lexer.cpp?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/Lexer.cpp (original)
+++ cfe/cfe/trunk/Lex/Lexer.cpp Wed Jul 11 11:22:30 2007
@@ -32,7 +32,6 @@
 #include "clang/Basic/SourceBuffer.h"
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/Config/alloca.h"
-#include <cassert>
 #include <cctype>
 #include <iostream>
 using namespace llvm;
@@ -64,37 +63,6 @@
 // LexerToken implementation.
 //===----------------------------------------------------------------------===//
 
-/// getSourceLocation - Return a source location identifier for the specified
-/// offset in the current file.
-SourceLocation LexerToken::getSourceLocation() const {
-  if (TheLexer)
-    return TheLexer->getSourceLocation(Start);
-  return SourceLocation();
-}
-
-
-/// dump - Print the token to stderr, used for debugging.
-///
-void LexerToken::dump(const LangOptions &Features, bool DumpFlags) const {
-  std::cerr << clang::tok::getTokenName(Kind) << " '";
-  
-  if (needsCleaning())
-    std::cerr << Lexer::getSpelling(*this, Features);
-  else
-    std::cerr << std::string(getStart(), getEnd());
-  std::cerr << "'";
-  
-  if (DumpFlags) {
-    std::cerr << "\t";
-    if (isAtStartOfLine())
-      std::cerr << " [StartOfLine]";
-    if (hasLeadingSpace())
-      std::cerr << " [LeadingSpace]";
-    if (needsCleaning())
-      std::cerr << " [Spelling='" << std::string(getStart(), getEnd()) << "']";
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Character information.
 //===----------------------------------------------------------------------===//
@@ -153,6 +121,7 @@
   return CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD);
 }
 
+
 //===----------------------------------------------------------------------===//
 // Diagnostics forwarding code.
 //===----------------------------------------------------------------------===//
@@ -225,8 +194,8 @@
 /// know that we can accumulate into Size, and that we have already incremented
 /// Ptr by Size bytes.
 ///
-/// When this method is updated, getCharAndSizeSlowNoWarn (below) should be
-/// updated to match.
+/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
+/// be updated to match.
 ///
 char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
                                LexerToken *Tok) {
@@ -289,13 +258,14 @@
   return *Ptr;
 }
 
+
 /// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
 /// getCharAndSizeNoWarn method.  Here we know that we can accumulate into Size,
 /// and that we have already incremented Ptr by Size bytes.
 ///
-/// When this method is updated, getCharAndSizeSlow (above) should be updated to
-/// match.
-static char getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
+/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
+/// be updated to match.
+char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
                                      const LangOptions &Features) {
   // If we have a slash, look for an escaped newline.
   if (Ptr[0] == '\\') {
@@ -348,80 +318,6 @@
   return *Ptr;
 }
 
-/// getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever
-/// emit a warning.
-static inline char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size,
-                                        const LangOptions &Features) {
-  // If this is not a trigraph and not a UCN or escaped newline, return
-  // quickly.
-  if (Ptr[0] != '?' && Ptr[0] != '\\') {
-    Size = 1;
-    return *Ptr;
-  }
-  
-  Size = 0;
-  return getCharAndSizeSlowNoWarn(Ptr, Size, Features);
-}
-
-
-/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
-/// token are the characters used to represent the token in the source file
-/// after trigraph expansion and escaped-newline folding.  In particular, this
-/// wants to get the true, uncanonicalized, spelling of things like digraphs
-/// UCNs, etc.
-std::string Lexer::getSpelling(const LexerToken &Tok,
-                               const LangOptions &Features) {
-  assert(Tok.getStart() <= Tok.getEnd() && "Token character range is bogus!");
-  
-  // If this token contains nothing interesting, return it directly.
-  if (!Tok.needsCleaning())
-    return std::string(Tok.getStart(), Tok.getEnd());
-  
-  // Otherwise, hard case, relex the characters into the string.
-  std::string Result;
-  Result.reserve(Tok.getLength());
-  
-  for (const char *Ptr = Tok.getStart(), *End = Tok.getEnd(); Ptr != End; ) {
-    unsigned CharSize;
-    Result.push_back(getCharAndSizeNoWarn(Ptr, CharSize, Features));
-    Ptr += CharSize;
-  }
-  assert(Result.size() != unsigned(Tok.getLength()) &&
-         "NeedsCleaning flag set on something that didn't need cleaning!");
-  return Result;
-}
-
-/// getSpelling - This method is used to get the spelling of a token into a
-/// preallocated buffer, instead of as an std::string.  The caller is required
-/// to allocate enough space for the token, which is guaranteed to be at most
-/// Tok.End-Tok.Start bytes long.  The actual length of the token is returned.
-unsigned Lexer::getSpelling(const LexerToken &Tok, char *Buffer,
-                            const LangOptions &Features) {
-  assert(Tok.getStart() <= Tok.getEnd() && "Token character range is bogus!");
-
-  // If this token contains nothing interesting, return it directly.
-  if (!Tok.needsCleaning()) {
-    unsigned Size = Tok.getLength();
-    memcpy(Buffer, Tok.getStart(), Size);
-    return Size;
-  }
-  // Otherwise, hard case, relex the characters into the string.
-  std::string Result;
-  Result.reserve(Tok.getLength());
-  
-  char *OutBuf = Buffer;
-  for (const char *Ptr = Tok.getStart(), *End = Tok.getEnd(); Ptr != End; ) {
-    unsigned CharSize;
-    *OutBuf++ = getCharAndSizeNoWarn(Ptr, CharSize, Features);
-    Ptr += CharSize;
-  }
-  assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
-         "NeedsCleaning flag set on something that didn't need cleaning!");
-  
-  return OutBuf-Buffer;
-}
-
-
 //===----------------------------------------------------------------------===//
 // Helper methods for lexing.
 //===----------------------------------------------------------------------===//
@@ -440,19 +336,20 @@
   // FIXME: universal chars.
   if (C != '\\' && C != '?' && (C != '$' || !Features.DollarIdents)) {
 FinishIdentifier:
-    Result.SetEnd(BufferPtr = CurPtr);
+    const char *IdStart = BufferPtr, *IdEnd = CurPtr;
+    FormTokenWithChars(Result, CurPtr);
     Result.SetKind(tok::identifier);
     
     // Look up this token, see if it is a macro, or if it is a language keyword.
     const char *SpelledTokStart, *SpelledTokEnd;
     if (!Result.needsCleaning()) {
       // No cleaning needed, just use the characters from the lexed buffer.
-      SpelledTokStart = Result.getStart();
-      SpelledTokEnd   = Result.getEnd();
+      SpelledTokStart = IdStart;
+      SpelledTokEnd   = IdEnd;
     } else {
       // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
       char *TmpBuf = (char*)alloca(Result.getLength());
-      unsigned Size = getSpelling(Result, TmpBuf);
+      unsigned Size = PP.getSpelling(Result, TmpBuf);
       SpelledTokStart = TmpBuf;
       SpelledTokEnd = TmpBuf+Size;
     }
@@ -516,8 +413,8 @@
   
   Result.SetKind(tok::numeric_constant);
 
-  // Update the end of token position as well as the BufferPtr instance var.
-  Result.SetEnd(BufferPtr = CurPtr);
+  // Update the location of token as well as BufferPtr.
+  FormTokenWithChars(Result, CurPtr);
 }
 
 /// LexStringLiteral - Lex the remainder of a string literal, after having lexed
@@ -533,7 +430,7 @@
       C = getAndAdvanceChar(CurPtr, Result);
     } else if (C == '\n' || C == '\r' ||             // Newline.
                (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
-      Diag(Result.getStart(), diag::err_unterminated_string);
+      Diag(BufferPtr, diag::err_unterminated_string);
       BufferPtr = CurPtr-1;
       return LexTokenInternal(Result);
     } else if (C == 0) {
@@ -546,8 +443,8 @@
 
   Result.SetKind(tok::string_literal);
 
-  // Update the end of token position as well as the BufferPtr instance var.
-  Result.SetEnd(BufferPtr = CurPtr);
+  // Update the location of the token as well as the BufferPtr instance var.
+  FormTokenWithChars(Result, CurPtr);
 }
 
 /// LexAngledStringLiteral - Lex the remainder of an angled string literal,
@@ -563,7 +460,7 @@
       C = getAndAdvanceChar(CurPtr, Result);
     } else if (C == '\n' || C == '\r' ||             // Newline.
                (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
-      Diag(Result.getStart(), diag::err_unterminated_string);
+      Diag(BufferPtr, diag::err_unterminated_string);
       BufferPtr = CurPtr-1;
       return LexTokenInternal(Result);
     } else if (C == 0) {
@@ -576,8 +473,8 @@
   
   Result.SetKind(tok::angle_string_literal);
   
-  // Update the end of token position as well as the BufferPtr instance var.
-  Result.SetEnd(BufferPtr = CurPtr);
+  // Update the location of token as well as BufferPtr.
+  FormTokenWithChars(Result, CurPtr);
 }
 
 
@@ -589,7 +486,7 @@
   // Handle the common case of 'x' and '\y' efficiently.
   char C = getAndAdvanceChar(CurPtr, Result);
   if (C == '\'') {
-    Diag(Result.getStart(), diag::err_empty_character);
+    Diag(BufferPtr, diag::err_empty_character);
     BufferPtr = CurPtr;
     return LexTokenInternal(Result);
   } else if (C == '\\') {
@@ -609,7 +506,7 @@
         C = getAndAdvanceChar(CurPtr, Result);
       } else if (C == '\n' || C == '\r' ||               // Newline.
                  (C == 0 && CurPtr-1 == BufferEnd)) {    // End of file.
-        Diag(Result.getStart(), diag::err_unterminated_char);
+        Diag(BufferPtr, diag::err_unterminated_char);
         BufferPtr = CurPtr-1;
         return LexTokenInternal(Result);
       } else if (C == 0) {
@@ -623,8 +520,8 @@
 
   Result.SetKind(tok::char_constant);
   
-  // Update the end of token position as well as the BufferPtr instance var.
-  Result.SetEnd(BufferPtr = CurPtr);
+  // Update the location of token as well as BufferPtr.
+  FormTokenWithChars(Result, CurPtr);
 }
 
 /// SkipWhitespace - Efficiently skip over a series of whitespace characters.
@@ -663,11 +560,11 @@
   // If the next token is obviously a // or /* */ comment, skip it efficiently
   // too (without going through the big switch stmt).
   if (Char == '/' && CurPtr[1] == '/') {
-    Result.SetStart(CurPtr);
+    BufferPtr = CurPtr;
     return SkipBCPLComment(Result, CurPtr+1);
   }
   if (Char == '/' && CurPtr[1] == '*') {
-    Result.SetStart(CurPtr);
+    BufferPtr = CurPtr;
     return SkipBlockComment(Result, CurPtr+2);
   }
   BufferPtr = CurPtr;
@@ -680,7 +577,7 @@
   // If BCPL comments aren't explicitly enabled for this language, emit an
   // extension warning.
   if (!Features.BCPLComment) {
-    Diag(Result.getStart(), diag::ext_bcpl_comment);
+    Diag(BufferPtr, diag::ext_bcpl_comment);
     
     // Mark them enabled so we only emit one warning for this translation
     // unit.
@@ -830,7 +727,7 @@
   // comments.
   unsigned char C = *CurPtr++;
   if (C == 0 && CurPtr == BufferEnd+1) {
-    Diag(Result.getStart(), diag::err_unterminated_block_comment);
+    Diag(BufferPtr, diag::err_unterminated_block_comment);
     BufferPtr = CurPtr-1;
     return;
   }
@@ -860,7 +757,7 @@
         Diag(CurPtr-1, diag::nested_block_comment);
       }
     } else if (C == 0 && CurPtr == BufferEnd+1) {
-      Diag(Result.getStart(), diag::err_unterminated_block_comment);
+      Diag(BufferPtr, diag::err_unterminated_block_comment);
       // Note: the user probably forgot a */.  We could continue immediately
       // after the /*, but this would involve lexing a lot of what really is the
       // comment, which surely would confuse the parser.
@@ -906,20 +803,21 @@
 
   // No filename?
   if (Result.getKind() == tok::eom) {
-    Diag(Result.getStart(), diag::err_pp_expects_filename);
+    PP.Diag(Result, diag::err_pp_expects_filename);
     return;
   }
   
-  // Verify that there is nothing after the filename, other than EOM.
+  // Verify that there is nothing after the filename, other than EOM.  Use the
+  // preprocessor to lex this in case lexing the filename entered a macro.
   LexerToken EndTok;
-  Lex(EndTok);
+  PP.Lex(EndTok);
 
   if (EndTok.getKind() != tok::eom) {
-    Diag(Result.getStart(), diag::err_pp_expects_filename);
+    PP.Diag(EndTok, diag::ext_pp_extra_tokens_at_eol, "#include");
     
     // Lex until the end of the preprocessor directive line.
     while (EndTok.getKind() != tok::eom)
-      Lex(EndTok);
+      PP.Lex(EndTok);
     
     Result.SetKind(tok::eom);
   }
@@ -935,8 +833,6 @@
 
   // CurPtr - Cache BufferPtr in an automatic variable.
   const char *CurPtr = BufferPtr;
-  Tmp.SetStart(CurPtr);
-
   while (1) {
     char Char = getAndAdvanceChar(CurPtr, Tmp);
     switch (Char) {
@@ -977,14 +873,15 @@
     // Done parsing the "line".
     ParsingPreprocessorDirective = false;
     Result.SetKind(tok::eom);
-    // Update the end of token position as well as the BufferPtr instance var.
-    Result.SetEnd(BufferPtr = CurPtr);
+    // Update the location of token as well as BufferPtr.
+    FormTokenWithChars(Result, CurPtr);
     return;
   }        
 
   // If we are in a #if directive, emit an error.
   while (!ConditionalStack.empty()) {
-    Diag(ConditionalStack.back().IfLoc, diag::err_pp_unterminated_conditional);
+    PP.Diag(ConditionalStack.back().IfLoc,
+            diag::err_pp_unterminated_conditional);
     ConditionalStack.pop_back();
   }  
   
@@ -1011,7 +908,6 @@
   
   // CurPtr - Cache BufferPtr in an automatic variable.
   const char *CurPtr = BufferPtr;
-  Result.SetStart(CurPtr);
 
   unsigned SizeTmp, SizeTmp2;   // Temporaries for use in cases below.
   
@@ -1281,7 +1177,7 @@
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else if (Features.CPPMinMax && Char == '?') {     // <?
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
-      Diag(Result.getStart(), diag::min_max_deprecated);
+      Diag(BufferPtr, diag::min_max_deprecated);
 
       if (getCharAndSize(CurPtr, SizeTmp) == '=') {     // <?= 
         Result.SetKind(tok::lessquestionequal);
@@ -1308,7 +1204,7 @@
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else if (Features.CPPMinMax && Char == '?') {
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
-      Diag(Result.getStart(), diag::min_max_deprecated);
+      Diag(BufferPtr, diag::min_max_deprecated);
 
       if (getCharAndSize(CurPtr, SizeTmp) == '=') {
         Result.SetKind(tok::greaterquestionequal);    // >?=
@@ -1418,6 +1314,6 @@
     goto LexNextToken;   // GCC isn't tail call eliminating.
   }
   
-  // Update the end of token position as well as the BufferPtr instance var.
-  Result.SetEnd(BufferPtr = CurPtr);
+  // Update the location of token as well as BufferPtr.
+  FormTokenWithChars(Result, CurPtr);
 }

Modified: cfe/cfe/trunk/Lex/MacroExpander.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/MacroExpander.cpp?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/MacroExpander.cpp (original)
+++ cfe/cfe/trunk/Lex/MacroExpander.cpp Wed Jul 11 11:22:30 2007
@@ -17,7 +17,18 @@
 using namespace llvm;
 using namespace clang;
 
+MacroExpander::MacroExpander(LexerToken &Tok, Preprocessor &pp)
+  : Macro(*Tok.getIdentifierInfo()->getMacroInfo()), PP(pp), CurToken(0),
+    InstantiateLoc(Tok.getSourceLocation()),
+    AtStartOfLine(Tok.isAtStartOfLine()),
+    HasLeadingSpace(Tok.hasLeadingSpace()) {
+}
+
+
+
+
 /// Lex - Lex and return a token from this macro stream.
+///
 void MacroExpander::Lex(LexerToken &Tok) {
   // Lexing off the end of the macro, pop this macro off the expansion stack.
   if (CurToken == Macro.getNumTokens())
@@ -25,6 +36,7 @@
   
   // Get the next token to return.
   Tok = Macro.getReplacementToken(CurToken++);
+  //Tok.SetLocation(InstantiateLoc);
 
   // If this is the first token, set the lexical properties of the token to
   // match the lexical properties of the macro identifier.

Modified: cfe/cfe/trunk/Lex/MacroInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/MacroInfo.cpp?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/MacroInfo.cpp (original)
+++ cfe/cfe/trunk/Lex/MacroInfo.cpp Wed Jul 11 11:22:30 2007
@@ -15,14 +15,3 @@
 #include <iostream>
 using namespace llvm;
 using namespace clang;
-
-/// dump - Print the macro to stderr, used for debugging.
-///
-void MacroInfo::dump(const LangOptions &Features) const {
-  std::cerr << "MACRO: ";
-  for (unsigned i = 0, e = ReplacementTokens.size(); i != e; ++i) {
-    ReplacementTokens[i].dump(Features);
-    std::cerr << "  ";
-  }
-  std::cerr << "\n";
-}

Modified: cfe/cfe/trunk/Lex/PPExpressions.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/PPExpressions.cpp?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/PPExpressions.cpp (original)
+++ cfe/cfe/trunk/Lex/PPExpressions.cpp Wed Jul 11 11:22:30 2007
@@ -127,7 +127,7 @@
     return true;
   case tok::numeric_constant: {
     // FIXME: faster.  FIXME: track signs.
-    std::string Spell = Lexer::getSpelling(PeekTok, getLangOptions());
+    std::string Spell = getSpelling(PeekTok);
     // FIXME: COMPUTE integer constants CORRECTLY.
     Result = atoi(Spell.c_str());
     Lex(PeekTok);

Modified: cfe/cfe/trunk/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/Preprocessor.cpp?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/Preprocessor.cpp (original)
+++ cfe/cfe/trunk/Lex/Preprocessor.cpp Wed Jul 11 11:22:30 2007
@@ -129,6 +129,33 @@
   Diag(Tok.getSourceLocation(), DiagID, Msg);
 }
 
+
+void Preprocessor::DumpToken(const LexerToken &Tok, bool DumpFlags) const {
+  std::cerr << tok::getTokenName(Tok.getKind()) << " '"
+            << getSpelling(Tok) << "'";
+  
+  if (!DumpFlags) return;
+  std::cerr << "\t";
+  if (Tok.isAtStartOfLine())
+    std::cerr << " [StartOfLine]";
+  if (Tok.hasLeadingSpace())
+    std::cerr << " [LeadingSpace]";
+  if (Tok.needsCleaning()) {
+    const char *Start = SourceMgr.getCharacterData(Tok.getSourceLocation());
+    std::cerr << " [UnClean='" << std::string(Start, Start+Tok.getLength())
+              << "']";
+  }
+}
+
+void Preprocessor::DumpMacro(const MacroInfo &MI) const {
+  std::cerr << "MACRO: ";
+  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
+    DumpToken(MI.getReplacementToken(i));
+    std::cerr << "  ";
+  }
+  std::cerr << "\n";
+}
+
 void Preprocessor::PrintStats() {
   std::cerr << "\n*** Preprocessor Stats:\n";
   std::cerr << FileInfo.size() << " files tracked.\n";
@@ -162,6 +189,73 @@
 }
 
 //===----------------------------------------------------------------------===//
+// Token Spelling
+//===----------------------------------------------------------------------===//
+
+
+/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
+/// token are the characters used to represent the token in the source file
+/// after trigraph expansion and escaped-newline folding.  In particular, this
+/// wants to get the true, uncanonicalized, spelling of things like digraphs
+/// UCNs, etc.
+std::string Preprocessor::getSpelling(const LexerToken &Tok) const {
+  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
+  
+  // If this token contains nothing interesting, return it directly.
+  const char *TokStart = SourceMgr.getCharacterData(Tok.getSourceLocation());
+  assert(TokStart && "Token has invalid location!");
+  if (!Tok.needsCleaning())
+    return std::string(TokStart, TokStart+Tok.getLength());
+  
+  // Otherwise, hard case, relex the characters into the string.
+  std::string Result;
+  Result.reserve(Tok.getLength());
+  
+  for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
+       Ptr != End; ) {
+    unsigned CharSize;
+    Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
+    Ptr += CharSize;
+  }
+  assert(Result.size() != unsigned(Tok.getLength()) &&
+         "NeedsCleaning flag set on something that didn't need cleaning!");
+  return Result;
+}
+
+/// getSpelling - This method is used to get the spelling of a token into a
+/// preallocated buffer, instead of as an std::string.  The caller is required
+/// to allocate enough space for the token, which is guaranteed to be at least
+/// Tok.getLength() bytes long.  The actual length of the token is returned.
+unsigned Preprocessor::getSpelling(const LexerToken &Tok, char *Buffer) const {
+  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
+  
+  const char *TokStart = SourceMgr.getCharacterData(Tok.getSourceLocation());
+  assert(TokStart && "Token has invalid location!");
+
+  // If this token contains nothing interesting, return it directly.
+  if (!Tok.needsCleaning()) {
+    unsigned Size = Tok.getLength();
+    memcpy(Buffer, TokStart, Size);
+    return Size;
+  }
+  // Otherwise, hard case, relex the characters into the string.
+  std::string Result;
+  Result.reserve(Tok.getLength());
+  
+  char *OutBuf = Buffer;
+  for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
+       Ptr != End; ) {
+    unsigned CharSize;
+    *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
+    Ptr += CharSize;
+  }
+  assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
+         "NeedsCleaning flag set on something that didn't need cleaning!");
+  
+  return OutBuf-Buffer;
+}
+
+//===----------------------------------------------------------------------===//
 // Source File Location Methods.
 //===----------------------------------------------------------------------===//
 
@@ -256,7 +350,7 @@
   IdentifierTokenInfo *Identifier = Tok.getIdentifierInfo();
   MacroInfo &MI = *Identifier->getMacroInfo();
   SourceLocation ExpandLoc = Tok.getSourceLocation();
-  unsigned MacroID = SourceMgr.getMacroID(Identifier, ExpandLoc);
+  //unsigned MacroID = SourceMgr.getMacroID(Identifier, ExpandLoc);
   if (CurLexer) {
     IncludeStack.push_back(IncludeStackInfo(CurLexer, CurNextDirLookup));
     CurLexer         = 0;
@@ -274,9 +368,7 @@
   // expanded.
   MI.DisableMacro();
   
-  CurMacroExpander = new MacroExpander(MI, MacroID, *this,
-                                       Tok.isAtStartOfLine(), 
-                                       Tok.hasLeadingSpace());
+  CurMacroExpander = new MacroExpander(Tok, *this);
 }
 
 
@@ -384,10 +476,10 @@
   // SkipExcludedConditionalBlock.  The Lexer will have already have issued
   // errors for the unterminated #if's on the conditional stack.
   if (isSkipping()) {
-    Result.StartToken(CurLexer);
+    Result.StartToken();
+    CurLexer->BufferPtr = CurLexer->BufferEnd;
+    CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd);
     Result.SetKind(tok::eof);
-    Result.SetStart(CurLexer->BufferEnd);
-    Result.SetEnd(CurLexer->BufferEnd);
     return;
   }
   
@@ -402,10 +494,10 @@
     return Lex(Result);
   }
   
-  Result.StartToken(CurLexer);
+  Result.StartToken();
+  CurLexer->BufferPtr = CurLexer->BufferEnd;
+  CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd);
   Result.SetKind(tok::eof);
-  Result.SetStart(CurLexer->BufferEnd);
-  Result.SetEnd(CurLexer->BufferEnd);
   
   // We're done with the #included file.
   delete CurLexer;
@@ -503,7 +595,7 @@
 /// is true, then #else directives are ok, if not, then we have already seen one
 /// so a #else directive is a duplicate.  When this returns, the caller can lex
 /// the first valid token.
-void Preprocessor::SkipExcludedConditionalBlock(const char *IfTokenLoc,
+void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
                                                 bool FoundNonSkipPortion,
                                                 bool FoundElse) {
   ++NumSkipped;
@@ -556,7 +648,8 @@
     // to spell an i/e in a strange way that is another letter.  Skipping this
     // allows us to avoid computing the spelling for #define/#undef and other
     // common directives.
-    char FirstChar = Tok.getStart()[0];
+    // FIXME: This should use a bit in the identifier information!
+    char FirstChar = SourceMgr.getCharacterData(Tok.getSourceLocation())[0];
     if (FirstChar >= 'a' && FirstChar <= 'z' && 
         FirstChar != 'i' && FirstChar != 'e') {
       CurLexer->ParsingPreprocessorDirective = false;
@@ -564,15 +657,17 @@
     }
     
     // Strip out trigraphs and embedded newlines.
-    std::string Directive = Lexer::getSpelling(Tok, Features);
+    std::string Directive = getSpelling(Tok);
     FirstChar = Directive[0];
     if (FirstChar == 'i' && Directive[1] == 'f') {
       if (Directive == "if" || Directive == "ifdef" || Directive == "ifndef") {
         // We know the entire #if/#ifdef/#ifndef block will be skipped, don't
         // bother parsing the condition.
         DiscardUntilEndOfDirective();
-        CurLexer->pushConditionalLevel(Tok.getStart(), /*wasskipping*/true,
-                                       /*foundnonskip*/false,/*fnddelse*/false);
+        CurLexer->pushConditionalLevel(Tok.getSourceLocation(),
+                                       /*wasskipping*/true,
+                                       /*foundnonskip*/false,
+                                       /*fnddelse*/false);
       }
     } else if (FirstChar == 'e') {
       if (Directive == "endif") {
@@ -682,7 +777,7 @@
     return HandleIfDirective(Result);
   case tok::identifier:
     // Strip out trigraphs and embedded newlines.
-    std::string Directive = Lexer::getSpelling(Result, Features);
+    std::string Directive = getSpelling(Result);
     bool isExtension = false;
     switch (Directive.size()) {
     case 4:
@@ -794,7 +889,7 @@
     return Diag(FilenameTok, diag::err_pp_include_too_deep);
   
   // Get the text form of the filename.
-  std::string Filename = CurLexer->getSpelling(FilenameTok);
+  std::string Filename = getSpelling(FilenameTok);
   assert(!Filename.empty() && "Can't have tokens with empty spellings!");
   
   // Make sure the filename is <x> or "x".
@@ -995,11 +1090,12 @@
   // Should we include the stuff contained by this directive?
   if (!MacroNameTok.getIdentifierInfo()->getMacroInfo() == isIfndef) {
     // Yes, remember that we are inside a conditional, then lex the next token.
-    CurLexer->pushConditionalLevel(DirectiveTok.getStart(), /*wasskip*/false,
+    CurLexer->pushConditionalLevel(DirectiveTok.getSourceLocation(),
+                                   /*wasskip*/false,
                                    /*foundnonskip*/true, /*foundelse*/false);
   } else {
     // No, skip the contents of this block and return the first token after it.
-    SkipExcludedConditionalBlock(DirectiveTok.getStart(),
+    SkipExcludedConditionalBlock(DirectiveTok.getSourceLocation(),
                                  /*Foundnonskip*/false, 
                                  /*FoundElse*/false);
   }
@@ -1016,11 +1112,12 @@
   // Should we include the stuff contained by this directive?
   if (ConditionalTrue) {
     // Yes, remember that we are inside a conditional, then lex the next token.
-    CurLexer->pushConditionalLevel(IfToken.getStart(), /*wasskip*/false,
+    CurLexer->pushConditionalLevel(IfToken.getSourceLocation(),
+                                   /*wasskip*/false,
                                    /*foundnonskip*/true, /*foundelse*/false);
   } else {
     // No, skip the contents of this block and return the first token after it.
-    SkipExcludedConditionalBlock(IfToken.getStart(),
+    SkipExcludedConditionalBlock(IfToken.getSourceLocation(),
                                  /*Foundnonskip*/false, 
                                  /*FoundElse*/false);
   }

Modified: cfe/cfe/trunk/include/clang/Basic/SourceManager.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Basic/SourceManager.h?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Basic/SourceManager.h (original)
+++ cfe/cfe/trunk/include/clang/Basic/SourceManager.h Wed Jul 11 11:22:30 2007
@@ -103,21 +103,12 @@
   }
   
   
-  /// getMacroID - Get or create a new FileID that represents a macro with the
-  /// specified identifier being expanded at the specified position.  This can
-  /// never fail.
-  unsigned getMacroID(const IdentifierTokenInfo *Identifier,
-                      SourceLocation ExpandPos) {
-    // FIXME: Implement ID's for macro expansions!
-    return ExpandPos.getFileID();
-  }
-  
   /// getBuffer - Return the buffer for the specified FileID.
   ///
   const SourceBuffer *getBuffer(unsigned FileID) {
     return getFileInfo(FileID)->Buffer;
   }
-
+  
   /// getIncludeLoc - Return the location of the #include for the specified
   /// FileID.
   SourceLocation getIncludeLoc(unsigned FileID) const {
@@ -136,6 +127,11 @@
            (ChunkNo << SourceLocation::FilePosBits);
   }
   
+  /// getCharacterData - Return a pointer to the start of the specified location
+  /// in the appropriate SourceBuffer.  This returns null if it cannot be
+  /// computed (e.g. invalid SourceLocation).
+  const char *getCharacterData(SourceLocation SL) const;
+  
   /// getColumnNumber - Return the column # for the specified include position.
   /// this is significantly cheaper to compute than the line number.  This
   /// returns zero if the column number isn't known.

Modified: cfe/cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/Lexer.h?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/Lexer.h Wed Jul 11 11:22:30 2007
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_LEXER_H
 
 #include "clang/Basic/TokenKinds.h"
+#include "clang/Basic/SourceLocation.h"
 #include <string>
 #include <vector>
 
@@ -53,13 +54,10 @@
 /// information as possible about each returned token.  This is expected to be
 /// compressed into a smaller form if memory footprint is important.
 class LexerToken {
-  /// The start and end of the token text itself.
-  const char *Start;
+  /// The location and length of the token text itself.
+  SourceLocation Loc;
   unsigned Length;
   
-  /// TheLexer - The lexer object this token came from.
-  const Lexer *TheLexer;
-  
   /// IdentifierInfo - If this was an identifier, this points to the uniqued
   /// information about this identifier.
   IdentifierTokenInfo *IdentifierInfo;
@@ -84,26 +82,20 @@
   tok::TokenKind getKind() const { return Kind; }
   void SetKind(tok::TokenKind K) { Kind = K; }
 
-  const char *getStart() const { return Start; }
-  const char *getEnd() const { return Start+Length; }
+  /// getLocation - Return a source location identifier for the specified
+  /// offset in the current file.
+  SourceLocation getSourceLocation() const { return Loc; }
   unsigned getLength() const { return Length; }
-  void SetStart(const char *S) { Start = S; }
-  
-  /// SetEnd - Specify the length of the token as lexed.  This relies on the
-  /// start of the token having already been set.
-  void SetEnd(const char *End) { Length = End-Start; }
+
+  void SetLocation(SourceLocation L) { Loc = L; }
+  void SetLength(unsigned Len) { Length = Len; }
   
-  /// ClearFlags - Reset all flags to cleared.
+  /// StartToken - Reset all flags to cleared.
   ///
-  void StartToken(const Lexer *L) {
+  void StartToken() {
     Flags = 0;
     IdentifierInfo = 0;
-    TheLexer = L;
-  }
-  
-  /// ClearPosition - Mark this token as not having a position, FIXME temporary.
-  void ClearPosition() {
-    TheLexer = 0;
+    Loc = SourceLocation();
   }
   
   IdentifierTokenInfo *getIdentifierInfo() const { return IdentifierInfo; }
@@ -129,10 +121,6 @@
       ClearFlag(Flag);
   }
   
-  /// getSourceLocation - Return a source location identifier for the specified
-  /// offset in the current file.
-  SourceLocation getSourceLocation() const;
-  
   /// isAtStartOfLine - Return true if this token is at the start of a line.
   ///
   bool isAtStartOfLine() const { return Flags & StartOfLine; }
@@ -145,17 +133,14 @@
   /// newlines in it.
   ///
   bool needsCleaning() const { return Flags & NeedsCleaning; }
-  
-  /// dump - Print the token to stderr, used for debugging.
-  ///
-  void dump(const LangOptions &Features, bool DumpFlags = false) const;
 };
 
 /// PPConditionalInfo - Information about the conditional stack (#if directives)
 /// currently active.
 struct PPConditionalInfo {
   /// IfLoc - Location where the conditional started.
-  const char *IfLoc;
+  ///
+  SourceLocation IfLoc;
   
   /// WasSkipping - True if this was contained in a skipping directive, e.g.
   /// in a "#if 0" block.
@@ -219,7 +204,7 @@
   /// the preprocessor.
   void Lex(LexerToken &Result) {
     // Start a new token.
-    Result.StartToken(this);
+    Result.StartToken();
     
     // NOTE, any changes here should also change code after calls to 
     // Preprocessor::HandleDirective
@@ -236,30 +221,7 @@
   /// uninterpreted string.  This switches the lexer out of directive mode.
   std::string ReadToEndOfLine();
   
-  /// getSpelling() - Return the 'spelling' of the Tok token.  The spelling of a
-  /// token is the characters used to represent the token in the source file
-  /// after trigraph expansion and escaped-newline folding.  In particular, this
-  /// wants to get the true, uncanonicalized, spelling of things like digraphs
-  /// UCNs, etc.
-  static std::string getSpelling(const LexerToken &Tok,
-                                 const LangOptions &Features);
-  std::string getSpelling(const LexerToken &Tok) const {
-    assert(this && "Can't get the spelling of a token with a null lexer!");
-    return getSpelling(Tok, Features);
-  }
-
-  /// getSpelling - This method is used to get the spelling of a token into a
-  /// preallocated buffer, instead of as an std::string.  The caller is required
-  /// to allocate enough space for the token, which is guaranteed to be at most
-  /// Tok.End-Tok.Start bytes long.  The actual length of the token is returned.
-  static unsigned getSpelling(const LexerToken &Tok, char *Buffer,
-                              const LangOptions &Features);
-  unsigned getSpelling(const LexerToken &Tok, char *Buffer) const {
-    assert(this && "Can't get the spelling of a token with a null lexer!");
-    return getSpelling(Tok, Buffer, Features);
-  }
-  
-  
+ 
   /// Diag - Forwarding function for diagnostics.  This translate a source
   /// position in the current buffer into a SourceLocation object for rendering.
   void Diag(const char *Loc, unsigned DiagID,
@@ -277,7 +239,18 @@
   /// by Lex.
   ///
   void LexTokenInternal(LexerToken &Result);
-    
+
+  /// FormTokenWithChars - When we lex a token, we have identified a span
+  /// starting at BufferPtr, going to TokEnd that forms the token.  This method
+  /// takes that range and assigns it to the token as its location and size.  In
+  /// addition, since tokens cannot overlap, this also updates BufferPtr to be
+  /// TokEnd.
+  void FormTokenWithChars(LexerToken &Result, const char *TokEnd) {
+    Result.SetLocation(getSourceLocation(BufferPtr));
+    Result.SetLength(TokEnd-BufferPtr);
+    BufferPtr = TokEnd;
+  }
+  
   
   //===--------------------------------------------------------------------===//
   // Lexer character reading interfaces.
@@ -349,6 +322,25 @@
   /// method.
   char getCharAndSizeSlow(const char *Ptr, unsigned &Size, LexerToken *Tok = 0);
   
+  /// getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever
+  /// emit a warning.
+  static inline char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size,
+                                          const LangOptions &Features) {
+    // If this is not a trigraph and not a UCN or escaped newline, return
+    // quickly.
+    if (Ptr[0] != '?' && Ptr[0] != '\\') {
+      Size = 1;
+      return *Ptr;
+    }
+    
+    Size = 0;
+    return getCharAndSizeSlowNoWarn(Ptr, Size, Features);
+  }
+  
+  /// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a
+  /// diagnostic.
+  static char getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
+                                       const LangOptions &Features);
   
   //===--------------------------------------------------------------------===//
   // #if directive handling.
@@ -356,7 +348,7 @@
   /// pushConditionalLevel - When we enter a #if directive, this keeps track of
   /// what we are currently in for diagnostic emission (e.g. #if with missing
   /// #endif).
-  void pushConditionalLevel(const char *DirectiveStart, bool WasSkipping,
+  void pushConditionalLevel(SourceLocation DirectiveStart, bool WasSkipping,
                             bool FoundNonSkip, bool FoundElse) {
     PPConditionalInfo CI;
     CI.IfLoc = DirectiveStart;

Modified: cfe/cfe/trunk/include/clang/Lex/MacroExpander.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/MacroExpander.h?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/MacroExpander.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/MacroExpander.h Wed Jul 11 11:22:30 2007
@@ -14,6 +14,8 @@
 #ifndef LLVM_CLANG_MACROEXPANDER_H
 #define LLVM_CLANG_MACROEXPANDER_H
 
+#include "clang/Basic/SourceLocation.h"
+
 namespace llvm {
 namespace clang {
   class MacroInfo;
@@ -28,10 +30,6 @@
   ///
   MacroInfo &Macro;
 
-  /// CurMacroID - This encodes the instantiation point of the macro being
-  /// expanded and the include stack.
-  unsigned CurMacroID;
-  
   /// PP - The current preprocessor object we are expanding for.
   ///
   Preprocessor &PP;
@@ -39,16 +37,16 @@
   /// CurToken - This is the next token that Lex will return.
   unsigned CurToken;
   
+  /// InstantiateLoc - The source location where this macro was instantiated.
+  ///
+  SourceLocation InstantiateLoc;
+  
   /// Lexical information about the expansion point of the macro: the identifier
   /// that the macro expanded from had these properties.
   bool AtStartOfLine, HasLeadingSpace;
   
 public:
-  MacroExpander(MacroInfo &macro, unsigned MacroID, Preprocessor &pp,
-                bool atStartOfLine, bool hasLeadingSpace)
-    : Macro(macro), CurMacroID(MacroID), PP(pp), CurToken(0),
-      AtStartOfLine(atStartOfLine), HasLeadingSpace(hasLeadingSpace) {
-  }
+  MacroExpander(LexerToken &Tok, Preprocessor &pp);
   
   MacroInfo &getMacro() const { return Macro; }
 

Modified: cfe/cfe/trunk/include/clang/Lex/MacroInfo.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/MacroInfo.h?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/MacroInfo.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/MacroInfo.h Wed Jul 11 11:22:30 2007
@@ -14,7 +14,6 @@
 #ifndef LLVM_CLANG_MACROINFO_H
 #define LLVM_CLANG_MACROINFO_H
 
-#include "clang/Basic/SourceLocation.h"
 #include "clang/Lex/Lexer.h"
 #include <vector>
 
@@ -72,10 +71,9 @@
   /// AddTokenToBody - Add the specified token to the replacement text for the
   /// macro.
   void AddTokenToBody(const LexerToken &Tok) {
-    ReplacementTokens.push_back(Tok);
     // FIXME: Remember where this token came from, do something intelligent with
     // its location.
-    ReplacementTokens.back().ClearPosition();
+    ReplacementTokens.push_back(Tok);
   }
   
   /// isEnabled - Return true if this macro is enabled: in other words, that we
@@ -92,10 +90,6 @@
     isDisabled = true;
   }
   
-  /// dump - Print the macro to stderr, used for debugging.
-  ///
-  void dump(const LangOptions &Features) const;
-  
   // Todo:
   // bool isDefinedInSystemHeader() { Look this up based on Location }
 };

Modified: cfe/cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=38551&r1=38550&r2=38551&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/Preprocessor.h Wed Jul 11 11:22:30 2007
@@ -273,6 +273,24 @@
   void Diag(const LexerToken &Tok, unsigned DiagID, const std::string &Msg="");  
   void Diag(SourceLocation Loc, unsigned DiagID, const std::string &Msg="");  
   
+  /// getSpelling() - Return the 'spelling' of the Tok token.  The spelling of a
+  /// token is the characters used to represent the token in the source file
+  /// after trigraph expansion and escaped-newline folding.  In particular, this
+  /// wants to get the true, uncanonicalized, spelling of things like digraphs
+  /// UCNs, etc.
+  std::string getSpelling(const LexerToken &Tok) const;
+  
+  /// getSpelling - This method is used to get the spelling of a token into a
+  /// preallocated buffer, instead of as an std::string.  The caller is required
+  /// to allocate enough space for the token, which is guaranteed to be at least
+  /// Tok.getLength() bytes long.  The length of the actual result is returned.
+  unsigned getSpelling(const LexerToken &Tok, char *Buffer) const;
+  
+  /// DumpToken - Print the token to stderr, used for debugging.
+  ///
+  void DumpToken(const LexerToken &Tok, bool DumpFlags = false) const;
+  void DumpMacro(const MacroInfo &MI) const;
+  
   void PrintStats();
 
   //===--------------------------------------------------------------------===//
@@ -326,7 +344,7 @@
   /// FoundElse is false, then #else directives are ok, if not, then we have
   /// already seen one so a #else directive is a duplicate.  When this returns,
   /// the caller can lex the first valid token.
-  void SkipExcludedConditionalBlock(const char *IfTokenLoc,
+  void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
                                     bool FoundNonSkipPortion, bool FoundElse);
   
   /// EvaluateDirectiveExpression - Evaluate an integer constant expression that





More information about the cfe-commits mailing list