[cfe-commits] r38548 - in /cfe/cfe/trunk: Driver/clang.cpp Lex/Lexer.cpp include/clang/Lex/Lexer.h

[sabre at cs.uiuc.edu]

Author: sabre
Date: Wed Jul 11 11:22:27 2007
New Revision: 38548

URL: http://llvm.org/viewvc/llvm-project?rev=38548&view=rev
Log:
Change the token representation to take a Start and Length instead of a
Start/End pointer.

Modified:
    cfe/cfe/trunk/Driver/clang.cpp
    cfe/cfe/trunk/Lex/Lexer.cpp
    cfe/cfe/trunk/include/clang/Lex/Lexer.h

Modified: cfe/cfe/trunk/Driver/clang.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Driver/clang.cpp?rev=38548&r1=38547&r2=38548&view=diff

==============================================================================
--- cfe/cfe/trunk/Driver/clang.cpp (original)
+++ cfe/cfe/trunk/Driver/clang.cpp Wed Jul 11 11:22:27 2007
@@ -626,7 +626,7 @@
     }
     isFirstToken = false;    
     
-    if (Tok.getEnd()-Tok.getStart() < 256) {
+    if (Tok.getLength() < 256) {
       unsigned Len = Lexer::getSpelling(Tok, Buffer, PP.getLangOptions());
       Buffer[Len] = 0;
       std::cout << Buffer;

Modified: cfe/cfe/trunk/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/Lexer.cpp?rev=38548&r1=38547&r2=38548&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/Lexer.cpp (original)
+++ cfe/cfe/trunk/Lex/Lexer.cpp Wed Jul 11 11:22:27 2007
@@ -86,10 +86,10 @@
       // #define TWELVE 1\    <whitespace only>
       // 2
       // TWELVE
-      std::cerr << "*unspelled*" << std::string(Start, End);
+      std::cerr << "*unspelled*" << std::string(getStart(), getEnd());
     }
   } else
-    std::cerr << std::string(Start, End);
+    std::cerr << std::string(getStart(), getEnd());
   std::cerr << "'";
   
   if (DumpFlags) {
@@ -99,7 +99,7 @@
     if (hasLeadingSpace())
       std::cerr << " [LeadingSpace]";
     if (needsCleaning())
-      std::cerr << " [Spelling='" << std::string(Start, End) << "']";
+      std::cerr << " [Spelling='" << std::string(getStart(), getEnd()) << "']";
   }
 }
 
@@ -387,14 +387,14 @@
   
   // Otherwise, hard case, relex the characters into the string.
   std::string Result;
-  Result.reserve(Tok.getEnd()-Tok.getStart());
+  Result.reserve(Tok.getLength());
   
   for (const char *Ptr = Tok.getStart(), *End = Tok.getEnd(); Ptr != End; ) {
     unsigned CharSize;
     Result.push_back(getCharAndSizeNoWarn(Ptr, CharSize, Features));
     Ptr += CharSize;
   }
-  assert(Result.size() != unsigned(Tok.getEnd()-Tok.getStart()) &&
+  assert(Result.size() != unsigned(Tok.getLength()) &&
          "NeedsCleaning flag set on something that didn't need cleaning!");
   return Result;
 }
@@ -409,13 +409,13 @@
 
   // If this token contains nothing interesting, return it directly.
   if (!Tok.needsCleaning()) {
-    unsigned Size = Tok.getEnd()-Tok.getStart();
+    unsigned Size = Tok.getLength();
     memcpy(Buffer, Tok.getStart(), Size);
     return Size;
   }
   // Otherwise, hard case, relex the characters into the string.
   std::string Result;
-  Result.reserve(Tok.getEnd()-Tok.getStart());
+  Result.reserve(Tok.getLength());
   
   char *OutBuf = Buffer;
   for (const char *Ptr = Tok.getStart(), *End = Tok.getEnd(); Ptr != End; ) {
@@ -423,7 +423,7 @@
     *OutBuf++ = getCharAndSizeNoWarn(Ptr, CharSize, Features);
     Ptr += CharSize;
   }
-  assert(OutBuf-Buffer != Tok.getEnd()-Tok.getStart() &&
+  assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
          "NeedsCleaning flag set on something that didn't need cleaning!");
   
   return OutBuf-Buffer;
@@ -459,7 +459,7 @@
       SpelledTokEnd   = Result.getEnd();
     } else {
       // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
-      char *TmpBuf = (char*)alloca(Result.getEnd()-Result.getStart());
+      char *TmpBuf = (char*)alloca(Result.getLength());
       unsigned Size = getSpelling(Result, TmpBuf);
       SpelledTokStart = TmpBuf;
       SpelledTokEnd = TmpBuf+Size;

Modified: cfe/cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/include/clang/Lex/Lexer.h?rev=38548&r1=38547&r2=38548&view=diff

==============================================================================
--- cfe/cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/cfe/trunk/include/clang/Lex/Lexer.h Wed Jul 11 11:22:27 2007
@@ -54,7 +54,8 @@
 /// compressed into a smaller form if memory footprint is important.
 class LexerToken {
   /// The start and end of the token text itself.
-  const char *Start, *End;
+  const char *Start;
+  unsigned Length;
   
   /// TheLexer - The lexer object this token came from.
   const Lexer *TheLexer;
@@ -84,9 +85,13 @@
   void SetKind(tok::TokenKind K) { Kind = K; }
 
   const char *getStart() const { return Start; }
-  const char *getEnd() const { return End; }
+  const char *getEnd() const { return Start+Length; }
+  unsigned getLength() const { return Length; }
   void SetStart(const char *S) { Start = S; }
-  void SetEnd  (const char *E) { End = E; }
+  
+  /// SetEnd - Specify the length of the token as lexed.  This relies on the
+  /// start of the token having already been set.
+  void SetEnd(const char *End) { Length = End-Start; }
   
   const Lexer *getLexer() const { return TheLexer; }