[cfe-commits] r38749 - /cfe/cfe/trunk/Lex/Lexer.cpp

[sabre at cs.uiuc.edu]

Author: sabre
Date: Wed Jul 11 11:24:18 2007
New Revision: 38749

URL: http://llvm.org/viewvc/llvm-project?rev=38749&view=rev
Log:
If an invalid string or character is read, return the invalid part as a tok::unknown token.

Modified:
    cfe/cfe/trunk/Lex/Lexer.cpp

Modified: cfe/cfe/trunk/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/cfe/trunk/Lex/Lexer.cpp?rev=38749&r1=38748&r2=38749&view=diff

==============================================================================
--- cfe/cfe/trunk/Lex/Lexer.cpp (original)
+++ cfe/cfe/trunk/Lex/Lexer.cpp Wed Jul 11 11:24:18 2007
@@ -453,14 +453,16 @@
     } else if (C == '\n' || C == '\r' ||             // Newline.
                (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
       Diag(BufferPtr, diag::err_unterminated_string);
-      BufferPtr = CurPtr-1;
-      return LexTokenInternal(Result);
+      Result.SetKind(tok::unknown);
+      FormTokenWithChars(Result, CurPtr-1);
+      return;
     } else if (C == 0) {
       NulCharacter = CurPtr-1;
     }
     C = getAndAdvanceChar(CurPtr, Result);
   }
   
+  // If a nul character existed in the string, warn about it.
   if (NulCharacter) Diag(NulCharacter, diag::null_in_string);
 
   Result.SetKind(tok::string_literal);
@@ -483,14 +485,16 @@
     } else if (C == '\n' || C == '\r' ||             // Newline.
                (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
       Diag(BufferPtr, diag::err_unterminated_string);
-      BufferPtr = CurPtr-1;
-      return LexTokenInternal(Result);
+      Result.SetKind(tok::unknown);
+      FormTokenWithChars(Result, CurPtr-1);
+      return;
     } else if (C == 0) {
       NulCharacter = CurPtr-1;
     }
     C = getAndAdvanceChar(CurPtr, Result);
   }
   
+  // If a nul character existed in the string, warn about it.
   if (NulCharacter) Diag(NulCharacter, diag::null_in_string);
   
   Result.SetKind(tok::angle_string_literal);
@@ -509,8 +513,9 @@
   char C = getAndAdvanceChar(CurPtr, Result);
   if (C == '\'') {
     Diag(BufferPtr, diag::err_empty_character);
-    BufferPtr = CurPtr;
-    return LexTokenInternal(Result);
+    Result.SetKind(tok::unknown);
+    FormTokenWithChars(Result, CurPtr);
+    return;
   } else if (C == '\\') {
     // Skip the escaped character.
     // FIXME: UCN's.
@@ -529,8 +534,9 @@
       } else if (C == '\n' || C == '\r' ||               // Newline.
                  (C == 0 && CurPtr-1 == BufferEnd)) {    // End of file.
         Diag(BufferPtr, diag::err_unterminated_char);
-        BufferPtr = CurPtr-1;
-        return LexTokenInternal(Result);
+        Result.SetKind(tok::unknown);
+        FormTokenWithChars(Result, CurPtr-1);
+        return;
       } else if (C == 0) {
         NulCharacter = CurPtr-1;
       }