[cfe-commits] r91586 - in /cfe/trunk: lib/Lex/Lexer.cpp test/Lexer/msdos-cpm-eof.c

[Chris Lattner]

Author: lattner
Date: Wed Dec 16 23:29:40 2009
New Revision: 91586

URL: http://llvm.org/viewvc/llvm-project?rev=91586&view=rev
Log:
reimplement r90860, fixing a couple of problems:
1. Don't make a copy of LangOptions every time a lexer is created.
2. Don't make CharInfo global mutable state.
3. Fix the implementation to properly treat ^Z as EOF instead of as
   horizontal whitespace, which matches the semantic implemented by VC++.

Modified:
    cfe/trunk/lib/Lex/Lexer.cpp
    cfe/trunk/test/Lexer/msdos-cpm-eof.c

Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=91586&r1=91585&r2=91586&view=diff

==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Wed Dec 16 23:29:40 2009
@@ -33,7 +33,7 @@
 #include <cctype>
 using namespace clang;
 
-static void InitCharacterInfo(LangOptions);
+static void InitCharacterInfo();
 
 //===----------------------------------------------------------------------===//
 // Token Class Implementation
@@ -59,7 +59,7 @@
 
 void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
                       const char *BufEnd) {
-  InitCharacterInfo(Features);
+  InitCharacterInfo();
 
   BufferStart = BufStart;
   BufferPtr = BufPtr;
@@ -254,7 +254,7 @@
 
 // Statically initialize CharInfo table based on ASCII character set
 // Reference: FreeBSD 7.2 /usr/share/misc/ascii
-static unsigned char CharInfo[256] =
+static const unsigned char CharInfo[256] =
 {
 // 0 NUL         1 SOH         2 STX         3 ETX
 // 4 EOT         5 ENQ         6 ACK         7 BEL
@@ -322,7 +322,7 @@
    0           , 0           , 0           , 0
 };
 
-static void InitCharacterInfo(LangOptions Features) {
+static void InitCharacterInfo() {
   static bool isInited = false;
   if (isInited) return;
   // check the statically-initialized CharInfo table
@@ -341,10 +341,6 @@
   for (unsigned i = '0'; i <= '9'; ++i)
     assert(CHAR_NUMBER == CharInfo[i]);
     
-  if (Features.Microsoft)
-    // Hack to treat DOS & CP/M EOF (^Z) as horizontal whitespace.
-    CharInfo[26/*sub*/] = CHAR_HORZ_WS;  
-
   isInited = true;
 }
 
@@ -1549,6 +1545,22 @@
       return; // KeepWhitespaceMode
 
     goto LexNextToken;   // GCC isn't tail call eliminating.
+      
+  case 26:  // DOS & CP/M EOF: "^Z".
+    // If we're in Microsoft extensions mode, treat this as end of file.
+    if (Features.Microsoft) {
+      // Read the PP instance variable into an automatic variable, because
+      // LexEndOfFile will often delete 'this'.
+      Preprocessor *PPCache = PP;
+      if (LexEndOfFile(Result, CurPtr-1))  // Retreat back into the file.
+        return;   // Got a token to return.
+      assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
+      return PPCache->Lex(Result);
+    }
+    // If Microsoft extensions are disabled, this is just random garbage.
+    Kind = tok::unknown;
+    break;
+      
   case '\n':
   case '\r':
     // If we are inside a preprocessor directive and we see the end of line,
@@ -1599,7 +1611,7 @@
       goto SkipHorizontalWhitespace;
     }
     goto LexNextToken;   // GCC isn't tail call eliminating.
-
+      
   // C99 6.4.4.1: Integer Constants.
   // C99 6.4.4.2: Floating Constants.
   case '0': case '1': case '2': case '3': case '4':

Modified: cfe/trunk/test/Lexer/msdos-cpm-eof.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Lexer/msdos-cpm-eof.c?rev=91586&r1=91585&r2=91586&view=diff

==============================================================================
--- cfe/trunk/test/Lexer/msdos-cpm-eof.c (original)
+++ cfe/trunk/test/Lexer/msdos-cpm-eof.c Wed Dec 16 23:29:40 2009
@@ -1,5 +1,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -fms-extensions %s
 
-int a;
+int x; 
 
 
+
+I am random garbage after ^Z