[cfe-commits] r173370 - in /cfe/trunk: lib/Lex/Lexer.cpp test/Lexer/unicode.c

[Jordan Rose]

Author: jrose
Date: Thu Jan 24 14:50:50 2013
New Revision: 173370

URL: http://llvm.org/viewvc/llvm-project?rev=173370&view=rev
Log:
As an extension, treat Unicode whitespace characters as whitespace.

Added:
    cfe/trunk/test/Lexer/unicode.c
Modified:
    cfe/trunk/lib/Lex/Lexer.cpp

Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=173370&r1=173369&r2=173370&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Thu Jan 24 14:50:50 2013
@@ -2791,7 +2791,30 @@
   return CodePoint;
 }
 
+static bool isUnicodeWhitespace(uint32_t C) {
+  return (C == 0x0085 || C == 0x00A0 || C == 0x1680 ||
+          C == 0x180E || (C >= 0x2000 && C <= 0x200A) ||
+          C == 0x2028 || C == 0x2029 || C == 0x202F ||
+          C == 0x205F || C == 0x3000);
+}
+
 void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
+  if (isUnicodeWhitespace(C)) {
+    if (!isLexingRawMode()) {
+      CharSourceRange CharRange =
+        CharSourceRange::getCharRange(getSourceLocation(),
+                                      getSourceLocation(CurPtr));
+      Diag(BufferPtr, diag::ext_unicode_whitespace)
+        << CharRange;
+    }
+
+    Result.setFlag(Token::LeadingSpace);
+    if (SkipWhitespace(Result, CurPtr))
+      return; // KeepWhitespaceMode
+
+    return LexTokenInternal(Result);
+  }
+
   if (isAllowedIDChar(C) && isAllowedInitiallyIDChar(C)) {
     MIOpt.ReadToken();
     return LexIdentifier(Result, CurPtr);

Added: cfe/trunk/test/Lexer/unicode.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Lexer/unicode.c?rev=173370&view=auto
==============================================================================
--- cfe/trunk/test/Lexer/unicode.c (added)
+++ cfe/trunk/test/Lexer/unicode.c Thu Jan 24 14:50:50 2013
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+// This file contains Unicode characters; please do not "fix" them!
+
+extern int x; // expected-warning {{treating Unicode character as whitespace}}
+extern int　x; // expected-warning {{treating Unicode character as whitespace}}