[llvm] r217688 - [lit] Parse all strings as UTF-8 rather than ASCII.
Jordan Rose
jordan_rose at apple.com
Fri Sep 12 09:46:05 PDT 2014
Author: jrose
Date: Fri Sep 12 11:46:05 2014
New Revision: 217688
URL: http://llvm.org/viewvc/llvm-project?rev=217688&view=rev
Log:
[lit] Parse all strings as UTF-8 rather than ASCII.
As far as I can tell UTF-8 has been supported since the beginning of Python's
codec support, and it's the de facto standard for text these days, at least
for primarily-English text. This allows us to put Unicode into lit RUN lines.
rdar://problem/18311663
Added:
llvm/trunk/test/Other/lit-unicode.txt
Modified:
llvm/trunk/utils/lit/lit/ProgressBar.py
llvm/trunk/utils/lit/lit/TestRunner.py
llvm/trunk/utils/lit/lit/formats/googletest.py
llvm/trunk/utils/lit/lit/util.py
Added: llvm/trunk/test/Other/lit-unicode.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Other/lit-unicode.txt?rev=217688&view=auto
==============================================================================
--- llvm/trunk/test/Other/lit-unicode.txt (added)
+++ llvm/trunk/test/Other/lit-unicode.txt Fri Sep 12 11:46:05 2014
@@ -0,0 +1,3 @@
+REQUIRES: shell
+RUN: echo "ãããã" | FileCheck %s
+CHECK: {{^}}ãããã{{$}}
Modified: llvm/trunk/utils/lit/lit/ProgressBar.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/ProgressBar.py?rev=217688&r1=217687&r2=217688&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/ProgressBar.py (original)
+++ llvm/trunk/utils/lit/lit/ProgressBar.py Fri Sep 12 11:46:05 2014
@@ -6,8 +6,8 @@
import sys, re, time
def to_bytes(str):
- # Encode to Latin1 to get binary data.
- return str.encode('ISO-8859-1')
+ # Encode to UTF-8 to get binary data.
+ return str.encode('utf-8')
class TerminalController:
"""
@@ -136,7 +136,7 @@ class TerminalController:
def _tparm(self, arg, index):
import curses
- return curses.tparm(to_bytes(arg), index).decode('ascii') or ''
+ return curses.tparm(to_bytes(arg), index).decode('utf-8') or ''
def _tigetstr(self, cap_name):
# String capabilities can include "delays" of the form "$<2>".
@@ -147,7 +147,7 @@ class TerminalController:
if cap is None:
cap = ''
else:
- cap = cap.decode('ascii')
+ cap = cap.decode('utf-8')
return re.sub(r'\$<\d+>[/*]?', '', cap)
def render(self, template):
Modified: llvm/trunk/utils/lit/lit/TestRunner.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/TestRunner.py?rev=217688&r1=217687&r2=217688&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/TestRunner.py (original)
+++ llvm/trunk/utils/lit/lit/TestRunner.py Fri Sep 12 11:46:05 2014
@@ -192,6 +192,11 @@ def executeShCmd(cmd, cfg, cwd, results)
f.seek(0, 0)
procData[i] = (procData[i][0], f.read())
+ def to_string(bytes):
+ if isinstance(bytes, str):
+ return bytes
+ return bytes.encode('utf-8')
+
exitCode = None
for i,(out,err) in enumerate(procData):
res = procs[i].wait()
@@ -201,11 +206,11 @@ def executeShCmd(cmd, cfg, cwd, results)
# Ensure the resulting output is always of string type.
try:
- out = str(out.decode('ascii'))
+ out = to_string(out.decode('utf-8'))
except:
out = str(out)
try:
- err = str(err.decode('ascii'))
+ err = to_string(err.decode('utf-8'))
except:
err = str(err)
@@ -314,13 +319,18 @@ def parseIntegratedTestScriptCommands(so
# Python2 and bytes in Python3.
#
# Once we find a match, we do require each script line to be decodable to
- # ascii, so we convert the outputs to ascii before returning. This way the
+ # UTF-8, so we convert the outputs to UTF-8 before returning. This way the
# remaining code can work with "strings" agnostic of the executing Python
# version.
def to_bytes(str):
- # Encode to Latin1 to get binary data.
- return str.encode('ISO-8859-1')
+ # Encode to UTF-8 to get binary data.
+ return str.encode('utf-8')
+ def to_string(bytes):
+ if isinstance(bytes, str):
+ return bytes
+ return to_bytes(bytes)
+
keywords = ('RUN:', 'XFAIL:', 'REQUIRES:', 'END.')
keywords_re = re.compile(
to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
@@ -341,13 +351,13 @@ def parseIntegratedTestScriptCommands(so
match_position)
last_match_position = match_position
- # Convert the keyword and line to ascii strings and yield the
+ # Convert the keyword and line to UTF-8 strings and yield the
# command. Note that we take care to return regular strings in
# Python 2, to avoid other code having to differentiate between the
# str and unicode types.
keyword,ln = match.groups()
- yield (line_number, str(keyword[:-1].decode('ascii')),
- str(ln.decode('ascii')))
+ yield (line_number, to_string(keyword[:-1].decode('utf-8')),
+ to_string(ln.decode('utf-8')))
finally:
f.close()
Modified: llvm/trunk/utils/lit/lit/formats/googletest.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/formats/googletest.py?rev=217688&r1=217687&r2=217688&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/formats/googletest.py (original)
+++ llvm/trunk/utils/lit/lit/formats/googletest.py Fri Sep 12 11:46:05 2014
@@ -31,7 +31,7 @@ class GoogleTest(TestFormat):
try:
lines = lit.util.capture([path, '--gtest_list_tests'],
env=localConfig.environment)
- lines = lines.decode('ascii')
+ lines = lines.decode('utf-8')
if kIsWindows:
lines = lines.replace('\r', '')
lines = lines.split('\n')
Modified: llvm/trunk/utils/lit/lit/util.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/utils/lit/lit/util.py?rev=217688&r1=217687&r2=217688&view=diff
==============================================================================
--- llvm/trunk/utils/lit/lit/util.py (original)
+++ llvm/trunk/utils/lit/lit/util.py Fri Sep 12 11:46:05 2014
@@ -156,13 +156,18 @@ def executeCommand(command, cwd=None, en
if exitCode == -signal.SIGINT:
raise KeyboardInterrupt
+ def to_string(bytes):
+ if isinstance(bytes, str):
+ return bytes
+ return bytes.encode('utf-8')
+
# Ensure the resulting output is always of string type.
try:
- out = str(out.decode('ascii'))
+ out = to_string(out.decode('utf-8'))
except:
out = str(out)
try:
- err = str(err.decode('ascii'))
+ err = to_string(err.decode('utf-8'))
except:
err = str(err)
More information about the llvm-commits
mailing list