[PATCH] D43165: [lit] Fix problem in how Python versions open files with different encodings

[Zachary Turner via llvm-commits]

Lgtm
On Mon, Apr 2, 2018 at 12:21 PM Aaron Smith via Phabricator <
reviews at reviews.llvm.org> wrote:

> asmith updated this revision to Diff 140661.
> asmith edited the summary of this revision.
> asmith added a comment.
>
> This is changed based on the reviewers suggestions. First open as usual
> and on failure reopen as UTF8 and when that fails reopen as binary.
> Verified on Windows and Linux.
>
>
> https://reviews.llvm.org/D43165
>
> Files:
>   lit/lit/TestRunner.py
>
>
> Index: lit/lit/TestRunner.py
> ===================================================================
> --- lit/lit/TestRunner.py
> +++ lit/lit/TestRunner.py
> @@ -2,6 +2,7 @@
>  import difflib
>  import errno
>  import functools
> +import io
>  import itertools
>  import getopt
>  import os, signal, subprocess, sys
> @@ -385,9 +386,51 @@
>              return path, sorted(child_trees)
>
>      def compareTwoFiles(filepaths):
> +        compare_bytes = False
> +        encoding = None
>          filelines = []
>          for file in filepaths:
> -            with open(file, 'r') as f:
> +            try:
> +                with open(file, 'r') as f:
> +                    filelines.append(f.readlines())
> +            except UnicodeDecodeError:
> +                try:
> +                    with open(file, 'r', encoding="utf-8") as f:
> +                        filelines.append(f.readlines())
> +                    encoding = "utf-8"
> +                except:
> +                    compare_bytes = True
> +
> +        if compare_bytes:
> +            return compareTwoBinaryFiles(filepaths)
> +        else:
> +            return compareTwoTextFiles(filepaths, encoding)
> +
> +    def compareTwoBinaryFiles(filepaths):
> +        filelines = []
> +        for file in filepaths:
> +            with open(file, 'rb') as f:
> +                filelines.append(f.readlines())
> +
> +        exitCode = 0
> +        if hasattr(difflib, 'diff_bytes'):
> +            # python 3.5 or newer
> +            diffs = difflib.diff_bytes(difflib.unified_diff,
> filelines[0], filelines[1], filepaths[0].encode(), filepaths[1].encode())
> +            diffs = [diff.decode() for diff in diffs]
> +        else:
> +            # python 2.7
> +            func = difflib.unified_diff if unified_diff else
> difflib.context_diff
> +            diffs = func(filelines[0], filelines[1], filepaths[0],
> filepaths[1])
> +
> +        for diff in diffs:
> +            stdout.write(diff)
> +            exitCode = 1
> +        return exitCode
> +
> +    def compareTwoTextFiles(filepaths, encoding):
> +        filelines = []
> +        for file in filepaths:
> +            with open(file, 'r', encoding=encoding) as f:
>                  filelines.append(f.readlines())
>
>          exitCode = 0
>
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20180402/4ebcbf52/attachment.html>