[clang-tools-extra] r356565 - Reland r356547 after fixing the tests for Linux.
Galina Kistanova via cfe-commits
cfe-commits at lists.llvm.org
Wed Mar 20 14:02:57 PDT 2019
Hello Zinovy,
This commit broke test on the next builder:
http://lab.llvm.org:8011/builders/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/builds/45557
. . .
Failing Tests (1):
Clang Tools :: clang-tidy/clang-tidy-diff.cpp
Please have a look?
Thanks
Galina
On Wed, Mar 20, 2019 at 8:49 AM Zinovy Nis via cfe-commits <
cfe-commits at lists.llvm.org> wrote:
> Author: zinovy.nis
> Date: Wed Mar 20 08:50:26 2019
> New Revision: 356565
>
> URL: http://llvm.org/viewvc/llvm-project?rev=356565&view=rev
> Log:
> Reland r356547 after fixing the tests for Linux.
>
> [clang-tidy] Parallelize clang-tidy-diff.py
>
> This patch has 2 rationales:
>
> - large patches lead to long command lines and often cause max command
> line length restrictions imposed by OS;
> - clang-tidy runs on modified files are independent and can be done in
> parallel, the same as done for run-clang-tidy.
>
> Differential Revision: https://reviews.llvm.org/D57662
>
>
> Modified:
> clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py
>
> Modified: clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py
> URL:
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py?rev=356565&r1=356564&r2=356565&view=diff
>
> ==============================================================================
> --- clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py (original)
> +++ clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py Wed Mar 20
> 08:50:26 2019
> @@ -24,10 +24,90 @@ Example usage for git/svn users:
> """
>
> import argparse
> +import glob
> import json
> +import multiprocessing
> +import os
> import re
> +import shutil
> import subprocess
> import sys
> +import tempfile
> +import threading
> +import traceback
> +import yaml
> +
> +is_py2 = sys.version[0] == '2'
> +
> +if is_py2:
> + import Queue as queue
> +else:
> + import queue as queue
> +
> +
> +def run_tidy(task_queue, lock, timeout):
> + watchdog = None
> + while True:
> + command = task_queue.get()
> + try:
> + proc = subprocess.Popen(command,
> + stdout=subprocess.PIPE,
> + stderr=subprocess.PIPE)
> +
> + if timeout is not None:
> + watchdog = threading.Timer(timeout, proc.kill)
> + watchdog.start()
> +
> + stdout, stderr = proc.communicate()
> +
> + with lock:
> + sys.stdout.write(stdout.decode('utf-8') + '\n')
> + sys.stdout.flush()
> + if stderr:
> + sys.stderr.write(stderr.decode('utf-8') + '\n')
> + sys.stderr.flush()
> + except Exception as e:
> + with lock:
> + sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
> + finally:
> + with lock:
> + if (not timeout is None) and (not watchdog is None):
> + if not watchdog.is_alive():
> + sys.stderr.write('Terminated by timeout: ' +
> + ' '.join(command) + '\n')
> + watchdog.cancel()
> + task_queue.task_done()
> +
> +
> +def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
> + for _ in range(max_tasks):
> + t = threading.Thread(target=tidy_caller, args=(task_queue, lock,
> timeout))
> + t.daemon = True
> + t.start()
> +
> +def merge_replacement_files(tmpdir, mergefile):
> + """Merge all replacement files in a directory into a single file"""
> + # The fixes suggested by clang-tidy >= 4.0.0 are given under
> + # the top level key 'Diagnostics' in the output yaml files
> + mergekey = "Diagnostics"
> + merged = []
> + for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
> + content = yaml.safe_load(open(replacefile, 'r'))
> + if not content:
> + continue # Skip empty files.
> + merged.extend(content.get(mergekey, []))
> +
> + if merged:
> + # MainSourceFile: The key is required by the definition inside
> + # include/clang/Tooling/ReplacementsYaml.h, but the value
> + # is actually never used inside clang-apply-replacements,
> + # so we set it to '' here.
> + output = { 'MainSourceFile': '', mergekey: merged }
> + with open(mergefile, 'w') as out:
> + yaml.safe_dump(output, out)
> + else:
> + # Empty the file:
> + open(mergefile, 'w').close()
>
>
> def main():
> @@ -47,7 +127,10 @@ def main():
> r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
> help='custom pattern selecting file paths to check '
> '(case insensitive, overridden by -regex)')
> -
> + parser.add_argument('-j', type=int, default=1,
> + help='number of tidy instances to be run in
> parallel.')
> + parser.add_argument('-timeout', type=int, default=None,
> + help='timeout per each file in seconds.')
> parser.add_argument('-fix', action='store_true', default=False,
> help='apply suggested fixes')
> parser.add_argument('-checks',
> @@ -84,7 +167,7 @@ def main():
> match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p,
> line)
> if match:
> filename = match.group(2)
> - if filename == None:
> + if filename is None:
> continue
>
> if args.regex is not None:
> @@ -102,44 +185,79 @@ def main():
> line_count = int(match.group(3))
> if line_count == 0:
> continue
> - end_line = start_line + line_count - 1;
> + end_line = start_line + line_count - 1
> lines_by_file.setdefault(filename, []).append([start_line,
> end_line])
>
> - if len(lines_by_file) == 0:
> + if not any(lines_by_file):
> print("No relevant changes found.")
> sys.exit(0)
>
> - line_filter_json = json.dumps(
> - [{"name" : name, "lines" : lines_by_file[name]} for name in
> lines_by_file],
> - separators = (',', ':'))
> -
> - quote = "";
> - if sys.platform == 'win32':
> - line_filter_json=re.sub(r'"', r'"""', line_filter_json)
> - else:
> - quote = "'";
> + max_task_count = args.j
> + if max_task_count == 0:
> + max_task_count = multiprocessing.cpu_count()
> + max_task_count = min(len(lines_by_file), max_task_count)
>
> - # Run clang-tidy on files containing changes.
> - command = [args.clang_tidy_binary]
> - command.append('-line-filter=' + quote + line_filter_json + quote)
> - if args.fix:
> - command.append('-fix')
> + tmpdir = None
> if args.export_fixes:
> - command.append('-export-fixes=' + args.export_fixes)
> + tmpdir = tempfile.mkdtemp()
> +
> + # Tasks for clang-tidy.
> + task_queue = queue.Queue(max_task_count)
> + # A lock for console output.
> + lock = threading.Lock()
> +
> + # Run a pool of clang-tidy workers.
> + start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
> +
> + # Form the common args list.
> + common_clang_tidy_args = []
> + if args.fix:
> + common_clang_tidy_args.append('-fix')
> if args.checks != '':
> - command.append('-checks=' + quote + args.checks + quote)
> + common_clang_tidy_args.append('-checks=' + args.checks)
> if args.quiet:
> - command.append('-quiet')
> + common_clang_tidy_args.append('-quiet')
> if args.build_path is not None:
> - command.append('-p=%s' % args.build_path)
> - command.extend(lines_by_file.keys())
> + common_clang_tidy_args.append('-p=%s' % args.build_path)
> for arg in args.extra_arg:
> - command.append('-extra-arg=%s' % arg)
> + common_clang_tidy_args.append('-extra-arg=%s' % arg)
> for arg in args.extra_arg_before:
> - command.append('-extra-arg-before=%s' % arg)
> - command.extend(clang_tidy_args)
> + common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
> +
> + for name in lines_by_file:
> + line_filter_json = json.dumps(
> + [{"name": name, "lines": lines_by_file[name]}],
> + separators=(',', ':'))
> +
> + # Run clang-tidy on files containing changes.
> + command = [args.clang_tidy_binary]
> + command.append('-line-filter=' + line_filter_json)
> + if args.export_fixes:
> + # Get a temporary file. We immediately close the handle so
> clang-tidy can
> + # overwrite it.
> + (handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
> + os.close(handle)
> + command.append('-export-fixes=' + tmp_name)
> + command.extend(common_clang_tidy_args)
> + command.append(name)
> + command.extend(clang_tidy_args)
> +
> + task_queue.put(command)
> +
> + # Wait for all threads to be done.
> + task_queue.join()
> +
> + if args.export_fixes:
> + print('Writing fixes to ' + args.export_fixes + ' ...')
> + try:
> + merge_replacement_files(tmpdir, args.export_fixes)
> + except:
> + sys.stderr.write('Error exporting fixes.\n')
> + traceback.print_exc()
> +
> + if tmpdir:
> + shutil.rmtree(tmpdir)
>
> - sys.exit(subprocess.call(' '.join(command), shell=True))
>
> if __name__ == '__main__':
> main()
>
>
> _______________________________________________
> cfe-commits mailing list
> cfe-commits at lists.llvm.org
> https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20190320/45535987/attachment-0001.html>
More information about the cfe-commits
mailing list