[clang-tools-extra] r356565 - Reland r356547 after fixing the tests for Linux.

Galina Kistanova via cfe-commits cfe-commits at lists.llvm.org
Wed Mar 20 14:02:57 PDT 2019


Hello Zinovy,

This commit broke test on the next builder:
http://lab.llvm.org:8011/builders/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/builds/45557
. . .
Failing Tests (1):
    Clang Tools :: clang-tidy/clang-tidy-diff.cpp

Please have a look?

Thanks

Galina


On Wed, Mar 20, 2019 at 8:49 AM Zinovy Nis via cfe-commits <
cfe-commits at lists.llvm.org> wrote:

> Author: zinovy.nis
> Date: Wed Mar 20 08:50:26 2019
> New Revision: 356565
>
> URL: http://llvm.org/viewvc/llvm-project?rev=356565&view=rev
> Log:
> Reland r356547 after fixing the tests for Linux.
>
> [clang-tidy] Parallelize clang-tidy-diff.py
>
> This patch has 2 rationales:
>
> - large patches lead to long command lines and often cause max command
> line length restrictions imposed by OS;
> - clang-tidy runs on modified files are independent and can be done in
> parallel, the same as done for run-clang-tidy.
>
> Differential Revision: https://reviews.llvm.org/D57662
>
>
> Modified:
>     clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py
>
> Modified: clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py
> URL:
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py?rev=356565&r1=356564&r2=356565&view=diff
>
> ==============================================================================
> --- clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py (original)
> +++ clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py Wed Mar 20
> 08:50:26 2019
> @@ -24,10 +24,90 @@ Example usage for git/svn users:
>  """
>
>  import argparse
> +import glob
>  import json
> +import multiprocessing
> +import os
>  import re
> +import shutil
>  import subprocess
>  import sys
> +import tempfile
> +import threading
> +import traceback
> +import yaml
> +
> +is_py2 = sys.version[0] == '2'
> +
> +if is_py2:
> +    import Queue as queue
> +else:
> +    import queue as queue
> +
> +
> +def run_tidy(task_queue, lock, timeout):
> +  watchdog = None
> +  while True:
> +    command = task_queue.get()
> +    try:
> +      proc = subprocess.Popen(command,
> +                              stdout=subprocess.PIPE,
> +                              stderr=subprocess.PIPE)
> +
> +      if timeout is not None:
> +        watchdog = threading.Timer(timeout, proc.kill)
> +        watchdog.start()
> +
> +      stdout, stderr = proc.communicate()
> +
> +      with lock:
> +        sys.stdout.write(stdout.decode('utf-8') + '\n')
> +        sys.stdout.flush()
> +        if stderr:
> +          sys.stderr.write(stderr.decode('utf-8') + '\n')
> +          sys.stderr.flush()
> +    except Exception as e:
> +      with lock:
> +        sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
> +    finally:
> +      with lock:
> +        if (not timeout is None) and (not watchdog is None):
> +          if not watchdog.is_alive():
> +              sys.stderr.write('Terminated by timeout: ' +
> +                               ' '.join(command) + '\n')
> +          watchdog.cancel()
> +      task_queue.task_done()
> +
> +
> +def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
> +  for _ in range(max_tasks):
> +    t = threading.Thread(target=tidy_caller, args=(task_queue, lock,
> timeout))
> +    t.daemon = True
> +    t.start()
> +
> +def merge_replacement_files(tmpdir, mergefile):
> +  """Merge all replacement files in a directory into a single file"""
> +  # The fixes suggested by clang-tidy >= 4.0.0 are given under
> +  # the top level key 'Diagnostics' in the output yaml files
> +  mergekey = "Diagnostics"
> +  merged = []
> +  for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
> +    content = yaml.safe_load(open(replacefile, 'r'))
> +    if not content:
> +      continue # Skip empty files.
> +    merged.extend(content.get(mergekey, []))
> +
> +  if merged:
> +    # MainSourceFile: The key is required by the definition inside
> +    # include/clang/Tooling/ReplacementsYaml.h, but the value
> +    # is actually never used inside clang-apply-replacements,
> +    # so we set it to '' here.
> +    output = { 'MainSourceFile': '', mergekey: merged }
> +    with open(mergefile, 'w') as out:
> +      yaml.safe_dump(output, out)
> +  else:
> +    # Empty the file:
> +    open(mergefile, 'w').close()
>
>
>  def main():
> @@ -47,7 +127,10 @@ def main():
>                        r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
>                        help='custom pattern selecting file paths to check '
>                        '(case insensitive, overridden by -regex)')
> -
> +  parser.add_argument('-j', type=int, default=1,
> +                      help='number of tidy instances to be run in
> parallel.')
> +  parser.add_argument('-timeout', type=int, default=None,
> +                      help='timeout per each file in seconds.')
>    parser.add_argument('-fix', action='store_true', default=False,
>                        help='apply suggested fixes')
>    parser.add_argument('-checks',
> @@ -84,7 +167,7 @@ def main():
>      match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p,
> line)
>      if match:
>        filename = match.group(2)
> -    if filename == None:
> +    if filename is None:
>        continue
>
>      if args.regex is not None:
> @@ -102,44 +185,79 @@ def main():
>          line_count = int(match.group(3))
>        if line_count == 0:
>          continue
> -      end_line = start_line + line_count - 1;
> +      end_line = start_line + line_count - 1
>        lines_by_file.setdefault(filename, []).append([start_line,
> end_line])
>
> -  if len(lines_by_file) == 0:
> +  if not any(lines_by_file):
>      print("No relevant changes found.")
>      sys.exit(0)
>
> -  line_filter_json = json.dumps(
> -    [{"name" : name, "lines" : lines_by_file[name]} for name in
> lines_by_file],
> -    separators = (',', ':'))
> -
> -  quote = "";
> -  if sys.platform == 'win32':
> -    line_filter_json=re.sub(r'"', r'"""', line_filter_json)
> -  else:
> -    quote = "'";
> +  max_task_count = args.j
> +  if max_task_count == 0:
> +      max_task_count = multiprocessing.cpu_count()
> +  max_task_count = min(len(lines_by_file), max_task_count)
>
> -  # Run clang-tidy on files containing changes.
> -  command = [args.clang_tidy_binary]
> -  command.append('-line-filter=' + quote + line_filter_json + quote)
> -  if args.fix:
> -    command.append('-fix')
> +  tmpdir = None
>    if args.export_fixes:
> -    command.append('-export-fixes=' + args.export_fixes)
> +    tmpdir = tempfile.mkdtemp()
> +
> +  # Tasks for clang-tidy.
> +  task_queue = queue.Queue(max_task_count)
> +  # A lock for console output.
> +  lock = threading.Lock()
> +
> +  # Run a pool of clang-tidy workers.
> +  start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
> +
> +  # Form the common args list.
> +  common_clang_tidy_args = []
> +  if args.fix:
> +    common_clang_tidy_args.append('-fix')
>    if args.checks != '':
> -    command.append('-checks=' + quote + args.checks + quote)
> +    common_clang_tidy_args.append('-checks=' + args.checks)
>    if args.quiet:
> -    command.append('-quiet')
> +    common_clang_tidy_args.append('-quiet')
>    if args.build_path is not None:
> -    command.append('-p=%s' % args.build_path)
> -  command.extend(lines_by_file.keys())
> +    common_clang_tidy_args.append('-p=%s' % args.build_path)
>    for arg in args.extra_arg:
> -      command.append('-extra-arg=%s' % arg)
> +    common_clang_tidy_args.append('-extra-arg=%s' % arg)
>    for arg in args.extra_arg_before:
> -      command.append('-extra-arg-before=%s' % arg)
> -  command.extend(clang_tidy_args)
> +    common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
> +
> +  for name in lines_by_file:
> +    line_filter_json = json.dumps(
> +      [{"name": name, "lines": lines_by_file[name]}],
> +      separators=(',', ':'))
> +
> +    # Run clang-tidy on files containing changes.
> +    command = [args.clang_tidy_binary]
> +    command.append('-line-filter=' + line_filter_json)
> +    if args.export_fixes:
> +      # Get a temporary file. We immediately close the handle so
> clang-tidy can
> +      # overwrite it.
> +      (handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
> +      os.close(handle)
> +      command.append('-export-fixes=' + tmp_name)
> +    command.extend(common_clang_tidy_args)
> +    command.append(name)
> +    command.extend(clang_tidy_args)
> +
> +    task_queue.put(command)
> +
> +  # Wait for all threads to be done.
> +  task_queue.join()
> +
> +  if args.export_fixes:
> +    print('Writing fixes to ' + args.export_fixes + ' ...')
> +    try:
> +      merge_replacement_files(tmpdir, args.export_fixes)
> +    except:
> +      sys.stderr.write('Error exporting fixes.\n')
> +      traceback.print_exc()
> +
> +  if tmpdir:
> +    shutil.rmtree(tmpdir)
>
> -  sys.exit(subprocess.call(' '.join(command), shell=True))
>
>  if __name__ == '__main__':
>    main()
>
>
> _______________________________________________
> cfe-commits mailing list
> cfe-commits at lists.llvm.org
> https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20190320/45535987/attachment-0001.html>


More information about the cfe-commits mailing list