[llvm] d9542db - [UpdateTestChecks] Share the code to parse RUN: lines between all scripts
Alex Richardson via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 2 03:06:43 PST 2019
Author: Alex Richardson
Date: 2019-12-02T11:06:30Z
New Revision: d9542db49e90457de62af3bfe395aaf4c47b68a5
URL: https://github.com/llvm/llvm-project/commit/d9542db49e90457de62af3bfe395aaf4c47b68a5
DIFF: https://github.com/llvm/llvm-project/commit/d9542db49e90457de62af3bfe395aaf4c47b68a5.diff
LOG: [UpdateTestChecks] Share the code to parse RUN: lines between all scripts
Summary:
This commit also introduces a common.debug() function to avoid many
`if args.verbose:` statements. Depends on D70428.
Reviewers: xbolva00, MaskRay, jdoerfert
Reviewed By: MaskRay
Subscribers: llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D70432
Added:
Modified:
llvm/utils/UpdateTestChecks/common.py
llvm/utils/update_analyze_test_checks.py
llvm/utils/update_cc_test_checks.py
llvm/utils/update_llc_test_checks.py
llvm/utils/update_mca_test_checks.py
llvm/utils/update_mir_test_checks.py
llvm/utils/update_test_checks.py
Removed:
################################################################################
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index f06460510905..dfb3b16ae6bb 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -14,12 +14,17 @@ class string:
##### Common utilities for update_*test_checks.py
+_verbose = False
+
def parse_commandline_args(parser):
parser.add_argument('-v', '--verbose', action='store_true',
help='Show verbose output')
parser.add_argument('-u', '--update-only', action='store_true',
help='Only update test if it was already autogened')
- return parser.parse_args()
+ args = parser.parse_args()
+ global _verbose
+ _verbose = args.verbose
+ return args
def should_add_line_to_output(input_line, prefix_set):
# Skip any blank comment lines in the IR.
@@ -53,7 +58,7 @@ def invoke_tool(exe, cmd_args, ir):
##### LLVM IR parser
-RUN_LINE_RE = re.compile(r'^\s*[;#]\s*RUN:\s*(.*)$')
+RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$')
CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)')
PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$')
CHECK_RE = re.compile(r'^\s*[;#]\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME)?:')
@@ -91,6 +96,28 @@ def warn(msg, test_file=None):
msg = '{}: {}'.format(msg, test_file)
print('WARNING: {}'.format(msg), file=sys.stderr)
+def debug(*args, **kwargs):
+ # Python2 does not allow def debug(*args, file=sys.stderr, **kwargs):
+ if 'file' not in kwargs:
+ kwargs['file'] = sys.stderr
+ if _verbose:
+ print(*args, **kwargs)
+
+def find_run_lines(test, lines):
+ debug('Scanning for RUN lines in test file:', test)
+ raw_lines = [m.group(1)
+ for m in [RUN_LINE_RE.match(l) for l in lines] if m]
+ run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
+ for l in raw_lines[1:]:
+ if run_lines[-1].endswith('\\'):
+ run_lines[-1] = run_lines[-1].rstrip('\\' + ' ' + l)
+ else:
+ run_lines.append(l)
+ debug('Found {} RUN lines in {}:'.format(len(run_lines), test))
+ for l in run_lines:
+ debug(' RUN: {}'.format(l))
+ return run_lines
+
def scrub_body(body):
# Scrub runs of whitespace out of the assembly, but leave the leading
# whitespace in place.
diff --git a/llvm/utils/update_analyze_test_checks.py b/llvm/utils/update_analyze_test_checks.py
index 37803656aa20..f3572e762549 100755
--- a/llvm/utils/update_analyze_test_checks.py
+++ b/llvm/utils/update_analyze_test_checks.py
@@ -69,8 +69,6 @@ def main():
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
for test in test_paths:
- if args.verbose:
- print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
with open(test) as f:
input_lines = [l.rstrip() for l in f]
@@ -84,20 +82,7 @@ def main():
common.warn("Skipping test which isn't autogenerated: " + test)
continue
- raw_lines = [m.group(1)
- for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
- run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
- for l in raw_lines[1:]:
- if run_lines[-1].endswith("\\"):
- run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
- else:
- run_lines.append(l)
-
- if args.verbose:
- print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
- for l in run_lines:
- print(' RUN: ' + l, file=sys.stderr)
-
+ run_lines = common.find_run_lines(test, input_lines)
prefix_list = []
for l in run_lines:
if '|' not in l:
@@ -132,9 +117,8 @@ def main():
for prefix in prefixes:
func_dict.update({prefix: dict()})
for prefixes, opt_args in prefix_list:
- if args.verbose:
- print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
- print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
+ common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr)
+ common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr)
raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test)
@@ -147,8 +131,7 @@ def main():
is_in_function = False
is_in_function_start = False
prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
- if args.verbose:
- print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
+ common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr)
output_lines = []
output_lines.append(autogenerated_note)
@@ -194,8 +177,7 @@ def main():
continue
is_in_function = is_in_function_start = True
- if args.verbose:
- print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
+ common.debug('Writing %d lines to %s...' % (len(output_lines), test))
with open(test, 'wb') as f:
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py
index a3227c67743f..414056fa6fdd 100755
--- a/llvm/utils/update_cc_test_checks.py
+++ b/llvm/utils/update_cc_test_checks.py
@@ -29,7 +29,6 @@
ADVERT = '// NOTE: Assertions have been autogenerated by '
CHECK_RE = re.compile(r'^\s*//\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
-RUN_LINE_RE = re.compile(r'^//\s*RUN:\s*(.*)$')
SUBST = {
'%clang': [],
@@ -38,9 +37,6 @@
}
def get_line2spell_and_mangled(args, clang_args):
- def debug_mangled(*print_args, **kwargs):
- if args.verbose:
- print(*print_args, file=sys.stderr, **kwargs)
ret = {}
# Use clang's JSON AST dump to get the mangled name
json_dump_args = [args.clang, *clang_args, '-fsyntax-only', '-o', '-']
@@ -49,7 +45,7 @@ def debug_mangled(*print_args, **kwargs):
# -Xclang -ast-dump=json instead:
json_dump_args.append('-Xclang')
json_dump_args.append('-ast-dump=json')
- debug_mangled('Running', ' '.join(json_dump_args))
+ common.debug('Running', ' '.join(json_dump_args))
status = subprocess.run(json_dump_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if status.returncode != 0:
sys.stderr.write('Failed to run ' + ' '.join(json_dump_args) + '\n')
@@ -67,20 +63,19 @@ def debug_mangled(*print_args, **kwargs):
if node['kind'] != 'FunctionDecl':
continue
if node.get('isImplicit') is True and node.get('storageClass') == 'extern':
- debug_mangled('Skipping builtin function:', node['name'], '@', node['loc'])
+ common.debug('Skipping builtin function:', node['name'], '@', node['loc'])
continue
- debug_mangled('Found function:', node['kind'], node['name'], '@', node['loc'])
+ common.debug('Found function:', node['kind'], node['name'], '@', node['loc'])
line = node['loc'].get('line')
# If there is no line it is probably a builtin function -> skip
if line is None:
- debug_mangled('Skipping function without line number:', node['name'], '@', node['loc'])
+ common.debug('Skipping function without line number:', node['name'], '@', node['loc'])
continue
spell = node['name']
mangled = node.get('mangledName', spell)
ret[int(line)-1] = (spell, mangled)
- if args.verbose:
- for line, func_name in sorted(ret.items()):
- print('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
+ for line, func_name in sorted(ret.items()):
+ common.debug('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
if not ret:
common.warn('Did not find any functions using', ' '.join(json_dump_args))
return ret
@@ -191,19 +186,7 @@ def main():
continue
# Extract RUN lines.
- raw_lines = [m.group(1)
- for m in [RUN_LINE_RE.match(l) for l in input_lines] if m]
- run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
- for l in raw_lines[1:]:
- if run_lines[-1].endswith("\\"):
- run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
- else:
- run_lines.append(l)
-
- if args.verbose:
- print('Found {} RUN lines:'.format(len(run_lines)), file=sys.stderr)
- for l in run_lines:
- print(' RUN: ' + l, file=sys.stderr)
+ run_lines = common.find_run_lines(filename, input_lines)
# Build a list of clang command lines and check prefixes from RUN lines.
run_list = []
@@ -260,9 +243,8 @@ def main():
for prefix in prefixes:
func_dict.update({prefix: dict()})
for prefixes, clang_args, extra_commands, triple_in_cmd in run_list:
- if args.verbose:
- print('Extracted clang cmd: clang {}'.format(clang_args), file=sys.stderr)
- print('Extracted FileCheck prefixes: {}'.format(prefixes), file=sys.stderr)
+ common.debug('Extracted clang cmd: clang {}'.format(clang_args))
+ common.debug('Extracted FileCheck prefixes: {}'.format(prefixes))
get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict)
diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py
index 1168eec9a33c..750650f8640c 100755
--- a/llvm/utils/update_llc_test_checks.py
+++ b/llvm/utils/update_llc_test_checks.py
@@ -43,8 +43,6 @@ def main():
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
for test in test_paths:
- if args.verbose:
- print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
with open(test) as f:
input_lines = [l.rstrip() for l in f]
@@ -65,20 +63,7 @@ def main():
triple_in_ir = m.groups()[0]
break
- raw_lines = [m.group(1)
- for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
- run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
- for l in raw_lines[1:]:
- if run_lines[-1].endswith("\\"):
- run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
- else:
- run_lines.append(l)
-
- if args.verbose:
- print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
- for l in run_lines:
- print(' RUN: ' + l, file=sys.stderr)
-
+ run_lines = common.find_run_lines(test, input_lines)
run_list = []
for l in run_lines:
if '|' not in l:
@@ -115,12 +100,6 @@ def main():
llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip()
if test.endswith('.mir'):
llc_cmd_args += ' -x mir'
- comment_sym = '#'
- check_indent = ' '
- else:
- comment_sym = ';'
- check_indent = ''
-
check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
for item in m.group(1).split(',')]
if not check_prefixes:
@@ -130,6 +109,12 @@ def main():
# now, we just ignore all but the last.
run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd))
+ if test.endswith('.mir'):
+ comment_sym = '#'
+ check_indent = ' '
+ else:
+ comment_sym = ';'
+ check_indent = ''
autogenerated_note = (comment_sym + ADVERT + 'utils/' + script_name)
func_dict = {}
@@ -138,9 +123,8 @@ def main():
for prefix in prefixes:
func_dict.update({prefix: dict()})
for prefixes, llc_args, triple_in_cmd, march_in_cmd in run_list:
- if args.verbose:
- print('Extracted LLC cmd: ' + llc_tool + ' ' + llc_args, file=sys.stderr)
- print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
+ common.debug('Extracted LLC cmd:', llc_tool, llc_args)
+ common.debug('Extracted FileCheck prefixes:', str(prefixes))
raw_tool_output = common.invoke_tool(args.llc_binary, llc_args, test)
triple = triple_in_cmd or triple_in_ir
@@ -154,8 +138,7 @@ def main():
is_in_function_start = False
func_name = None
prefix_set = set([prefix for p in run_list for prefix in p[0]])
- if args.verbose:
- print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
+ common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
output_lines = []
output_lines.append(autogenerated_note)
@@ -199,8 +182,7 @@ def main():
continue
is_in_function = is_in_function_start = True
- if args.verbose:
- print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
+ common.debug('Writing %d lines to %s...' % (len(output_lines), test))
with open(test, 'wb') as f:
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
diff --git a/llvm/utils/update_mca_test_checks.py b/llvm/utils/update_mca_test_checks.py
index ba0a99392e02..0522c80be4d5 100755
--- a/llvm/utils/update_mca_test_checks.py
+++ b/llvm/utils/update_mca_test_checks.py
@@ -83,26 +83,6 @@ def _parse_args():
return args
-def _find_run_lines(input_lines, args):
- raw_lines = [m.group(1)
- for m in [common.RUN_LINE_RE.match(l) for l in input_lines]
- if m]
- run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
- for l in raw_lines[1:]:
- if run_lines[-1].endswith(r'\\'):
- run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
- else:
- run_lines.append(l)
-
- if args.verbose:
- sys.stderr.write('Found {} RUN line{}:\n'.format(
- len(run_lines), '' if len(run_lines) == 1 else 's'))
- for line in run_lines:
- sys.stderr.write(' RUN: {}\n'.format(line))
-
- return run_lines
-
-
def _get_run_infos(run_lines, args):
run_infos = []
for run_line in run_lines:
@@ -544,9 +524,7 @@ def _write_output(test_path, input_lines, prefix_list, block_infos, # noqa
return
sys.stderr.write(' [{} lines total]\n'.format(len(output_lines)))
- if args.verbose:
- sys.stderr.write(
- 'Writing {} lines to {}...\n\n'.format(len(output_lines), test_path))
+ common.debug('Writing', len(output_lines), 'lines to', test_path, '..\n\n')
with open(test_path, 'wb') as f:
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
@@ -562,17 +540,13 @@ def main():
# will be written once per source location per test.
_configure_warnings(args)
- if args.verbose:
- sys.stderr.write(
- 'Scanning for RUN lines in test file: {}\n'.format(test_path))
-
if not os.path.isfile(test_path):
raise Error('could not find test file: {}'.format(test_path))
with open(test_path) as f:
input_lines = [l.rstrip() for l in f]
- run_lines = _find_run_lines(input_lines, args)
+ run_lines = common.find_run_lines(test_path, input_lines)
run_infos = _get_run_infos(run_lines, args)
common_prefix, prefix_pad = _get_useful_prefix_info(run_infos)
block_infos = _get_block_infos(run_infos, test_path, args, common_prefix)
diff --git a/llvm/utils/update_mir_test_checks.py b/llvm/utils/update_mir_test_checks.py
index 6e9061309585..46f497007fc9 100755
--- a/llvm/utils/update_mir_test_checks.py
+++ b/llvm/utils/update_mir_test_checks.py
@@ -96,22 +96,6 @@ def find_triple_in_ir(lines, verbose=False):
return None
-def find_run_lines(test, lines, verbose=False):
- raw_lines = [m.group(1)
- for m in [common.RUN_LINE_RE.match(l) for l in lines] if m]
- run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
- for l in raw_lines[1:]:
- if run_lines[-1].endswith("\\"):
- run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
- else:
- run_lines.append(l)
- if verbose:
- log('Found {} RUN lines:'.format(len(run_lines)))
- for l in run_lines:
- log(' RUN: {}'.format(l))
- return run_lines
-
-
def build_run_list(test, run_lines, verbose=False):
run_list = []
all_prefixes = []
@@ -296,7 +280,6 @@ def should_add_line_to_output(input_line, prefix_set):
def update_test_file(args, test):
- log('Scanning for RUN lines in test file: {}'.format(test), args.verbose)
with open(test) as fd:
input_lines = [l.rstrip() for l in fd]
@@ -313,7 +296,7 @@ def update_test_file(args, test):
return
triple_in_ir = find_triple_in_ir(input_lines, args.verbose)
- run_lines = find_run_lines(test, input_lines, args.verbose)
+ run_lines = common.find_run_lines(test, input_lines)
run_list, common_prefixes = build_run_list(test, run_lines, args.verbose)
simple_functions = find_functions_with_one_bb(input_lines, args.verbose)
diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py
index 31122b2f7b75..3fd8dd7dd7e9 100755
--- a/llvm/utils/update_test_checks.py
+++ b/llvm/utils/update_test_checks.py
@@ -84,8 +84,6 @@ def main():
# On Windows we must expand the patterns ourselves.
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
for test in test_paths:
- if args.verbose:
- print('Scanning for RUN lines in test file: ' + test, file=sys.stderr)
with open(test) as f:
input_lines = [l.rstrip() for l in f]
@@ -99,20 +97,7 @@ def main():
common.warn("Skipping test which isn't autogenerated: " + test)
continue
- raw_lines = [m.group(1)
- for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
- run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
- for l in raw_lines[1:]:
- if run_lines[-1].endswith('\\'):
- run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
- else:
- run_lines.append(l)
-
- if args.verbose:
- print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
- for l in run_lines:
- print(' RUN: ' + l, file=sys.stderr)
-
+ run_lines = common.find_run_lines(test, input_lines)
prefix_list = []
for l in run_lines:
if '|' not in l:
@@ -146,9 +131,8 @@ def main():
for prefix in prefixes:
func_dict.update({prefix: dict()})
for prefixes, opt_args in prefix_list:
- if args.verbose:
- print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
- print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
+ common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
+ common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
common.build_function_body_dictionary(
@@ -159,8 +143,7 @@ def main():
is_in_function = False
is_in_function_start = False
prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
- if args.verbose:
- print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
+ common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
output_lines = []
output_lines.append(autogenerated_note)
@@ -207,8 +190,7 @@ def main():
continue
is_in_function = is_in_function_start = True
- if args.verbose:
- print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
+ common.debug('Writing %d lines to %s...' % (len(output_lines), test))
with open(test, 'wb') as f:
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
More information about the llvm-commits
mailing list