<div dir="ltr">Hi Aaron,<div><br></div><div>I tried using this script to update a json dump test, and it seems to not really work very well:</div><div><br></div><div>1) it's really inconvenient to invoke; you need to pass in a --clang, copy some options out from two different places in the test file (from the RUN line and from a "--filters" line), and guess how to form the proper command line for the tool</div><div>2) it generates a file with the wrong name (adding a -json before the extension)</div><div>3) it doesn't strip out the CHECK lines from the previous run of the tool</div><div>4) it adds in a bunch of trailing whitespace, causing the file to not match the one in the repository</div><div><br></div><div>Have you had a chance to look at making this more usable? I think the approach taken by utils/make-ast-dump-check.sh helps a lot: run the test in its normal configuration to generate the output, and then replace FileCheck with a tool that updates the CHECK lines instead of checking them. (That saves you needing to pass in --clang and --opts, at least, and makes it really easy to update a whole bunch of tests at once by running them all with lit.)</div></div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Wed, 19 Jun 2019 at 08:21, Aaron Ballman via cfe-commits <<a href="mailto:cfe-commits@lists.llvm.org">cfe-commits@lists.llvm.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Author: aaronballman<br>
Date: Wed Jun 19 08:25:24 2019<br>
New Revision: 363820<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=363820&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=363820&view=rev</a><br>
Log:<br>
Add a script to help generate expected test output for dumping the AST to JSON.<br>
<br>
Patch by Abhishek Bhaskar.<br>
<br>
Added:<br>
cfe/trunk/test/AST/gen_ast_dump_json_test.py<br>
<br>
Added: cfe/trunk/test/AST/gen_ast_dump_json_test.py<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/cfe/trunk/test/AST/gen_ast_dump_json_test.py?rev=363820&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/cfe/trunk/test/AST/gen_ast_dump_json_test.py?rev=363820&view=auto</a><br>
==============================================================================<br>
--- cfe/trunk/test/AST/gen_ast_dump_json_test.py (added)<br>
+++ cfe/trunk/test/AST/gen_ast_dump_json_test.py Wed Jun 19 08:25:24 2019<br>
@@ -0,0 +1,137 @@<br>
+#!/usr/bin/env python<br>
+<br>
+from collections import OrderedDict<br>
+from sets import Set<br>
+from shutil import copyfile<br>
+import argparse<br>
+import json<br>
+import os<br>
+import pprint<br>
+import re<br>
+import subprocess<br>
+ <br>
+def normalize(dict_var):<br>
+ for k, v in dict_var.items():<br>
+ if isinstance(v, OrderedDict):<br>
+ normalize(v)<br>
+ elif isinstance(v, list):<br>
+ for e in v:<br>
+ if isinstance(e, OrderedDict):<br>
+ normalize(e)<br>
+ elif type(v) is unicode:<br>
+ st = v.encode('utf-8')<br>
+ if re.match(r"0x[0-9A-Fa-f]+", v):<br>
+ dict_var[k] = u'0x{{.*}}'<br>
+ elif os.path.isfile(v):<br>
+ dict_var[k] = u'{{.*}}'<br>
+ else:<br>
+ splits = (v.split(u' '))<br>
+ out_splits = []<br>
+ for split in splits:<br>
+ inner_splits = split.rsplit(u':',2)<br>
+ if os.path.isfile(inner_splits[0]):<br>
+ out_splits.append(<br>
+ u'{{.*}}:%s:%s'<br>
+ %(inner_splits[1],<br>
+ inner_splits[2]))<br>
+ continue<br>
+ out_splits.append(split)<br>
+<br>
+ dict_var[k] = ' '.join(out_splits)<br>
+<br>
+def filter_json(dict_var, filters, out):<br>
+ for k, v in dict_var.items():<br>
+ if type(v) is unicode:<br>
+ st = v.encode('utf-8')<br>
+ if st in filters:<br>
+ out.append(dict_var)<br>
+ break<br>
+ elif isinstance(v, OrderedDict):<br>
+ filter_json(v, filters, out)<br>
+ elif isinstance(v, list):<br>
+ for e in v:<br>
+ if isinstance(e, OrderedDict):<br>
+ filter_json(e, filters, out)<br>
+ <br>
+def main():<br>
+ parser = argparse.ArgumentParser()<br>
+ parser.add_argument("--clang", help="The clang binary (could be a relative or absolute path)",<br>
+ action="store", required=True)<br>
+ parser.add_argument("--opts", help="other options",<br>
+ action="store", default='', type=str)<br>
+ parser.add_argument("--source", help="the source file. Command used to generate the json will be of the format <clang> -cc1 -ast-dump=json <opts> <source>",<br>
+ action="store", required=True)<br>
+ parser.add_argument("--filters", help="comma separated list of AST filters. Ex: --filters=TypedefDecl,BuiltinType",<br>
+ action="store", default='')<br>
+<br>
+ args = parser.parse_args()<br>
+<br>
+ if not args.source:<br>
+ print("Specify the source file to give to clang.")<br>
+ return -1<br>
+<br>
+ clang_binary = os.path.abspath(args.clang)<br>
+ if not os.path.isfile(clang_binary):<br>
+ print("clang binary specified not present.")<br>
+ return -1<br>
+<br>
+ options = args.opts.split(' ')<br>
+ filters = Set(args.filters.split(',')) if args.filters else Set([])<br>
+ <br>
+ cmd = [clang_binary, "-cc1"]<br>
+ cmd.extend(options)<br>
+<br>
+ using_ast_dump_filter = 'ast-dump-filter' in args.opts<br>
+ <br>
+ cmd.extend(["-ast-dump=json", args.source])<br>
+<br>
+ try:<br>
+ json_str = subprocess.check_output(cmd)<br>
+ except Exception as ex:<br>
+ print("The clang command failed with %s" % ex)<br>
+ return -1<br>
+ <br>
+ out_asts = []<br>
+ if using_ast_dump_filter:<br>
+ splits = re.split('Dumping .*:\n', json_str)<br>
+ if len(splits) > 1:<br>
+ for split in splits[1:]:<br>
+ j = json.loads(split.decode('utf-8'), object_pairs_hook=OrderedDict)<br>
+ normalize(j)<br>
+ out_asts.append(j)<br>
+ else:<br>
+ j = json.loads(json_str.decode('utf-8'), object_pairs_hook=OrderedDict)<br>
+ normalize(j)<br>
+<br>
+ if len(filters) == 0:<br>
+ out_asts.append(j)<br>
+ else:<br>
+ #assert using_ast_dump_filter is False,\<br>
+ # "Does not support using compiler's ast-dump-filter "\<br>
+ # "and the tool's filter option at the same time yet."<br>
+ <br>
+ filter_json(j, filters, out_asts)<br>
+ <br>
+ partition = args.source.rpartition('.')<br>
+ dest_path = '%s-json%s%s' % (partition[0], partition[1], partition[2])<br>
+<br>
+ print("Writing json appended source file to %s." %(dest_path))<br>
+ copyfile(args.source, dest_path) <br>
+ with open(dest_path, "a") as f:<br>
+ for out_ast in out_asts:<br>
+ append_str = json.dumps(out_ast, indent=1, ensure_ascii=False)<br>
+ out_str = '\n\n'<br>
+ index = 0<br>
+ for append_line in append_str.splitlines()[2:]:<br>
+ if index == 0:<br>
+ out_str += '// CHECK: %s\n' %(append_line)<br>
+ index += 1<br>
+ else:<br>
+ out_str += '// CHECK-NEXT: %s\n' %(append_line)<br>
+ <br>
+ f.write(out_str)<br>
+ <br>
+ return 0<br>
+ <br>
+if __name__ == '__main__':<br>
+ main()<br>
<br>
<br>
_______________________________________________<br>
cfe-commits mailing list<br>
<a href="mailto:cfe-commits@lists.llvm.org" target="_blank">cfe-commits@lists.llvm.org</a><br>
<a href="https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits" rel="noreferrer" target="_blank">https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits</a><br>
</blockquote></div>