[flang-commits] [clang] [flang] [Clang][Driver][Test] Created test for unsupported driver options (PR #120900)
Mészáros Gergely via flang-commits
flang-commits at lists.llvm.org
Mon Mar 24 14:45:46 PDT 2025
================
@@ -0,0 +1,534 @@
+#!/usr/bin/env python3
+
+"""generate_unsupported_in_drivermode.py
+
+This script generates Lit regression test files that validate that options are only exposed to intended driver modes.
+
+The options and driver modes are parsed from Options.td, whose path should be provided on the command line.
+See clang/include/clang/Driver/Options.td
+
+The LLVM TableGen executable can optionally be provided along with the path to the LLVM build tree bin directory.
+Otherwise, the script will search for it.
+
+The primary maintenance task for this script would be updating the expected return message for a driver mode if
+there are changes over time. See the instantiations of DriverData, specifically the check_str.
+
+Logic:
+1) For each option, (records of class "Option"), and for each driver, (records of class "OptionVisibility")
+ a. if the option's "Visibility" field includes the driver flavour, skip processing this option for this driver
+ b. if the option is part of an option group, (the record has the "Group" property),
+ and the group's "Visibility" field includes the driver flavour, skip processing this option for this driver
+ c. otherwise this option is not supported by this driver flavour, and this pairing is saved for testing
+2) For each unsupported pairing, generate a Lit RUN line, and a CHECK line to parse for expected output. Ex: "error: unknown argument"
+"""
+
+import os
+import json
+import subprocess
+from bisect import bisect_left
+from dataclasses import dataclass
+import argparse
+import dataclasses
+import itertools
+
+# Strings defined in Options.td for the various driver flavours. See "OptionVisibility"
+VISIBILITY_CC1AS = "CC1AsOption"
+VISIBILITY_CC1 = "CC1Option"
+VISIBILITY_CL = "CLOption"
+VISIBILITY_DXC = "DXCOption"
+VISIBILITY_DEFAULT = "DefaultVis"
+VISIBILITY_FC1 = "FC1Option"
+VISIBILITY_FLANG = "FlangOption"
+
+# Lit test prefix strings
+SLASH_SLASH = "// "
+EXCLAMATION = "! "
+
+exceptions_sequence = [
+ # Invalid usage of the driver options below causes unique output, so skip testing
+ "cc1",
+ "cc1as",
+ # There is currently a bug with "_no_warnings", i.e. --no-warnings. Diagnostic related options
+ # are parsed first, and always with CC1 visibility. They're used to set up the diagnostic
+ # engine, which parses "_no_warnings" (and its alias "w", i.e. -w) and sets an internal flag
+ # that suppresses all warnings.
+ "_no_warnings",
+ "w",
+]
+
+
+ at dataclass
+class DriverOption:
+ """Defines an unsupported driver-option combination
+ driver: The driver string as defined by OptionVisibility in Options.td
+ option: The option object from Options.td
+ option_name: Corresponding string for an option. See "Name" for a given option in Options.td
+ prefix: String that precedes the option. Ex. "-"
+ """
+
+ driver: str
+ option: str
+ option_name: str
+ prefix: str
+
+
+ at dataclass
+class DriverData:
+ """Dataclass for data specific to each driver
+ lit_cmd_prefix: The beginning string of the Lit command
+ lit_cmd_options: Strings containing additional options for this driver
+ visibility_str: The corresponding visibility string from OptionVisibility in Options.td
+ lit_cmd_end: String at the end of the Lit command
+ check_str: The string or regex to be sent to FileCheck
+ supported_joined_option_sequence: List of DriverOption objects for supported options
+ that are Kind *JOINED*, as defined in Options.td
+ supported_non_joined_option_sequence: List of DriverOption objects for supported options
+ that are not Kind *JOINED*, as defined in Options.td
+ test_option_sequence: A list of all the prefix-option pairs that will be tested for this driver
+ """
+
+ lit_cmd_prefix: str
+ lit_cmd_options: str
+ visibility_str: str
+ lit_cmd_end: str
+ check_str: str
+ supported_joined_option_sequence: list[DriverOption] = dataclasses.field(
+ default_factory=list
+ )
+ supported_non_joined_option_sequence: list[DriverOption] = dataclasses.field(
+ default_factory=list
+ )
+ test_option_sequence: list[str] = dataclasses.field(default_factory=list)
+
+
+def collect_transitive_groups(member, options_dictionary):
+ """Find the groups for a given member, where a member can be an option or a group.
+ Note that groups can themselves be part of groups, hence the recursion
+
+ For example, considering option 'C', it has the following 'Group' field as defined by Options.td:
+ "C": {
+ "Group": {
+ "def": "Preprocessor_Group",
+ // ...
+ },
+ // ...
+ },
+ 'Preprocessor_Group' is itself part of 'CompileOnly_Group', so option 'C' would be part of both groups
+ "Preprocessor_Group": {
+ // ...
+ "Group": {
+ "def": "CompileOnly_Group",
+ // ...
+ },
+ // ...
+ },
+
+ member: An option object or group object from Options.td.
+ options_dictionary: The converted Python dictionary from the Options.td json string
+
+ Return: A set including the group(s) found for the member. If no groups found, returns an empty set
+ """
+ parent_field = options_dictionary[member]["Group"]
+ if parent_field is None:
+ return set()
+
+ parent_name = parent_field["def"]
+ return {parent_name} | collect_transitive_groups(parent_name, options_dictionary)
+
+
+def get_visibility(option):
+ """Get a list of drivers that a given option is exposed to
+ option: The option object from Options.td
+ Return: Set that contains the visibilities of the given option
+ """
+ visibility_set = set()
+ # Check for the option's explicit visibility
+ for visibility in options_dictionary[option]["Visibility"]:
+ if visibility is not None:
+ visibility_set.add(visibility["def"])
+
+ # Check for the option's group's visibility
+ group_set = collect_transitive_groups(option, options_dictionary)
+ if group_set:
+ for group_name in group_set:
+ for visibility in options_dictionary[group_name]["Visibility"]:
+ visibility_set.add(visibility["def"])
+
+ return visibility_set
+
+
+def get_lit_test_note(test_visibility):
+ """Return the note to be included at the start of the Lit test file
+ test_visibility: Any VISIBILITY_* variable. VISIBILITY_DEFAULT will return the .c formatted test note.
+ All other will return the .f90 formatted test note
+ """
+ test_prefix = SLASH_SLASH if test_visibility == VISIBILITY_DEFAULT else EXCLAMATION
+
+ return (
+ f"{test_prefix}NOTE: This lit test was automatically generated to validate "
+ "unintentionally exposed arguments to various driver flavours.\n"
+ f"{test_prefix}NOTE: To make changes, see llvm-project/clang/utils/generate_unsupported_in_drivermode.py"
+ + " from which it was generated.\n"
+ f"{test_prefix}NOTE: Regenerate this Lit test with the following:\n"
+ f"{test_prefix}NOTE: python generate_unsupported_in_drivermode.py "
+ "--options-td-dir llvm-project/clang/include/clang/Driver --llvm-include-dir llvm-project/llvm/include --llvm-tblgen llvm-project/build/bin/llvm-tblgen\n\n"
+ )
+
+
+def write_lit_test(test_path, test_visibility):
+ """Write the Lit tests to file
+ test_path: File write path
+ test_visibility: VISIBILITY_DEFAULT, VISIBILITY_FLANG, or VISIBILITY_FC1 which indicates whether to write
+ to the main Lit test file, the flang test file, or the flang -fc1 test file
+ """
+ lit_file = open(test_path, "w")
+
+ lit_file.write(get_lit_test_note(test_visibility))
+ batch_size = 100
+
+ for visibility, driver_data in driver_data_dict.items():
+ is_flang_pair = visibility == VISIBILITY_FLANG or visibility == VISIBILITY_FC1
+
+ if (
+ (test_visibility == VISIBILITY_FLANG and visibility != VISIBILITY_FLANG)
+ or (test_visibility == VISIBILITY_FC1 and visibility != VISIBILITY_FC1)
+ or (test_visibility == VISIBILITY_DEFAULT and is_flang_pair)
+ ):
+ continue
+
+ comment_str = EXCLAMATION if is_flang_pair else SLASH_SLASH
+
+ unflattened_option_data = list(
+ itertools.batched(driver_data.test_option_sequence, batch_size)
+ )
+
+ for i, batch in enumerate(unflattened_option_data):
+ # Example run line: "// RUN: not %clang -cc1 -A ... -x c++ - < /dev/null 2>&1 | FileCheck -check-prefix=CC1OptionCHECK0 %s"
+ run_cmd = (
+ f"\n{comment_str}RUN: not " + driver_data.lit_cmd_prefix
+ ) # "// RUN: not %clang -cc1 "
+
+ # // RUN: <command up to this point> \
+ # // RUN: --one-option \
+ # // RUN: -a-different-option \
+ # ...
+ run_cmd += f" \\\n{comment_str}RUN: ".join(itertools.chain(("",), batch))
+
+ run_cmd += (
+ driver_data.lit_cmd_options # "-x c++"
+ + driver_data.lit_cmd_end # " - < /dev/null 2>&1 | FileCheck -check-prefix=CC1OptionCHECK"
+ + str(i) # "0"
+ + " %s\n\n" # " %s"
+ )
+
+ lit_file.write(run_cmd)
+
+ for option_str in batch:
+ # Example check line: "// CC1OptionCHECK0: {{(unknown argument).*-A}}"
+ check_cmd = (
+ comment_str # "//
+ + visibility # "CC1Option"
+ + "CHECK"
+ + str(i) # "0"
+ + ": {{("
+ + driver_data.check_str # "unknown argument"
+ + ").*"
+ + option_str.replace("+", "\\+") # "-A"
+ + "}}\n"
+ )
+ lit_file.write(check_cmd)
+
+ lit_file.close()
+
+
+# List of driver flavours
+driver_sequence = []
+# List of unsupported driver-option pairs
+unsupported_sequence = []
+# List of driver-option pairs that will be skipped due to overlapping supported and unsupported option names.
+# See later comments for detail
+skipped_sequence = []
+
+# Parse arguments
+parser = argparse.ArgumentParser(
+ description="Generate a lit test to validate that options are only exposed to "
+ "intended driver modes. "
+ "The options and driver modes are parsed from Options.td."
+)
+
+default_options_td_dir = os.path.join(
+ os.path.dirname(__file__), "../include/clang/Driver"
+)
+parser.add_argument(
+ "--options-td-dir",
+ help="Include directory for Options.td. Typically found under clang/include/clang/Driver, which is the default.",
+ default=default_options_td_dir,
+)
+default_llvm_include_dir = os.path.join(os.path.dirname(__file__), "../../llvm/include")
+parser.add_argument(
+ "--llvm-include-dir",
+ help="Include directory for LLVM TableGen executable. The typical source tree path will be used by default.",
+ default=default_llvm_include_dir,
+)
+parser.add_argument(
+ "--llvm-bin",
+ help="llvm build tree bin directory path. Only used if --llvm-tblgen is not specified.",
+)
+parser.add_argument(
+ "--llvm-tblgen",
+ help="LLVM TableGen executable. If --llvm-bin is also not specified, the PATH will be searched.",
+)
+parser.add_argument(
+ "--general-test-path",
+ help="File where most driver flavour tests will be written. The typical source tree path will be used by default.",
+)
+parser.add_argument(
+ "--flang-test-path",
+ help="File where flang tests will be written. The typical source tree path will be used by default.",
+)
+parser.add_argument(
+ "--flang-fc1-test-path",
+ help="File where flang -fc1 tests will be written. The typical source tree path will be used by default.",
+)
+args = parser.parse_args()
+
+if not args.llvm_tblgen:
+ if args.llvm_bin:
+ args.llvm_tblgen = os.path.join(args.llvm_bin, "llvm-tblgen")
+ else:
+ args.llvm_tblgen = "llvm-tblgen"
+
+# Run TableGen to convert Options.td to json
+# Ex: llvm-tblgen -I llvm-project/llvm/include llvm-project/clang/include/clang/Driver/Options.td -dump-json
+options_json_str = subprocess.run(
+ [
+ args.llvm_tblgen,
+ "-I",
+ args.llvm_include_dir,
+ args.options_td_dir + "/Options.td",
+ "-dump-json",
+ ],
+ stdout=subprocess.PIPE,
+)
+options_dictionary = json.loads(options_json_str.stdout.decode("utf-8"))
+
+# Establish the dataclass objects for each driver
+driver_cc1as = DriverData(
+ "%clang -cc1as ",
+ "",
+ VISIBILITY_CC1AS,
+ f" - < /dev/null 2>&1 | FileCheck -check-prefix={VISIBILITY_CC1AS}CHECK",
+ "unknown argument",
+)
+driver_cc1 = DriverData(
+ "%clang -cc1 ",
+ " -x c++",
+ VISIBILITY_CC1,
+ f" - < /dev/null 2>&1 | FileCheck -check-prefix={VISIBILITY_CC1}CHECK",
+ "unknown argument",
+)
+driver_cl = DriverData(
+ "%clang_cl ",
+ " -### /c /WX -Werror",
+ VISIBILITY_CL,
+ f" 2>&1 | FileCheck -check-prefix={VISIBILITY_CL}CHECK",
+ "unknown argument ignored in clang-cl",
+)
+driver_dxc = DriverData(
+ "%clang_dxc ",
+ " -### /T lib_6_7",
+ VISIBILITY_DXC,
+ f" 2>&1 | FileCheck -check-prefix={VISIBILITY_DXC}CHECK",
+ "unknown argument",
+)
+driver_default = DriverData(
+ "%clang ",
+ " -### -x c++ -c",
+ VISIBILITY_DEFAULT,
+ f" - < /dev/null 2>&1 | FileCheck -check-prefix={VISIBILITY_DEFAULT}CHECK",
+ "unknown argument",
+)
+driver_fc1 = DriverData(
+ "%flang_fc1 ",
+ "",
+ VISIBILITY_FC1,
+ f" - < /dev/null 2>&1 | FileCheck -check-prefix={VISIBILITY_FC1}CHECK",
+ "unknown argument",
+)
+driver_flang = DriverData(
+ "%clang --driver-mode=flang ",
+ " -### -x c++ -c",
+ VISIBILITY_FLANG,
+ f" - < /dev/null 2>&1 | FileCheck -check-prefix={VISIBILITY_FLANG}CHECK",
----------------
Maetveis wrote:
```suggestion
VISIBILITY_CC1AS,
f" - < /dev/null 2>&1 | FileCheck -check-prefix=CHECK_CC1AS_",
"unknown argument",
)
driver_cc1 = DriverData(
"%clang -cc1 ",
" -x c++",
VISIBILITY_CC1,
f" - < /dev/null 2>&1 | FileCheck -check-prefix=CHECK_CC1_",
"unknown argument",
)
driver_cl = DriverData(
"%clang_cl ",
" -### /c /WX -Werror",
VISIBILITY_CL,
f" 2>&1 | FileCheck -check-prefix=CHECK_CL_",
"unknown argument ignored in clang-cl",
)
driver_dxc = DriverData(
"%clang_dxc ",
" -### /T lib_6_7",
VISIBILITY_DXC,
f" 2>&1 | FileCheck -check-prefix=CHECK_DXC_",
"unknown argument",
)
driver_default = DriverData(
"%clang ",
" -### -x c++ -c",
VISIBILITY_DEFAULT,
f" - < /dev/null 2>&1 | FileCheck -check-prefix=CHECK_CLANG_",
"unknown argument",
)
driver_fc1 = DriverData(
"%flang_fc1 ",
"",
VISIBILITY_FC1,
f" - < /dev/null 2>&1 | FileCheck -check-prefix=CHECK_FC1_",
"unknown argument",
)
driver_flang = DriverData(
"%clang --driver-mode=flang ",
" -### -x c++ -c",
VISIBILITY_FLANG,
f" - < /dev/null 2>&1 | FileCheck -check-prefix=CHECK_FLANG_",
```
I think these are more fitting than `FC1OptionCHECK`.
https://github.com/llvm/llvm-project/pull/120900
More information about the flang-commits
mailing list