[clang] [Clang][Driver][Test] Created test for unsupported driver options (PR #120900)

[Mészáros Gergely via cfe-commits]

================
@@ -0,0 +1,626 @@
+#!/usr/bin/env python3
+
+""" generate_unsupported_in_drivermode.py
+
+usage: python generate_unsupported_in_drivermode.py <path>/Options.td [<path>/llvm-tblgen]
+
+This script generates a Lit regression test file that validates that options
+are only exposed to intended driver modes.
+
+The options and driver modes are parsed from Options.td, whose path should be
+provided on the command line. See clang/include/clang/Driver/Options.td
+
+The path to the TableGen executable can optionally be provided. Otherwise, the
+script will search for it.
+
+The primary maintenance task for this script would be updating the expected return message for a driver mode if
+there are changes over time. See the instantiations of DriverController, specifically the check_string.
+
+Logic:
+1) For each option, (records of class "Option"), and for each driver, (records of class "OptionVisibility")
+    a. if the option's "Visibility" field includes the driver flavour, skip processing this option for this driver
+    b. if the option is part of an option group, (the record has the "Group" property),
+       and the group's "Visibility" field includes the driver flavour, skip processing this option for this driver
+    c. otherwise this option is not supported by this driver flavour, and this pairing is saved for testing
+2) For each unsupported pairing, generate a Lit RUN line, and a CHECK line to parse for expected output. Ex: "error: unknown argument"
+"""
+
+import sys
+import shutil
+import os
+import json
+import subprocess
+import math
+from pathlib import Path
+
+LLVM_TABLEGEN = "llvm-tblgen"
+LIT_TEST_PATH = "../test/Driver/unsupported_in_drivermode.c"
+LIT_TEST_PATH_FLANG = "../test/Driver/flang/unsupported_in_flang.f90"
+INCLUDE_PATH = "../../llvm/include"
+
+# Strings defined in Options.td for the various driver flavours. See "OptionVisibility"
+VISIBILITY_CC1AS = "CC1AsOption"
+VISIBILITY_CC1 = "CC1Option"
+VISIBILITY_CL = "CLOption"
+VISIBILITY_DXC = "DXCOption"
+VISIBILITY_DEFAULT = "DefaultVis"
+VISIBILITY_FC1 = "FC1Option"
+VISIBILITY_FLANG = "FlangOption"
+
+# Strings used in the commands to be tested
+CLANG = "clang"
+CLANG_CL = f"{CLANG} --driver-mode=cl"
+CLANG_DXC = f"{CLANG} --driver-mode=dxc"
+FLANG = f"{CLANG} --driver-mode=flang"
+CLANG_LIT = "%clang"
+CLANG_CL_LIT = "%clang_cl"
+CLANG_DXC_LIT = "%clang_dxc"
+FLANG_LIT = f"%{FLANG}"
+OPTION_HASH = "-###"
+OPTION_X = "-x"
+OPTION_WX = "/WX"
+OPTION_CPP = "c++"
+OPTION_C = "-c"
+OPTION_CC1 = "-cc1"
+OPTION_CC1AS = "-cc1as"
+OPTION_FC1 = "-fc1"
+OPTION_SLASH_C = "/c"
+OPTION_T = "/T lib_6_7"
+SLASH_SLASH = "// "
+EXCLAMATION = "! "
+
+# Invalid usage of the driver options below causes unique output, so skip testing
+exceptions_sequence = [
+    "cc1",
+    "cc1as",
+]
+
+
+class DriverController:
+    """Controller for data specific to each driver
+    shell_cmd_prefix: The beginning string of the command to be tested
+    lit_cmd_prefix: The beginning string of the Lit command
+    visibility_str: The corresponding visibility string from OptionVisibility in Options.td
+    shell_cmd_suffix: Strings near the end of the command to be tested
+    check_string: The string or regex to be sent to FileCheck
+    lit_cmd_end: String at the end of the Lit command
+
+    supported_sequence: List of UnsupportedDriverOption objects for supported options
+                        that are Kind KIND_JOINED*, as defined in Options.td
+    """
+
+    def __init__(
+        self,
+        shell_cmd_prefix="",
+        lit_cmd_prefix="",
+        visibility_str="",
+        shell_cmd_suffix="",
+        check_string="{{(unknown argument|n?N?o such file or directory)}}",
+        lit_cmd_end=" - < /dev/null 2>&1 | FileCheck -check-prefix=",
+    ):
+        self.shell_cmd_prefix = shell_cmd_prefix
+        self.lit_cmd_prefix = lit_cmd_prefix
+        self.visibility_str = visibility_str
+        self.shell_cmd_suffix = shell_cmd_suffix
+        self.supported_sequence = []
+        self.check_string = check_string
+        self.lit_cmd_end = lit_cmd_end
+
+
+class UnsupportedDriverOption:
+    """Defines an unsupported driver-option combination
+    driver: The driver string as defined by OptionVisibility in Options.td
+    option: The option object from Options.td
+    option_name: Corresponding string for an option. See "Name" for a given option in Options.td
+    prefix: String that precedes the option. Ex. "-"
+    is_error: Boolean indicating whether the corresponding command generates an error
+    """
+
+    def __init__(self, driver, option, option_name, prefix):
+        self.driver = driver
+        self.option = option
+        self.option_name = option_name
+        self.prefix = prefix
+        self.is_error = True
+
+    # For sorting
+    def __len__(self):
+        return len(self.option_name)
+
+
+def print_usage():
+    """Print valid usage of this script"""
+    sys.exit("usage: python " + sys.argv[0] + " <path>/Options.td [<path>/llvm-tblgen]")
+
+
+def find_file(file_name, search_path):
+    """Find the given file name under a search path"""
+    result = []
+
+    for root, dir, files in os.walk(search_path):
+        if file_name in files:
+            result.append(os.path.join(root, file_name))
+    return result
+
+
+def is_valid_file(path, expected_name):
+    """Is a file valid
+    Check if a given path is to a file, and if it matches the expected file name
+    """
+    if path.is_file() and path.name == expected_name:
+        return True
+    else:
+        return False
+
+
+def find_tablegen():
+    """Validate the TableGen executable"""
+    result = shutil.which(LLVM_TABLEGEN)
+    if result is None:
+        print(f"Unable to find {LLVM_TABLEGEN}")
+        sys.exit("\nExiting")
+    else:
+        print(f"{LLVM_TABLEGEN} found: {result}")
+        return result
+
+
+def find_groups(group_sequence, options_json, option):
+    """Find the groups for a given option
+    Note that groups can themselves be part of groups, hence the recursion
+
+    group_sequence: A sequence to which group names will be appended.
+    options_json: The converted Python dictionary from the Options.td json string
+    option: The option object from Options.td
+    """
+    group_json = options_json[option]["Group"]
+
+    if group_json is None:
+        return
+
+    # Prevent circular group membership lookup
+    if len(group_sequence) > 0:
+        for group in group_sequence:
+            if group_json["def"] == group:
+                return
+
+    group_sequence.append(group_json["def"])
+    return find_groups(group_sequence, options_json, option)
----------------
Maetveis wrote:

> group_sequence: A sequence to which group names will be appended.

This is really not conventional in Python, rather this function should **return** a list (actually maybe a set, because we do not want duplicate groups). In the outer levels of the recursion you can append the new item to what the inner levels returned.

Honestly I'm not getting how this function is supposed to work, you are recursively calling the function with the exact same set of parameters, I think this will always stop exactly at the second level of the recursion.

Also a comment with the structure of how group memberships look like in the tablegen output would help understanding this a lot more I think.



https://github.com/llvm/llvm-project/pull/120900