[clang] [Clang][Driver][Test] Created test for unsupported driver options (PR #120900)

[Mészáros Gergely via cfe-commits]

================
@@ -0,0 +1,626 @@
+#!/usr/bin/env python3
+
+""" generate_unsupported_in_drivermode.py
+
+usage: python generate_unsupported_in_drivermode.py <path>/Options.td [<path>/llvm-tblgen]
+
+This script generates a Lit regression test file that validates that options
+are only exposed to intended driver modes.
+
+The options and driver modes are parsed from Options.td, whose path should be
+provided on the command line. See clang/include/clang/Driver/Options.td
+
+The path to the TableGen executable can optionally be provided. Otherwise, the
+script will search for it.
+
+The primary maintenance task for this script would be updating the expected return message for a driver mode if
+there are changes over time. See the instantiations of DriverController, specifically the check_string.
+
+Logic:
+1) For each option, (records of class "Option"), and for each driver, (records of class "OptionVisibility")
+    a. if the option's "Visibility" field includes the driver flavour, skip processing this option for this driver
+    b. if the option is part of an option group, (the record has the "Group" property),
+       and the group's "Visibility" field includes the driver flavour, skip processing this option for this driver
+    c. otherwise this option is not supported by this driver flavour, and this pairing is saved for testing
+2) For each unsupported pairing, generate a Lit RUN line, and a CHECK line to parse for expected output. Ex: "error: unknown argument"
+"""
+
+import sys
+import shutil
+import os
+import json
+import subprocess
+import math
+from pathlib import Path
+
+LLVM_TABLEGEN = "llvm-tblgen"
+LIT_TEST_PATH = "../test/Driver/unsupported_in_drivermode.c"
+LIT_TEST_PATH_FLANG = "../test/Driver/flang/unsupported_in_flang.f90"
+INCLUDE_PATH = "../../llvm/include"
+
+# Strings defined in Options.td for the various driver flavours. See "OptionVisibility"
+VISIBILITY_CC1AS = "CC1AsOption"
+VISIBILITY_CC1 = "CC1Option"
+VISIBILITY_CL = "CLOption"
+VISIBILITY_DXC = "DXCOption"
+VISIBILITY_DEFAULT = "DefaultVis"
+VISIBILITY_FC1 = "FC1Option"
+VISIBILITY_FLANG = "FlangOption"
+
+# Strings used in the commands to be tested
+CLANG = "clang"
+CLANG_CL = f"{CLANG} --driver-mode=cl"
+CLANG_DXC = f"{CLANG} --driver-mode=dxc"
+FLANG = f"{CLANG} --driver-mode=flang"
+CLANG_LIT = "%clang"
+CLANG_CL_LIT = "%clang_cl"
+CLANG_DXC_LIT = "%clang_dxc"
+FLANG_LIT = f"%{FLANG}"
+OPTION_HASH = "-###"
+OPTION_X = "-x"
+OPTION_WX = "/WX"
+OPTION_CPP = "c++"
+OPTION_C = "-c"
+OPTION_CC1 = "-cc1"
+OPTION_CC1AS = "-cc1as"
+OPTION_FC1 = "-fc1"
+OPTION_SLASH_C = "/c"
+OPTION_T = "/T lib_6_7"
+SLASH_SLASH = "// "
+EXCLAMATION = "! "
+
+# Invalid usage of the driver options below causes unique output, so skip testing
+exceptions_sequence = [
+    "cc1",
+    "cc1as",
+]
+
+
+class DriverController:
+    """Controller for data specific to each driver
+    shell_cmd_prefix: The beginning string of the command to be tested
+    lit_cmd_prefix: The beginning string of the Lit command
+    visibility_str: The corresponding visibility string from OptionVisibility in Options.td
+    shell_cmd_suffix: Strings near the end of the command to be tested
+    check_string: The string or regex to be sent to FileCheck
+    lit_cmd_end: String at the end of the Lit command
+
+    supported_sequence: List of UnsupportedDriverOption objects for supported options
+                        that are Kind KIND_JOINED*, as defined in Options.td
+    """
+
+    def __init__(
+        self,
+        shell_cmd_prefix="",
+        lit_cmd_prefix="",
+        visibility_str="",
+        shell_cmd_suffix="",
+        check_string="{{(unknown argument|n?N?o such file or directory)}}",
+        lit_cmd_end=" - < /dev/null 2>&1 | FileCheck -check-prefix=",
+    ):
+        self.shell_cmd_prefix = shell_cmd_prefix
+        self.lit_cmd_prefix = lit_cmd_prefix
+        self.visibility_str = visibility_str
+        self.shell_cmd_suffix = shell_cmd_suffix
+        self.supported_sequence = []
+        self.check_string = check_string
+        self.lit_cmd_end = lit_cmd_end
+
+
+class UnsupportedDriverOption:
+    """Defines an unsupported driver-option combination
+    driver: The driver string as defined by OptionVisibility in Options.td
+    option: The option object from Options.td
+    option_name: Corresponding string for an option. See "Name" for a given option in Options.td
+    prefix: String that precedes the option. Ex. "-"
+    is_error: Boolean indicating whether the corresponding command generates an error
+    """
+
+    def __init__(self, driver, option, option_name, prefix):
+        self.driver = driver
+        self.option = option
+        self.option_name = option_name
+        self.prefix = prefix
+        self.is_error = True
+
+    # For sorting
+    def __len__(self):
+        return len(self.option_name)
+
+
+def print_usage():
+    """Print valid usage of this script"""
+    sys.exit("usage: python " + sys.argv[0] + " <path>/Options.td [<path>/llvm-tblgen]")
+
+
+def find_file(file_name, search_path):
+    """Find the given file name under a search path"""
+    result = []
+
+    for root, dir, files in os.walk(search_path):
+        if file_name in files:
+            result.append(os.path.join(root, file_name))
+    return result
+
+
+def is_valid_file(path, expected_name):
+    """Is a file valid
+    Check if a given path is to a file, and if it matches the expected file name
+    """
+    if path.is_file() and path.name == expected_name:
+        return True
+    else:
+        return False
+
+
+def find_tablegen():
+    """Validate the TableGen executable"""
+    result = shutil.which(LLVM_TABLEGEN)
+    if result is None:
+        print(f"Unable to find {LLVM_TABLEGEN}")
+        sys.exit("\nExiting")
+    else:
+        print(f"{LLVM_TABLEGEN} found: {result}")
+        return result
----------------
Maetveis wrote:

It might be good to take the path to the llvm build tree bin directory and the executable name for tablegen, similar to how it's done here:

https://github.com/llvm/llvm-project/blob/8035d38daab028b8da3cf2b01090b5f0ceacd695/llvm/utils/update_cc_test_checks.py#L182-L185

Also calling sys.exit() inside this function is really not clean, I'd rather it returned `None` and let the caller handle it. Additionally the exit code should not be zero in this case, because the script failed to execute.

https://github.com/llvm/llvm-project/pull/120900