[libc-commits] [libc] [libc] Add CMake formatting utility (PR #213102)
Jeff Bailey via libc-commits
libc-commits at lists.llvm.org
Fri Sep 4 03:47:41 PDT 2026
================
@@ -0,0 +1,1515 @@
+#!/usr/bin/env python3
+"""
+LLVM and Subproject CMake Formatter Utility
+
+Token-stream formatter with schema-aware keyword handling, based on the
+`cmake-language(7)` EBNF grammar.
+
+Architecture Overview:
+ 1. Lexer (tokenize): Converts raw CMake text into a flat list of typed Token
+ objects according to the cmake-language(7) EBNF. Handles bracket comments,
+ bracket arguments, quoted arguments, whitespace, newlines, and parens.
+ 2. Schema Scanner (scan_dynamic_schemas): Walks the token stream to learn
+ custom keyword schemas from cmake_parse_arguments() calls and
+ set(*_ARGS ...) variable conventions, populating a FormatterContext.
+ 3. Formatter (format_cmake_content): Iterates the token stream line-by-line,
+ applying indentation, keyword casing, comment buffering, and multi-line
+ argument layout rules using the populated FormatterContext.
+
+Minimum Python Version: 3.12 (uses pathlib.Path.walk(), added in 3.12).
+
+Grammar (from cmake-language(7)):
+ file ::= file_element*
+ file_element ::= command_invocation line_ending | (bracket_comment|space)* line_ending
+ line_ending ::= line_comment? newline
+ command_invocation ::= space* identifier space* '(' arguments ')'
+ arguments ::= argument? separated_arguments*
+ separated_arguments ::= separation+ argument? | separation* '(' arguments ')'
+ argument ::= bracket_argument | quoted_argument | unquoted_argument
+
+Keyword Schema Classification:
+ 1. Options / Flags (0 values): e.g. `OUTPUT_STRIP_TRAILING_WHITESPACE`, `EXCLUDE_FROM_ALL`, `POST_BUILD`, `PARENT_SCOPE`, `FORCE`, `PARSE_ARGV`, `PARSE_ARGN`.
+ - Option keywords consume 0 arguments and close immediately unless inside an active multi-value list keyword scope.
+ 2. Single-Value Keywords (1 value): e.g. `RESULT_VARIABLE`, `OUTPUT_VARIABLE`, `TARGET`, `WORKING_DIRECTORY`, `ALIAS`, `SUITE`, `CACHE`, `DEPFILE`.
+ - After value is consumed, keyword closes and subsequent keywords/values align at top-level keyword indent (+2 spaces).
+ 3. Multi-Value List Keywords (1+ values): e.g. `SRCS`, `HDRS`, `DEPENDS`, `FULL_BUILD_DEPENDS`, `COMPILE_OPTIONS`, `LINK_LIBRARIES`, `LOADER_ARGS`, `ARGS`, `ENV`, `PROPERTIES`, `OBJECT`, `STATIC`, `SHARED`, `MODULE`.
+ - Keywords on the command header line (`cmd(KEYWORD...`) do NOT grant extra nesting to child lines (+2 spaces relative to call base).
+ - Keywords on their own separate line (`\n KEYWORD...`) indent child list items +4 spaces (+2 relative to keyword line).
+ 4. Dynamic Schema Learning:
+ - Automatically parses `cmake_parse_arguments(...)` calls in `function(...)` / `macro(...)` AST blocks.
+ - Automatically learns custom option, single-value, and multi-value argument lists from `set(...)` calls using standard naming conventions (`*_OPTION_ARGS`, `*_SINGLE_VALUE_ARGS`, `*_MULTI_VALUE_ARGS`).
+
+Formatting Rules Enforced:
+ 1. Command Casing: Built-in language commands cased in lowercase (`add_entrypoint_object`, `set`, `if`); module commands (like `ExternalProject_Add`) and custom functions retain canonical/declared casing.
+ 2. Parenthesis Spacing: No space between command name and opening `(`. Collapses multiple spaces between arguments down to a single space.
+ 3. Quoted String Immutability: Quoted arguments (`"..."`) and bracket arguments (`[=[...]=]`) are single immutable AST tokens. Multi-line quoted strings are preserved 100% untouched.
+ 4. Empty Closures: `endif()`, `else()`, `endfunction()`, `endmacro()`, `endforeach()`, `endwhile()`.
+ 5. Schema-Aware Keyword Casing: Keywords in command schema upper-cased; positional args, function parameters & file paths untouched.
+ 6. Multi-line Argument Layout: Keywords and positional args indented +2 spaces relative to call base; multi-value list items indented +4 spaces; closing `)` at +0 spaces.
+ 7. Control Block Indentation: 2-space indentation inside `if`/`foreach`/`function`/`macro`.
+ 8. Comment Formatting: Line comments buffer and align with the indentation level of the code element immediately following them, unless separated by a blank line (standalone comments) or immediately preceding a closing parenthesis `)`.
+ 9. Cleanliness: Trailing whitespace stripped, single trailing newline for non-empty files; empty files preserved 0-byte.
+
+Usage:
+ cmake_format.py [options] <file|directory>...
+
+Options:
+ -i, --inplace, --fix Format files in-place.
+ -n, --dry-run Check formatting without modifying files (Evaluation Mode).
+ --diff Output unified diffs for files that need formatting.
+ -j, --jobs N Number of parallel worker processes to use (default: 1).
+ -h, --help Show this help message.
+"""
+
+import sys
+import os
+import re
+import argparse
+import difflib
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+import copy
+from typing import NamedTuple
+
+
+class LexError(ValueError):
+ """Raised by tokenize() when the input CMake source is malformed.
+
+ Attributes:
+ msg: Human-readable description of the problem.
+ line: 1-based line number where the unterminated token started.
+ col: 1-based column number where the unterminated token started.
+ """
+
+ def __init__(self, msg: str, line: int, col: int) -> None:
+ super().__init__(f"line {line}, col {col}: {msg}")
+ self.msg = msg
+ self.line = line
+ self.col = col
+
+
+# Control block commands
+CONTROL_START_BLOCKS = {"if", "function", "macro", "foreach", "while"}
+CONTROL_MIDDLE_BLOCKS = {"elseif", "else"}
+EMPTY_CLOSE_BLOCKS = {
+ "endif",
+ "else",
+ "endfunction",
+ "endmacro",
+ "endforeach",
+ "endwhile",
+}
+
+# Target creation commands
+TARGET_CREATION_COMMANDS = {"add_library", "add_executable", "add_custom_target"}
+
+# Formatting Constants
+INDENT_WIDTH = 2
+INDENT_STR = " " * INDENT_WIDTH
+# Indent levels relative to a command's base_indent:
+# +1 level = keyword line or positional arg line (+2 spaces)
+# +2 levels = multi-value list item under a keyword on its own line (+4 spaces)
+KEYWORD_INDENT_LEVELS = 1
+LIST_ITEM_INDENT_LEVELS = 2
+
+
+ at dataclass
+class FormatterContext:
+ """Holds learned schema state for formatting operations to ensure isolation and thread-safety."""
+
+ learned_options: set[str] = field(default_factory=set)
+ learned_one_value: set[str] = field(default_factory=lambda: {"ALIAS"})
+ learned_multi_value: set[str] = field(default_factory=set)
+ list_keywords: set[str] = field(
+ default_factory=lambda: {
+ "SRCS",
+ "HDRS",
+ "DEPENDS",
+ "FULL_BUILD_DEPENDS",
+ "COMPILE_OPTIONS",
+ "LINK_LIBRARIES",
+ "LINK_LIBS",
+ "FLAGS",
+ "SOURCES",
+ "BYPRODUCTS",
+ "BUILD_BYPRODUCTS",
+ "CMAKE_ARGS",
+ "CMAKE_CACHE_ARGS",
+ "LOADER_ARGS",
+ "ARGS",
+ "ENV",
+ "COMPILE_DEFINITIONS",
+ "PROPERTIES",
+ "OBJECT",
+ "STATIC",
+ "SHARED",
+ "MODULE",
+ }
+ )
+ dynamic_schemas: dict[str, "CommandSchema"] = field(default_factory=dict)
+ # Per-context cache for get_schema_for_cmd() results. Keyed by lowercased
+ # command name. Invalidated whenever the context is mutated by
+ # scan_dynamic_schemas(). Excluded from __init__ so it doesn't appear in
+ # the constructor or backward-compat aliases.
+ _schema_cache: dict[str, "CommandSchema"] = field(
+ default_factory=dict, init=False, repr=False, compare=False
+ )
+
+ def clone(self) -> "FormatterContext":
+ """Returns a deep copy of this context.
+
+ Uses copy.deepcopy() so that any new fields added to FormatterContext
+ or CommandSchema are automatically included — no lockstep update needed.
+ """
+ return copy.deepcopy(self)
+
+
+WORKSPACE_CONTEXT = FormatterContext()
+
+
+def _init_worker(ctx: FormatterContext) -> None:
+ """Initializer for ProcessPoolExecutor workers.
+
+ Replaces WORKSPACE_CONTEXT in the worker process with the fully
+ pre-scanned context from the main process. Using an initializer
+ (rather than relying on fork memory inheritance) ensures the
+ pre-scanned schemas are available regardless of the multiprocessing
+ start method ('fork' on Linux, 'spawn' on macOS/Windows).
+ """
+ global WORKSPACE_CONTEXT
+ WORKSPACE_CONTEXT = ctx
+
+
+# Pre-compiled regular expressions for Lexer, Schema Scanner, and Formatter.
+
+# Matches uppercase identifier tokens (A-Z, 0-9, _) — used to extract
+# keyword names from set(*_ARGS ...) string values.
+RE_IDENTIFIER_TOKENS = re.compile(r"[A-Z0-9_]+")
+# Matches mixed-case identifiers (a-zA-Z, 0-9, _) — used to extract
+# keyword names from cmake_parse_arguments() string arguments, which may
+# use any casing.
+RE_IDENTIFIER_WORDS = re.compile(r"[a-zA-Z0-9_]+")
+
+RE_OPTION_ARGS = re.compile(
+ r"set\s*\(\s*([A-Za-z0-9_]*(?:OPTION|OPTIONAL)_ARGS)\s+([^)]+)\)", re.IGNORECASE
+)
+RE_SINGLE_VALUE_ARGS = re.compile(
+ r"set\s*\(\s*([A-Za-z0-9_]*(?:SINGLE|ONE)_VALUE_ARGS)\s+([^)]+)\)", re.IGNORECASE
+)
+RE_MULTI_VALUE_ARGS = re.compile(
+ r"set\s*\(\s*([A-Za-z0-9_]*(?:MULTI_VALUE|LIST)_ARGS)\s+([^)]+)\)", re.IGNORECASE
+)
+
+RE_BRACKET_COMMENT_START = re.compile(r"#\[(=*)\[")
+RE_BRACKET_ARG_START = re.compile(r"\[(=*)\[")
+
+RE_COMMENT_HASH = re.compile(r"^(#+)(.*)$")
+
+RE_ARG_FORWARDING_VAR = re.compile(r'^"?\$(?:\{|\()ARG[NV]\d*(?:\}|\))"?$')
+RE_COMMAND_INVOCATION = re.compile(r"^(\s*)([a-zA-Z0-9_]+)(\s*)\((.*)$", re.DOTALL)
+# Matches the CMake CACHE keyword as a whole word, used to exclude CACHE variable
+# declarations from dynamic schema learning.
+RE_CACHE = re.compile(r"\bCACHE\b")
+
+
+ at dataclass
+class CommandSchema:
+ """Represents option, single-value, and multi-value keyword argument schemas for a CMake command."""
+
+ options: set[str] = field(default_factory=set)
+ one_value: set[str] = field(default_factory=set)
+ multi_value: set[str] = field(default_factory=set)
+ explicit_keywords: set[str] | None = None
+ all_keywords: set[str] = field(default_factory=set, init=False)
+
+ def __post_init__(self) -> None:
+ self.options = set(self.options)
+ self.one_value = set(self.one_value)
+ self.multi_value = set(self.multi_value)
+ self.all_keywords = self.options | self.one_value | self.multi_value
+ if self.explicit_keywords is None:
+ self.explicit_keywords = set(self.all_keywords)
+ else:
+ self.explicit_keywords = set(self.explicit_keywords)
+
+
+_IF_OPTIONS = {
+ "NOT",
+ "AND",
+ "OR",
+ "COMMAND",
+ "POLICY",
+ "TARGET",
+ "EXISTS",
+ "IS_DIRECTORY",
+ "IS_SYMLINK",
+ "IS_ABSOLUTE",
+ "MATCHES",
+ "LESS",
+ "GREATER",
+ "EQUAL",
+ "LESS_EQUAL",
+ "GREATER_EQUAL",
+ "STRLESS",
+ "STRGREATER",
+ "STREQUAL",
+ "STRLESS_EQUAL",
+ "STRGREATER_EQUAL",
+ "VERSION_LESS",
+ "VERSION_GREATER",
+ "VERSION_EQUAL",
+ "VERSION_LESS_EQUAL",
+ "VERSION_GREATER_EQUAL",
+ "IN_LIST",
+ "DEFINED",
+}
+
+# Official Built-in CMake Command Schema Registry
+BUILTIN_COMMAND_SCHEMAS = {
+ "function": CommandSchema(),
+ "macro": CommandSchema(),
+ "foreach": CommandSchema(options={"IN", "LISTS", "ITEMS", "ZIP_LISTS"}),
+ "while": CommandSchema(),
+ "if": CommandSchema(options=_IF_OPTIONS),
+ "elseif": CommandSchema(options=_IF_OPTIONS),
+ "else": CommandSchema(),
+ "endif": CommandSchema(),
+ "endfunction": CommandSchema(),
+ "endmacro": CommandSchema(),
+ "endforeach": CommandSchema(),
+ "endwhile": CommandSchema(),
+ "cmake_parse_arguments": CommandSchema(options={"PARSE_ARGV", "PARSE_ARGN"}),
+ "execute_process": CommandSchema(
+ options={
+ "OUTPUT_STRIP_TRAILING_WHITESPACE",
+ "ERROR_STRIP_TRAILING_WHITESPACE",
+ "OUTPUT_QUIET",
+ "ERROR_QUIET",
+ "ECHO_OUTPUT_VARIABLE",
+ "ECHO_ERROR_VARIABLE",
+ },
+ one_value={
+ "WORKING_DIRECTORY",
+ "TIMEOUT",
+ "RESULT_VARIABLE",
+ "RESULTS_VARIABLE",
+ "OUTPUT_VARIABLE",
+ "ERROR_VARIABLE",
+ "INPUT_FILE",
+ "OUTPUT_FILE",
+ "ERROR_FILE",
+ "ENCODING",
+ "COMMAND_ERROR_IS_FATAL",
+ },
+ multi_value={"COMMAND"},
+ ),
+ "try_compile": CommandSchema(
+ options={"GLOBAL_ERROR", "NO_CACHE"},
+ one_value={"OUTPUT_VARIABLE", "COPY_FILE", "COPY_FILE_ERROR"},
+ multi_value={
+ "SOURCES",
+ "COMPILE_DEFINITIONS",
+ "LINK_LIBRARIES",
+ "LINK_OPTIONS",
+ "CMAKE_FLAGS",
+ },
+ ),
+ "add_custom_command": CommandSchema(
+ options={
+ "POST_BUILD",
+ "PRE_BUILD",
+ "PRE_LINK",
+ "VERBATIM",
+ "APPEND",
+ "USES_TERMINAL",
+ "COMMAND_EXPAND_LISTS",
+ },
+ one_value={
+ "TARGET",
+ "MAIN_DEPENDENCY",
+ "WORKING_DIRECTORY",
+ "COMMENT",
+ "DEPFILE",
+ "JOB_POOL",
+ "JOB_SERVER_AWARE",
+ },
+ multi_value={"COMMAND", "OUTPUT", "BYPRODUCTS", "DEPENDS", "IMPLICIT_DEPENDS"},
+ ),
+ "add_custom_target": CommandSchema(
+ options={"ALL", "VERBATIM", "USES_TERMINAL", "COMMAND_EXPAND_LISTS"},
+ one_value={"WORKING_DIRECTORY", "COMMENT"},
+ multi_value={"COMMAND", "DEPENDS", "BYPRODUCTS", "SOURCES"},
+ ),
+ "ExternalProject_Add": CommandSchema(
+ options={"EXCLUDE_FROM_ALL"},
+ one_value={
+ "PREFIX",
+ "SOURCE_DIR",
+ "BINARY_DIR",
+ "INSTALL_DIR",
+ "DOWNLOAD_COMMAND",
+ "CONFIGURE_COMMAND",
+ "BUILD_COMMAND",
+ "INSTALL_COMMAND",
+ },
+ multi_value={
+ "BUILD_BYPRODUCTS",
+ "CMAKE_ARGS",
+ "CMAKE_CACHE_ARGS",
+ "STEP_TARGETS",
+ "INDEPENDENT_STEP_TARGETS",
+ "DEPENDS",
+ },
+ ),
+ "find_package": CommandSchema(
+ options={"EXACT", "QUIET", "REQUIRED", "CONFIG", "NO_MODULE"},
+ multi_value={"COMPONENTS", "OPTIONAL_COMPONENTS"},
+ ),
+ "add_library": CommandSchema(
+ options={"EXCLUDE_FROM_ALL", "GLOBAL", "IMPORTED"},
+ one_value={"ALIAS"},
+ multi_value={
+ "STATIC",
+ "SHARED",
+ "MODULE",
+ "OBJECT",
+ "PUBLIC",
+ "PRIVATE",
+ "INTERFACE",
+ "SOURCES",
+ },
+ ),
+ "add_executable": CommandSchema(
+ options={"WIN32", "MACOSX_BUNDLE", "EXCLUDE_FROM_ALL", "GLOBAL", "IMPORTED"},
+ one_value={"ALIAS"},
+ multi_value={"SOURCES"},
+ ),
+ "target_link_libraries": CommandSchema(
+ multi_value={"PUBLIC", "PRIVATE", "INTERFACE", "LINK_PRIVATE", "LINK_PUBLIC"}
+ ),
+ "target_include_directories": CommandSchema(
+ options={"BEFORE", "SYSTEM"}, multi_value={"PUBLIC", "PRIVATE", "INTERFACE"}
+ ),
+ "target_compile_options": CommandSchema(
+ options={"BEFORE"}, multi_value={"PUBLIC", "PRIVATE", "INTERFACE"}
+ ),
+ "set_target_properties": CommandSchema(multi_value={"PROPERTIES"}),
+ "set_source_files_properties": CommandSchema(multi_value={"PROPERTIES"}),
+ "set_directory_properties": CommandSchema(multi_value={"PROPERTIES"}),
+ "set_property": CommandSchema(
+ options={
+ "GLOBAL",
+ "DIRECTORY",
+ "TARGET",
+ "SOURCE",
+ "INSTALL",
+ "TEST",
+ "CACHE",
+ "INHERITED",
+ },
+ one_value={"PROPERTY"},
+ multi_value={"APPEND", "APPEND_STRING"},
+ ),
+ "get_target_property": CommandSchema(),
+ "get_property": CommandSchema(
+ options={
+ "GLOBAL",
+ "DIRECTORY",
+ "TARGET",
+ "SOURCE",
+ "INSTALL",
+ "TEST",
+ "CACHE",
+ "SET",
+ "DEFINED",
+ "BRIEF_DOCS",
+ "FULL_DOCS",
+ },
+ one_value={"PROPERTY"},
+ ),
+ "list": CommandSchema(
+ options={
+ "APPEND",
+ "PREPEND",
+ "POP_BACK",
+ "POP_FRONT",
+ "REMOVE_AT",
+ "REMOVE_ITEM",
+ "REMOVE_DUPLICATES",
+ "TRANSFORM",
+ "SORT",
+ "REVERSE",
+ "JOIN",
+ "SUBLIST",
+ "FILTER",
+ "FIND",
+ "GET",
+ "LENGTH",
+ "INSERT",
+ }
+ ),
+ "set": CommandSchema(options={"PARENT_SCOPE", "FORCE"}, one_value={"CACHE"}),
+}
+
+CANONICAL_CMD_CASING = {}
+for k in BUILTIN_COMMAND_SCHEMAS.keys():
+ if k != k.lower():
+ CANONICAL_CMD_CASING[k.lower()] = k
+
+
+def get_schema_for_cmd(
+ cmd_name: str, ctx: FormatterContext | None = None
+) -> CommandSchema:
+ """Resolves official built-in or dynamically learned argument schema for a given CMake command."""
+ cmd_lower = cmd_name.lower()
+
+ # Standard built-in CMake commands use exact official schemas
+ if cmd_lower in BUILTIN_COMMAND_SCHEMAS:
+ return BUILTIN_COMMAND_SCHEMAS[cmd_lower]
+
+ context = ctx if ctx is not None else WORKSPACE_CONTEXT
+
+ # Return cached schema if available, avoiding repeated set allocations.
+ cached = context._schema_cache.get(cmd_lower)
+ if cached is not None:
+ return cached
+
+ options = set(context.learned_options)
+ one_value_args = set(context.learned_one_value)
+ multi_value_args = set(context.list_keywords) | set(context.learned_multi_value)
+
+ explicit_kws = None
+ if cmd_lower in context.dynamic_schemas:
+ ds = context.dynamic_schemas[cmd_lower]
+ options.update(ds.options)
+ one_value_args.update(ds.one_value)
+ multi_value_args.update(ds.multi_value)
+ explicit_kws = set(ds.explicit_keywords)
+
+ # Resolve overlaps so `multi_value` and `one_value` take precedence over `options`.
+ options -= multi_value_args | one_value_args
+ one_value_args -= multi_value_args
+
+ schema = CommandSchema(
+ options=options,
+ one_value=one_value_args,
+ multi_value=multi_value_args,
+ explicit_keywords=explicit_kws if explicit_kws is not None else set(),
+ )
+ context._schema_cache[cmd_lower] = schema
+ return schema
+
+
+class TokenType(str, Enum):
+ IDENTIFIER = "IDENTIFIER"
+ LPAREN = "LPAREN"
+ RPAREN = "RPAREN"
+ QUOTED_ARG = "QUOTED_ARG"
+ BRACKET_ARG = "BRACKET_ARG"
+ UNQUOTED_ARG = "UNQUOTED_ARG"
+ LINE_COMMENT = "LINE_COMMENT"
+ BRACKET_COMMENT = "BRACKET_COMMENT"
+ WHITESPACE = "WHITESPACE"
+ NEWLINE = "NEWLINE"
+
+
+class Token(NamedTuple):
+ """Represents a single AST token with its TokenType and string payload."""
+
+ type: TokenType
+ value: str
+
+
+class KeywordType(str, Enum):
+ ONE_VALUE = "ONE_VALUE"
+ MULTI_VALUE = "MULTI_VALUE"
+ OPTION = "OPTION"
+
+
+def tokenize(text: str) -> list[Token]:
+ """Formal Lexer based on cmake-language(7) EBNF specification.
+
+ Raises:
+ LexError: If the input contains an unterminated quoted argument,
+ bracket argument, or bracket comment.
+ """
+ tokens = []
+ i = 0
+ n = len(text)
+ # Track line/col for error reporting. line and col are 1-based.
+ line = 1
+ line_start = 0
+
+ while i < n:
+ col = i - line_start + 1
+
+ # Bracket comment or line comment starting with '#'
+ if text[i] == "#":
+ m = RE_BRACKET_COMMENT_START.match(text, pos=i)
+ if m:
+ eq_len = len(m.group(1))
+ close_pat = "]" + "=" * eq_len + "]"
+ end_idx = text.find(close_pat, i)
+ if end_idx == -1:
+ raise LexError(
+ f"unterminated bracket comment '#[{'=' * eq_len}['",
+ line,
+ col,
+ )
+ end_pos = end_idx + len(close_pat)
+ tokens.append(Token(TokenType.BRACKET_COMMENT, text[i:end_pos]))
+ # Advance line/col tracking over the consumed span.
+ newlines = text[i:end_pos].count("\n")
+ if newlines:
+ line += newlines
+ line_start = end_pos - len(text[i:end_pos].rsplit("\n", 1)[-1])
+ i = end_pos
+ continue
+
+ # Line comment: #...
+ end_idx = text.find("\n", i)
+ if end_idx == -1:
+ end_idx = n
+ tokens.append(Token(TokenType.LINE_COMMENT, text[i:end_idx]))
+ i = end_idx
+ continue
+
+ # Bracket argument: [=[...]=]
+ if text[i] == "[":
+ m = RE_BRACKET_ARG_START.match(text, pos=i)
+ if m:
+ eq_len = len(m.group(1))
+ close_pat = "]" + "=" * eq_len + "]"
+ end_idx = text.find(close_pat, i)
+ if end_idx == -1:
+ raise LexError(
+ f"unterminated bracket argument '[{'=' * eq_len}['",
+ line,
+ col,
+ )
+ end_pos = end_idx + len(close_pat)
+ tokens.append(Token(TokenType.BRACKET_ARG, text[i:end_pos]))
+ newlines = text[i:end_pos].count("\n")
+ if newlines:
+ line += newlines
+ line_start = end_pos - len(text[i:end_pos].rsplit("\n", 1)[-1])
+ i = end_pos
+ continue
+
+ # Quoted argument: "..." (Single immutable token!)
+ if text[i] == '"':
+ j = i + 1
+ closed = False
+ while j < n:
+ if text[j] == "\\":
+ j = min(n, j + 2)
+ elif text[j] == '"':
+ j += 1
+ closed = True
+ break
+ else:
+ j += 1
+ if not closed:
+ raise LexError("unterminated quoted argument", line, col)
+ token_val = text[i:j]
+ tokens.append(Token(TokenType.QUOTED_ARG, token_val))
+ newlines = token_val.count("\n")
+ if newlines:
+ line += newlines
+ line_start = j - len(token_val.rsplit("\n", 1)[-1])
+ i = j
+ continue
+
+ # Newline
+ if text[i] == "\n":
+ tokens.append(Token(TokenType.NEWLINE, "\n"))
+ i += 1
+ line += 1
+ line_start = i
+ continue
+
+ # Whitespace
+ if text[i] in " \t\r":
+ j = i
+ while j < n and text[j] in " \t\r":
+ j += 1
+ tokens.append(Token(TokenType.WHITESPACE, text[i:j]))
+ i = j
+ continue
+
+ # Parens
+ if text[i] == "(":
+ tokens.append(Token(TokenType.LPAREN, "("))
+ i += 1
+ continue
+ if text[i] == ")":
+ tokens.append(Token(TokenType.RPAREN, ")"))
+ i += 1
+ continue
+
+ # Unquoted argument / Identifier
+ j = i
+ while j < n and text[j] not in ' \t\r\n()#"':
+ if text[j] == "\\":
+ j = min(n, j + 2)
+ else:
+ j += 1
+ token_val = text[i:j]
+ tokens.append(Token(TokenType.UNQUOTED_ARG, token_val))
+ i = j
+
+ return tokens
+
+
+def scan_dynamic_schemas(
+ content: str,
+ ctx: FormatterContext | None = None,
+ tokens: list[Token] | None = None,
+) -> None:
+ """Scans cmake_parse_arguments and set(..._ARGS) calls to learn keyword schemas dynamically.
+
+ Mutates ctx in-place: appends newly discovered keywords to ctx.learned_options,
+ ctx.learned_one_value, ctx.learned_multi_value, ctx.list_keywords, and
+ ctx.dynamic_schemas. If ctx is None, mutates the module-level WORKSPACE_CONTEXT.
+ """
+ if "_ARGS" not in content and "cmake_parse_arguments" not in content:
+ return
+ context = ctx if ctx is not None else WORKSPACE_CONTEXT
+ # Any mutation of the context invalidates previously cached merged schemas.
+ context._schema_cache.clear()
+
+ # 1. Parse set(*_OPTION_ARGS ...), set(*_SINGLE_VALUE_ARGS ...), set(*_MULTI_VALUE_ARGS ...), etc.
+ schema_patterns = [
+ (RE_OPTION_ARGS, context.learned_options),
+ (RE_SINGLE_VALUE_ARGS, context.learned_one_value),
+ (RE_MULTI_VALUE_ARGS, context.learned_multi_value),
+ ]
+ for regex, target_set in schema_patterns:
+ for m in regex.finditer(content):
+ if not RE_CACHE.search(m.group(2)):
+ words = set(RE_IDENTIFIER_TOKENS.findall(m.group(2)))
+ target_set.update(words)
+ if target_set is context.learned_multi_value:
+ context.list_keywords.update(words)
+
+ # 2. Token-based scanning for function()/macro() declarations and cmake_parse_arguments
+ if tokens is None:
+ tokens = tokenize(content)
+ n = len(tokens)
+ current_fn_name = None
+
+ for i in range(n):
+ tok = tokens[i]
+ if tok.type != TokenType.UNQUOTED_ARG:
+ continue
+
+ cmd_lower = tok.value.lower()
+
+ # Track function/macro start
+ if cmd_lower in ("function", "macro"):
+ for j in range(i + 1, n):
+ tok_j = tokens[j]
+ if tok_j.type in (
+ TokenType.WHITESPACE,
+ TokenType.NEWLINE,
+ TokenType.LINE_COMMENT,
+ TokenType.BRACKET_COMMENT,
+ ):
+ continue
+ if tok_j.type == TokenType.LPAREN:
+ for k in range(j + 1, n):
+ tok_k = tokens[k]
+ if tok_k.type in (
+ TokenType.WHITESPACE,
+ TokenType.NEWLINE,
+ TokenType.LINE_COMMENT,
+ TokenType.BRACKET_COMMENT,
+ ):
+ continue
+ if tok_k.type in (TokenType.UNQUOTED_ARG, TokenType.QUOTED_ARG):
+ current_fn_name = tok_k.value.strip('"')
+ break
+ break
+
+ # Track function/macro end
+ elif cmd_lower in ("endfunction", "endmacro"):
+ current_fn_name = None
+
+ # Parse cmake_parse_arguments(...) call inside function/macro
+ elif cmd_lower == "cmake_parse_arguments" and current_fn_name:
+ arg_tokens = []
+ in_parens = False
+ paren_depth = 0
+ for j in range(i + 1, n):
+ tok_j = tokens[j]
+ if tok_j.type in (
+ TokenType.WHITESPACE,
+ TokenType.NEWLINE,
+ TokenType.LINE_COMMENT,
+ TokenType.BRACKET_COMMENT,
+ ):
+ continue
+ if tok_j.type == TokenType.LPAREN:
+ in_parens = True
+ paren_depth += 1
+ continue
+ if in_parens:
+ if tok_j.type == TokenType.RPAREN:
+ paren_depth -= 1
+ if paren_depth <= 0:
+ break
+ else:
+ arg_tokens.append(tok_j.value)
+
+ # cmake_parse_arguments positional layout:
+ # cmake_parse_arguments(<prefix> <options> <one_value> <multi_value> <args>...)
+ # cmake_parse_arguments(PARSE_ARGV <n> <prefix> <options> <one_value> <multi_value>)
+ # When PARSE_ARGV/PARSE_ARGN is present, the first two tokens are the
+ # mode keyword and the index argument; the prefix/options/... follow at +2.
+ _CPA_MIN_ARGS = 4 # prefix + options + one_value + multi_value
+ _CPA_PARSE_ARGV_EXTRA = 2 # extra tokens: mode keyword + integer index
+ _CPA_PARSE_ARGV_MIN_ARGS = _CPA_MIN_ARGS + _CPA_PARSE_ARGV_EXTRA
+
+ if len(arg_tokens) >= _CPA_MIN_ARGS:
+ offset = 0
+ if (
+ arg_tokens[0].upper() in ("PARSE_ARGV", "PARSE_ARGN")
+ and len(arg_tokens) >= _CPA_PARSE_ARGV_MIN_ARGS
+ ):
+ offset = _CPA_PARSE_ARGV_EXTRA
+
+ if len(arg_tokens) >= offset + _CPA_MIN_ARGS:
+ # arg_tokens[offset + 0] is the prefix — not needed for schema building
+ opt_str = arg_tokens[offset + 1]
+ one_str = arg_tokens[offset + 2]
+ multi_str = arg_tokens[offset + 3]
+
+ def _extract_literal_kws(arg_s: str) -> set[str]:
+ cleaned = arg_s.strip('"\t\r\n ')
+ if cleaned.startswith("${") or cleaned.startswith("$"):
+ return set()
+ non_var = re.sub(r"\$\{[^}]*\}", "", cleaned)
+ return set(RE_IDENTIFIER_WORDS.findall(non_var))
+
+ opts = _extract_literal_kws(opt_str)
+ ones = _extract_literal_kws(one_str)
+ multis = _extract_literal_kws(multi_str)
+ context.list_keywords.update(multis)
+ context.dynamic_schemas[current_fn_name.lower()] = CommandSchema(
+ options=opts, one_value=ones, multi_value=multis
+ )
+
+
+# Filesystem markers that indicate a project root boundary. Used by pre_scan_workspace_modules
+# to stop walking upward before escaping the repository tree.
+_PROJECT_ROOT_SENTINELS = frozenset({".git", ".gitmodules"})
+
+
+def pre_scan_workspace_modules(
+ paths: list[str], ctx: FormatterContext | None = None
+) -> None:
+ """Pre-scans all CMake module files in the repository directory tree to learn dynamic schemas before formatting."""
+ context = ctx if ctx is not None else WORKSPACE_CONTEXT
+ module_files = set()
+ scanned_base_dirs = set()
+
+ for p in paths:
+ path = Path(p)
+ base_dir = path.resolve() if path.is_dir() else path.resolve().parent
+ if base_dir in scanned_base_dirs:
+ continue
+ scanned_base_dirs.add(base_dir)
+
+ # Walk upward to find cmake/modules/, stopping at recognized repository
+ # root boundaries (.git, .gitmodules) so we don't accidentally scan
+ # unrelated cmake/modules/ directories in parent directories.
+ curr = base_dir
+ while curr and curr != curr.parent:
+ mod_dir = curr / "cmake" / "modules"
+ if mod_dir.is_dir():
+ module_files.update(str(f) for f in mod_dir.glob("*.cmake"))
+ break
+ # Stop after checking current dir if we've reached a repository root boundary
+ if any((curr / sentinel).exists() for sentinel in _PROJECT_ROOT_SENTINELS):
+ break
+ curr = curr.parent
+
+ for mf in module_files:
+ try:
+ with open(mf, "r", encoding="utf-8") as f:
+ scan_dynamic_schemas(f.read(), ctx=context)
+ except (OSError, UnicodeDecodeError, LexError):
+ pass
----------------
kaladron wrote:
Yeah, switching to failing hard. I don't want to be a diagnostic for people's cmake, just a formatter.
https://github.com/llvm/llvm-project/pull/213102
More information about the libc-commits
mailing list