From 7aa17f6263aca61e6bcfcfbf5c5edc2fa5d4f4d9 Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:23:33 +0100 Subject: [PATCH 1/4] MAINT: type pre-commit config with `TypedDict` --- docs/conf.py | 1 + src/compwa_policy/check_dev_files/black.py | 14 +- src/compwa_policy/check_dev_files/citation.py | 18 +- src/compwa_policy/check_dev_files/cspell.py | 31 +- .../check_dev_files/editorconfig.py | 12 +- .../check_dev_files/github_workflows.py | 18 +- .../check_dev_files/nbstripout.py | 8 +- .../check_dev_files/precommit.py | 53 ++-- src/compwa_policy/check_dev_files/prettier.py | 17 +- .../check_dev_files/pyupgrade.py | 11 +- src/compwa_policy/check_dev_files/ruff.py | 8 +- src/compwa_policy/check_dev_files/toml.py | 17 +- src/compwa_policy/self_check.py | 52 ++-- src/compwa_policy/utilities/precommit.py | 273 +++++++----------- tests/.gitignore | 1 - tests/check_dev_files/test_cspell.py | 12 +- tests/utilities/dummy-pre-commit-config.yml | 38 --- tests/utilities/test_precommit.py | 82 ------ 18 files changed, 260 insertions(+), 406 deletions(-) delete mode 100644 tests/.gitignore delete mode 100644 tests/utilities/dummy-pre-commit-config.yml delete mode 100644 tests/utilities/test_precommit.py diff --git a/docs/conf.py b/docs/conf.py index 947c0baa..6e6221fc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ "Array": "tomlkit.items.Array", "ConfigParser": "configparser.ConfigParser", "K": "typing.TypeVar", + "NotRequired": ("obj", "typing.NotRequired"), "P": "typing.ParamSpec", "P.args": ("attr", "typing.ParamSpec.args"), "P.kwargs": ("attr", "typing.ParamSpec.kwargs"), diff --git a/src/compwa_policy/check_dev_files/black.py b/src/compwa_policy/check_dev_files/black.py index 28f2b26f..6aed5eb2 100644 --- a/src/compwa_policy/check_dev_files/black.py +++ b/src/compwa_policy/check_dev_files/black.py @@ -1,12 +1,13 @@ """Update :file:`pyproject.toml` black configuration.""" from ruamel.yaml import YAML -from ruamel.yaml.comments import CommentedMap from compwa_policy.errors import PrecommitError from compwa_policy.utilities import CONFIG_PATH, vscode from compwa_policy.utilities.executor import Executor from compwa_policy.utilities.precommit import ( + Hook, + Repo, remove_precommit_hook, update_single_hook_precommit_repo, ) @@ -89,15 +90,16 @@ def _update_black_settings() -> None: def _update_precommit_repo(has_notebooks: bool) -> None: - expected_hook = CommentedMap( + expected_repo = Repo( repo="https://github.com/psf/black-pre-commit-mirror", - hooks=[CommentedMap(id="black")], + rev="", + hooks=[Hook(id="black")], ) if has_notebooks: - black_jupyter = CommentedMap( + black_jupyter = Hook( id="black-jupyter", args=YAML(typ="rt").load("[--line-length=85]"), types_or=YAML(typ="rt").load("[jupyter]"), ) - expected_hook["hooks"].append(black_jupyter) - update_single_hook_precommit_repo(expected_hook) + expected_repo["hooks"].append(black_jupyter) + update_single_hook_precommit_repo(expected_repo) diff --git a/src/compwa_policy/check_dev_files/citation.py b/src/compwa_policy/check_dev_files/citation.py index b5e132e2..d799a762 100644 --- a/src/compwa_policy/check_dev_files/citation.py +++ b/src/compwa_policy/check_dev_files/citation.py @@ -5,6 +5,7 @@ import json import os from textwrap import dedent +from typing import cast from html2text import HTML2Text from ruamel.yaml import YAML @@ -18,6 +19,8 @@ from compwa_policy.utilities import CONFIG_PATH, vscode from compwa_policy.utilities.executor import Executor from compwa_policy.utilities.precommit import ( + Hook, + Repo, find_repo, load_round_trip_precommit_config, update_single_hook_precommit_repo, @@ -171,7 +174,7 @@ def add_json_schema_precommit() -> None: if not CONFIG_PATH.citation.exists(): return # cspell:ignore jsonschema schemafile - expected_hook = CommentedMap( + expected_hook = Hook( id="check-jsonschema", name="Check CITATION.cff", args=[ @@ -181,21 +184,21 @@ def add_json_schema_precommit() -> None: "https://citation-file-format.github.io/1.2.0/schema.json", "CITATION.cff", ], - pass_filenames=False, ) config, yaml = load_round_trip_precommit_config() repo_url = "https://github.com/python-jsonschema/check-jsonschema" idx_and_repo = find_repo(config, repo_url) - existing_repos: CommentedSeq = config["repos"] + existing_repos = config["repos"] if idx_and_repo is None: - repo = CommentedMap( + repo = Repo( repo=repo_url, + rev="", hooks=[expected_hook], ) update_single_hook_precommit_repo(repo) else: repo_idx, repo = idx_and_repo - existing_hooks: CommentedSeq = repo["hooks"] + existing_hooks = repo["hooks"] hook_idx = None for i, hook in enumerate(existing_hooks): if hook == expected_hook: @@ -206,9 +209,10 @@ def add_json_schema_precommit() -> None: existing_hooks.append(expected_hook) else: existing_hooks[hook_idx] = expected_hook - existing_repos.yaml_set_comment_before_after_key(repo_idx + 1, before="\n") + repos_yaml = cast(CommentedSeq, existing_repos) + repos_yaml.yaml_set_comment_before_after_key(repo_idx + 1, before="\n") yaml.dump(config, CONFIG_PATH.precommit) - msg = f"Updated check-jsonschema hook in {CONFIG_PATH.citation}" + msg = f"Updated check-jsonschema hook in {CONFIG_PATH.precommit}" raise PrecommitError(msg) diff --git a/src/compwa_policy/check_dev_files/cspell.py b/src/compwa_policy/check_dev_files/cspell.py index 89834323..ffd5f796 100644 --- a/src/compwa_policy/check_dev_files/cspell.py +++ b/src/compwa_policy/check_dev_files/cspell.py @@ -11,13 +11,13 @@ from glob import glob from typing import TYPE_CHECKING, Any, Iterable, Sequence -from ruamel.yaml.comments import CommentedMap - from compwa_policy.errors import PrecommitError from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH, rename_file, vscode from compwa_policy.utilities.executor import Executor from compwa_policy.utilities.precommit import ( - PrecommitConfig, + Hook, + Repo, + find_repo, load_round_trip_precommit_config, update_single_hook_precommit_repo, ) @@ -47,9 +47,11 @@ def main(no_cspell_update: bool) -> None: rename_file("cspell.json", str(CONFIG_PATH.cspell)) executor = Executor() executor(_update_cspell_repo_url) - config = PrecommitConfig.load() - repo = config.find_repo(__REPO_URL) - if repo is None: + has_cspell_hook = False + if CONFIG_PATH.cspell.exists(): + config, _ = load_round_trip_precommit_config() + has_cspell_hook = find_repo(config, __REPO_URL) is not None + if not has_cspell_hook: executor(_remove_configuration) else: executor(_update_precommit_repo) @@ -65,14 +67,14 @@ def _update_cspell_repo_url(path: Path = CONFIG_PATH.precommit) -> None: old_url_patters = [ r".*/mirrors-cspell(.git)?$", ] - config = PrecommitConfig.load(path) + config, yaml = load_round_trip_precommit_config(path) for pattern in old_url_patters: - repo_index = config.get_repo_index(pattern) - if repo_index is None: + repo_and_idx = find_repo(config, pattern) + if repo_and_idx is None: continue - config_dict, yaml_parser = load_round_trip_precommit_config(path) - config_dict["repos"][repo_index]["repo"] = __REPO_URL - yaml_parser.dump(config_dict, path) + _, repo_def = repo_and_idx + repo_def["repo"] = __REPO_URL + yaml.dump(config, path) msg = f"Updated cSpell pre-commit repo URL to {__REPO_URL} in {path}" raise PrecommitError(msg) @@ -102,9 +104,10 @@ def _remove_configuration() -> None: def _update_precommit_repo() -> None: - expected_hook = CommentedMap( + expected_hook = Repo( repo=__REPO_URL, - hooks=[CommentedMap(id="cspell")], + rev="", + hooks=[Hook(id="cspell")], ) update_single_hook_precommit_repo(expected_hook) diff --git a/src/compwa_policy/check_dev_files/editorconfig.py b/src/compwa_policy/check_dev_files/editorconfig.py index 456cc00f..c0a336b4 100644 --- a/src/compwa_policy/check_dev_files/editorconfig.py +++ b/src/compwa_policy/check_dev_files/editorconfig.py @@ -7,11 +7,14 @@ from textwrap import dedent -from ruamel.yaml.comments import CommentedMap from ruamel.yaml.scalarstring import FoldedScalarString from compwa_policy.utilities import CONFIG_PATH -from compwa_policy.utilities.precommit import update_single_hook_precommit_repo +from compwa_policy.utilities.precommit import ( + Hook, + Repo, + update_single_hook_precommit_repo, +) def main(no_python: bool) -> None: @@ -20,7 +23,7 @@ def main(no_python: bool) -> None: def _update_precommit_config(no_python: bool) -> None: - hook = CommentedMap( + hook = Hook( id="editorconfig-checker", name="editorconfig", alias="ec", @@ -33,8 +36,9 @@ def _update_precommit_config(no_python: bool) -> None: """).strip() hook["exclude"] = FoldedScalarString(excludes) - expected_hook = CommentedMap( + expected_hook = Repo( repo="https://github.com/editorconfig-checker/editorconfig-checker.python", + rev="", hooks=[hook], ) update_single_hook_precommit_repo(expected_hook) diff --git a/src/compwa_policy/check_dev_files/github_workflows.py b/src/compwa_policy/check_dev_files/github_workflows.py index beea8a29..1679b2db 100644 --- a/src/compwa_policy/check_dev_files/github_workflows.py +++ b/src/compwa_policy/check_dev_files/github_workflows.py @@ -18,7 +18,7 @@ write, ) from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import PrecommitConfig +from compwa_policy.utilities.precommit import load_round_trip_precommit_config from compwa_policy.utilities.project_info import PythonVersion, get_pypi_name from compwa_policy.utilities.yaml import create_prettier_round_trip_yaml @@ -174,12 +174,18 @@ def __update_style_section(config: CommentedMap, python_version: PythonVersion) config["jobs"]["style"]["with"] = { "python-version": DoubleQuotedScalarString(python_version) } - if not CONFIG_PATH.precommit.exists(): + if __is_remove_style_job(): del config["jobs"]["style"] - else: - cfg = PrecommitConfig.load() - if cfg.ci is not None and cfg.ci.skip is None: - del config["jobs"]["style"] + + +def __is_remove_style_job() -> bool: + if not CONFIG_PATH.precommit.exists(): + return True + cfg, _ = load_round_trip_precommit_config() + precommit_ci = cfg.get("ci") + if precommit_ci is not None and "skip" not in precommit_ci: + return True + return False def __update_pytest_section( diff --git a/src/compwa_policy/check_dev_files/nbstripout.py b/src/compwa_policy/check_dev_files/nbstripout.py index 0dbef261..1b2084cf 100644 --- a/src/compwa_policy/check_dev_files/nbstripout.py +++ b/src/compwa_policy/check_dev_files/nbstripout.py @@ -1,10 +1,11 @@ """Check the nbstripout hook in the pre-commit config.""" -from ruamel.yaml.comments import CommentedMap from ruamel.yaml.scalarstring import LiteralScalarString from compwa_policy.utilities import CONFIG_PATH from compwa_policy.utilities.precommit import ( + Hook, + Repo, find_repo, load_round_trip_precommit_config, update_single_hook_precommit_repo, @@ -39,10 +40,11 @@ def main() -> None: "metadata.varInspector", "metadata.vscode", ] - expected_hook = CommentedMap( + expected_hook = Repo( repo=repo_url, + rev="", hooks=[ - CommentedMap( + Hook( id="nbstripout", args=[ "--extra-keys", diff --git a/src/compwa_policy/check_dev_files/precommit.py b/src/compwa_policy/check_dev_files/precommit.py index 0b663419..ce0103b2 100644 --- a/src/compwa_policy/check_dev_files/precommit.py +++ b/src/compwa_policy/check_dev_files/precommit.py @@ -13,18 +13,18 @@ from compwa_policy.utilities.executor import Executor from compwa_policy.utilities.precommit import ( PrecommitConfig, + find_repo, load_round_trip_precommit_config, ) from compwa_policy.utilities.yaml import create_prettier_round_trip_yaml def main() -> None: - cfg = PrecommitConfig.load() executor = Executor() executor(_sort_hooks) - executor(_update_conda_environment, cfg) + executor(_update_conda_environment) executor(_update_precommit_ci_commit_msg) - executor(_update_precommit_ci_skip, cfg) + executor(_update_precommit_ci_skip) executor.finalize() @@ -55,7 +55,7 @@ def _update_precommit_ci_commit_msg() -> None: if not CONFIG_PATH.precommit.exists(): return config, yaml = load_round_trip_precommit_config() - precommit_ci: CommentedMap | None = config.get("ci") + precommit_ci = config.get("ci") if precommit_ci is None: return if CONFIG_PATH.pip_constraints.exists(): @@ -65,23 +65,24 @@ def _update_precommit_ci_commit_msg() -> None: key = "autoupdate_commit_msg" autoupdate_commit_msg = precommit_ci.get(key) if autoupdate_commit_msg != expected_msg: - precommit_ci[key] = expected_msg + precommit_ci[key] = expected_msg # type:ignore[literal-required] yaml.dump(config, CONFIG_PATH.precommit) msg = f"Updated ci.{key} in {CONFIG_PATH.precommit} to {expected_msg!r}" raise PrecommitError(msg) -def _update_precommit_ci_skip(config: PrecommitConfig) -> None: - if config.ci is None: +def _update_precommit_ci_skip() -> None: + config, _ = load_round_trip_precommit_config() + if config.get("ci") is None: return local_hooks = get_local_hooks(config) non_functional_hooks = get_non_functional_hooks(config) expected_skips = set(non_functional_hooks) | set(local_hooks) if not expected_skips: - if config.ci.skip is not None: + if config.get("ci", {}).get("skip") is not None: yaml = create_prettier_round_trip_yaml() contents: CommentedMap = yaml.load(CONFIG_PATH.precommit) - del contents["ci"]["skip"] + del contents.get("ci")["skip"] contents.yaml_set_comment_before_after_key("repos", before="\n") yaml.dump(contents, CONFIG_PATH.precommit) msg = f"No need for a ci.skip in {CONFIG_PATH.precommit}" @@ -95,7 +96,7 @@ def _update_precommit_ci_skip(config: PrecommitConfig) -> None: def __update_precommit_ci_skip(expected_skips: Iterable[str]) -> None: yaml = create_prettier_round_trip_yaml() contents = yaml.load(CONFIG_PATH.precommit) - ci_section: CommentedMap = contents["ci"] + ci_section: CommentedMap = contents.get("ci") if "skip" in ci_section.ca.items: del ci_section.ca.items["skip"] skips = CommentedSeq(sorted(expected_skips)) @@ -107,29 +108,29 @@ def __update_precommit_ci_skip(expected_skips: Iterable[str]) -> None: def __get_precommit_ci_skips(config: PrecommitConfig) -> set[str]: - if config.ci is None: + precommit_ci = config.get("ci") + if precommit_ci is None: msg = "Pre-commit config does not contain a ci section" raise ValueError(msg) - if config.ci.skip is None: - return set() - return set(config.ci.skip) + return set(precommit_ci.get("skip", [])) def get_local_hooks(config: PrecommitConfig) -> list[str]: - return [h.id for r in config.repos for h in r.hooks if r.repo == "local"] + repos = config["repos"] + return [h["id"] for r in repos for h in r["hooks"] if r["repo"] == "local"] def get_non_functional_hooks(config: PrecommitConfig) -> list[str]: return [ - hook.id - for repo in config.repos - for hook in repo.hooks - if repo.repo - if hook.id in __get_skipped_hooks(config) + hook["id"] + for repo in config["repos"] + for hook in repo["hooks"] + if repo["repo"] + if hook["id"] in __get_skipped_hooks(config) ] -def _update_conda_environment(precommit_config: PrecommitConfig) -> None: +def _update_conda_environment() -> None: """Temporary fix for Prettier v4 alpha releases. https://prettier.io/blog/2023/11/30/cli-deep-dive#installation @@ -141,6 +142,7 @@ def _update_conda_environment(precommit_config: PrecommitConfig) -> None: conda_env: CommentedMap = yaml.load(path) variables: CommentedMap = conda_env.get("variables", {}) key = "PRETTIER_LEGACY_CLI" + precommit_config, _ = load_round_trip_precommit_config() if __has_prettier_v4alpha(precommit_config): if key not in variables: variables[key] = DoubleQuotedScalarString("1") @@ -169,10 +171,9 @@ def __get_skipped_hooks(config: PrecommitConfig) -> set[str]: def __has_prettier_v4alpha(config: PrecommitConfig) -> bool: - repo = config.find_repo(r"^.*/mirrors-prettier$") - if repo is None: - return False - if repo.rev is None: + idx_and_repo = find_repo(config, search_pattern=r"^.*/mirrors-prettier$") + if idx_and_repo is None: return False - rev = repo.rev + _, repo = idx_and_repo + rev = repo.get("rev", "") return rev.startswith("v4") and "alpha" in rev diff --git a/src/compwa_policy/check_dev_files/prettier.py b/src/compwa_policy/check_dev_files/prettier.py index c5dfad80..f41eb7a9 100644 --- a/src/compwa_policy/check_dev_files/prettier.py +++ b/src/compwa_policy/check_dev_files/prettier.py @@ -8,15 +8,17 @@ from compwa_policy.errors import PrecommitError from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH, vscode from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import PrecommitConfig +from compwa_policy.utilities.precommit import ( + find_repo, + load_round_trip_precommit_config, +) from compwa_policy.utilities.readme import add_badge, remove_badge # cspell:ignore esbenp rettier __VSCODE_EXTENSION_NAME = "esbenp.prettier-vscode" - -# fmt: off -__BADGE = "[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)" -# fmt: on +__BADGE = """ +[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier) +""".strip() __BADGE_PATTERN = r"\[\!\[[Pp]rettier.*\]\(.*prettier.*\)\]\(.*prettier.*\)\n?" @@ -25,9 +27,8 @@ def main(no_prettierrc: bool) -> None: - config = PrecommitConfig.load() - repo = config.find_repo(r".*/mirrors-prettier") - if repo is None: + config, _ = load_round_trip_precommit_config() + if find_repo(config, r".*/mirrors-prettier") is None: _remove_configuration() else: executor = Executor() diff --git a/src/compwa_policy/check_dev_files/pyupgrade.py b/src/compwa_policy/check_dev_files/pyupgrade.py index 1487ba97..6d5c7ae4 100644 --- a/src/compwa_policy/check_dev_files/pyupgrade.py +++ b/src/compwa_policy/check_dev_files/pyupgrade.py @@ -1,11 +1,13 @@ """Install `pyupgrade `_ as a hook.""" from ruamel.yaml import YAML -from ruamel.yaml.comments import CommentedMap, CommentedSeq +from ruamel.yaml.comments import CommentedSeq from compwa_policy.utilities import natural_sorting from compwa_policy.utilities.executor import Executor from compwa_policy.utilities.precommit import ( + Hook, + Repo, remove_precommit_hook, update_precommit_hook, update_single_hook_precommit_repo, @@ -24,10 +26,11 @@ def main(no_ruff: bool) -> None: def _update_precommit_repo() -> None: - expected_hook = CommentedMap( + expected_hook = Repo( repo="https://github.com/asottile/pyupgrade", + rev="", hooks=[ - CommentedMap( + Hook( id="pyupgrade", args=__get_pyupgrade_version_argument(), ) @@ -39,7 +42,7 @@ def _update_precommit_repo() -> None: def _update_precommit_nbqa_hook() -> None: update_precommit_hook( repo_url="https://github.com/nbQA-dev/nbQA", - expected_hook=CommentedMap( + expected_hook=Hook( id="nbqa-pyupgrade", args=__get_pyupgrade_version_argument(), ), diff --git a/src/compwa_policy/check_dev_files/ruff.py b/src/compwa_policy/check_dev_files/ruff.py index 84f77307..f0787172 100644 --- a/src/compwa_policy/check_dev_files/ruff.py +++ b/src/compwa_policy/check_dev_files/ruff.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Iterable from ruamel.yaml import YAML -from ruamel.yaml.comments import CommentedMap from compwa_policy.check_dev_files.setup_cfg import ( has_pyproject_build_system, @@ -17,6 +16,8 @@ from compwa_policy.utilities import CONFIG_PATH, natural_sorting, remove_configs, vscode from compwa_policy.utilities.executor import Executor from compwa_policy.utilities.precommit import ( + Hook, + Repo, remove_precommit_hook, update_single_hook_precommit_repo, ) @@ -519,12 +520,13 @@ def _update_precommit_hook(has_notebooks: bool) -> None: if not CONFIG_PATH.precommit.exists(): return yaml = YAML(typ="rt") - ruff_hook = CommentedMap(id="ruff", args=yaml.load("[--fix]")) + ruff_hook = Hook(id="ruff", args=yaml.load("[--fix]")) if has_notebooks: types = yaml.load("[python, pyi, jupyter]") ruff_hook["types_or"] = types - expected_repo = CommentedMap( + expected_repo = Repo( repo="https://github.com/astral-sh/ruff-pre-commit", + rev="", hooks=[ruff_hook], ) update_single_hook_precommit_repo(expected_repo) diff --git a/src/compwa_policy/check_dev_files/toml.py b/src/compwa_policy/check_dev_files/toml.py index c841183b..1f43da67 100644 --- a/src/compwa_policy/check_dev_files/toml.py +++ b/src/compwa_policy/check_dev_files/toml.py @@ -9,12 +9,15 @@ import tomlkit from ruamel.yaml import YAML -from ruamel.yaml.comments import CommentedMap from compwa_policy.errors import PrecommitError from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH, vscode from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import update_single_hook_precommit_repo +from compwa_policy.utilities.precommit import ( + Hook, + Repo, + update_single_hook_precommit_repo, +) from compwa_policy.utilities.pyproject import ( get_sub_table, load_pyproject, @@ -76,9 +79,10 @@ def _update_tomlsort_config() -> None: def _update_tomlsort_hook() -> None: - expected_hook = CommentedMap( + expected_hook = Repo( repo="https://github.com/pappasam/toml-sort", - hooks=[CommentedMap(id="toml-sort", args=YAML(typ="rt").load("[--in-place]"))], + rev="", + hooks=[Hook(id="toml-sort", args=YAML(typ="rt").load("[--in-place]"))], ) excludes = [] if glob("labels/*.toml"): @@ -128,9 +132,10 @@ def _update_taplo_config() -> None: def _update_precommit_repo() -> None: - expected_hook = CommentedMap( + expected_hook = Repo( repo="https://github.com/ComPWA/mirrors-taplo", - hooks=[CommentedMap(id="taplo")], + rev="", + hooks=[Hook(id="taplo")], ) update_single_hook_precommit_repo(expected_hook) diff --git a/src/compwa_policy/self_check.py b/src/compwa_policy/self_check.py index e06e8874..8d2d65c4 100644 --- a/src/compwa_policy/self_check.py +++ b/src/compwa_policy/self_check.py @@ -2,7 +2,6 @@ from __future__ import annotations -from functools import lru_cache from io import StringIO from textwrap import dedent, indent @@ -10,54 +9,51 @@ from compwa_policy.errors import PrecommitError from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import Hook, PrecommitConfig, asdict, fromdict +from compwa_policy.utilities.precommit import Hook, load_round_trip_precommit_config __HOOK_DEFINITION_FILE = ".pre-commit-hooks.yaml" -__IGNORE_KEYS = {"args"} def main() -> int: - cfg = PrecommitConfig.load() - local_repos = [repo for repo in cfg.repos if repo.repo == "local"] + cfg, _ = load_round_trip_precommit_config() + local_repos = [repo for repo in cfg["repos"] if repo["repo"] == "local"] + hook_definitions = _load_precommit_hook_definitions() executor = Executor() for repo in local_repos: - for hook in repo.hooks: - executor(_check_hook_definition, hook) + for hook in repo["hooks"]: + executor(_check_hook_definition, hook, hook_definitions) return executor.finalize(exception=False) -def _check_hook_definition(hook: Hook) -> None: - hook_definitions = _load_precommit_hook_definitions() - expected = hook_definitions.get(hook.id) +def _load_precommit_hook_definitions() -> dict[str, Hook]: + with open(__HOOK_DEFINITION_FILE) as f: + hooks: list[Hook] = yaml.load(f, Loader=yaml.SafeLoader) + hook_ids = [h["id"] for h in hooks] + if len(hook_ids) != len(set(hook_ids)): + msg = f"{__HOOK_DEFINITION_FILE} contains duplicate IDs" + raise PrecommitError(msg) + return {h["id"]: h for h in hooks} + + +def _check_hook_definition(hook: Hook, definitions: dict[str, Hook]) -> None: + expected = definitions.get(hook["id"]) if expected is None: return - if _to_dict(hook) != _to_dict(expected): + if __reduce(hook) != __reduce(expected): msg = f""" - Local hook with ID '{hook.id}' does not match the definitions in + Local hook with ID '{hook["id"]}' does not match the definitions in {__HOOK_DEFINITION_FILE}. Should be at least: """ msg = dedent(msg).replace("\n", " ") stream = StringIO() - yaml.dump([_to_dict(expected)], stream, sort_keys=False) + yaml.dump([__reduce(expected)], stream, sort_keys=False) expected_content = indent(stream.getvalue(), prefix=" ") raise PrecommitError(msg + "\n\n" + expected_content) -def _to_dict(hook: Hook) -> dict: - hook_dict = asdict(hook) - return {k: v for k, v in hook_dict.items() if k not in __IGNORE_KEYS} - - -@lru_cache(maxsize=None) -def _load_precommit_hook_definitions() -> dict[str, Hook]: - with open(__HOOK_DEFINITION_FILE) as f: - hook_definitions = yaml.load(f, Loader=yaml.SafeLoader) - hooks = [fromdict(h, Hook) for h in hook_definitions] - hook_ids = [h.id for h in hooks] - if len(hook_ids) != len(set(hook_ids)): - msg = f"{__HOOK_DEFINITION_FILE} contains duplicate IDs" - raise PrecommitError(msg) - return {h.id: h for h in hooks} +def __reduce(hook: Hook) -> dict: + ignore_keys = {"args"} + return {k: v for k, v in hook.items() if k not in ignore_keys} if __name__ == "__main__": diff --git a/src/compwa_policy/utilities/precommit.py b/src/compwa_policy/utilities/precommit.py index 8712c741..68e439dd 100644 --- a/src/compwa_policy/utilities/precommit.py +++ b/src/compwa_policy/utilities/precommit.py @@ -2,16 +2,14 @@ from __future__ import annotations -import os.path import re import socket +import sys from functools import lru_cache -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, cast -import attrs -import yaml -from attrs import define, field from pre_commit.commands.autoupdate import autoupdate as precommit_autoupdate +from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.scalarstring import PlainScalarString from compwa_policy.errors import PrecommitError @@ -19,65 +17,70 @@ from . import CONFIG_PATH from .yaml import create_prettier_round_trip_yaml +if sys.version_info < (3, 8): + from typing_extensions import Literal, TypedDict +else: + from typing import Literal, TypedDict +if sys.version_info < (3, 11): + from typing_extensions import NotRequired +else: + from typing import NotRequired if TYPE_CHECKING: from pathlib import Path from ruamel.yaml import YAML - from ruamel.yaml.comments import CommentedMap, CommentedSeq def load_round_trip_precommit_config( path: Path = CONFIG_PATH.precommit, -) -> tuple[CommentedMap, YAML]: +) -> tuple[PrecommitConfig, YAML]: yaml_parser = create_prettier_round_trip_yaml() config = yaml_parser.load(path) return config, yaml_parser -def find_repo( - config: CommentedMap, search_pattern: str -) -> tuple[int, CommentedMap] | None: +def find_repo(config: PrecommitConfig, search_pattern: str) -> tuple[int, Repo] | None: """Find pre-commit hook definition and its index in pre-commit config.""" - repos: CommentedSeq = config.get("repos", []) + repos = config.get("repos", []) for i, repo in enumerate(repos): - url: str = repo.get("repo", "") + url = repo.get("repo", "") if re.search(search_pattern, url): return i, repo return None def remove_precommit_hook(hook_id: str, repo_url: str | None = None) -> None: - config, yaml_parser = load_round_trip_precommit_config() + config, yaml = load_round_trip_precommit_config() repo_and_hook_idx = __find_repo_and_hook_idx(config, hook_id, repo_url) if repo_and_hook_idx is None: return repo_idx, hook_idx = repo_and_hook_idx - repos: CommentedSeq = config["repos"] - hooks: CommentedSeq = repos[repo_idx]["hooks"] + repos = config["repos"] + hooks = repos[repo_idx]["hooks"] if len(hooks) <= 1: repos.pop(repo_idx) else: hooks.pop(hook_idx) - yaml_parser.dump(config, CONFIG_PATH.precommit) + yaml.dump(config, CONFIG_PATH.precommit) msg = f"Removed {hook_id!r} from {CONFIG_PATH.precommit}" raise PrecommitError(msg) def __find_repo_and_hook_idx( - config: CommentedMap, hook_id: str, repo_url: str | None = None + config: PrecommitConfig, hook_id: str, repo_url: str | None = None ) -> tuple[int, int] | None: - repos: CommentedSeq = config.get("repos", []) + repos = config.get("repos", []) for repo_idx, repo in enumerate(repos): if repo_url is not None and repo.get("repo") != repo_url: continue - hooks: CommentedSeq = repo.get("hooks", []) + hooks = repo.get("hooks", []) for hook_idx, hook in enumerate(hooks): if hook.get("id") == hook_id: return repo_idx, hook_idx return None -def update_single_hook_precommit_repo(expected_repo_def: CommentedMap) -> None: +def update_single_hook_precommit_repo(expected: Repo) -> None: """Update the repo definition in :code:`.pre-commit-config.yaml`. If the repository is not yet listed under the :code:`repos` key, a new entry will @@ -86,21 +89,24 @@ def update_single_hook_precommit_repo(expected_repo_def: CommentedMap) -> None: """ if not CONFIG_PATH.precommit.exists(): return - config, yaml_parser = load_round_trip_precommit_config() - repos: CommentedSeq = config.get("repos", []) - repo_url: str = expected_repo_def["repo"] + expected_yaml = CommentedMap(expected) + config, yaml = load_round_trip_precommit_config() + repos = config.get("repos", []) + repo_url = expected["repo"] idx_and_repo = find_repo(config, repo_url) - hook_id: str = expected_repo_def["hooks"][0]["id"] + hook_id = expected["hooks"][0]["id"] if idx_and_repo is None: - if "rev" not in expected_repo_def: - expected_repo_def.insert(1, "rev", "PLEASE-UPDATE") + if not expected_yaml.get("rev"): + expected_yaml.pop("rev", None) + expected_yaml.insert(1, "rev", "PLEASE-UPDATE") idx = _determine_expected_repo_index(config, hook_id) - repos.insert(idx, expected_repo_def) - repos.yaml_set_comment_before_after_key( + repos_yaml = cast(CommentedSeq, repos) + repos_yaml.insert(idx, expected_yaml) + repos_yaml.yaml_set_comment_before_after_key( idx if idx + 1 == len(repos) else idx + 1, before="\n", ) - yaml_parser.dump(config, CONFIG_PATH.precommit) + yaml.dump(config, CONFIG_PATH.precommit) if has_internet_connection(): precommit_autoupdate( CONFIG_PATH.precommit, @@ -111,21 +117,22 @@ def update_single_hook_precommit_repo(expected_repo_def: CommentedMap) -> None: msg = f"Added {hook_id} hook to {CONFIG_PATH.precommit}." raise PrecommitError(msg) idx, existing_hook = idx_and_repo - if not _is_equivalent_repo(existing_hook, expected_repo_def): + if not _is_equivalent_repo(existing_hook, expected): existing_rev = existing_hook.get("rev") if existing_rev is not None: - expected_repo_def.insert(1, "rev", PlainScalarString(existing_rev)) - repos[idx] = expected_repo_def - repos.yaml_set_comment_before_after_key(idx + 1, before="\n") - yaml_parser.dump(config, CONFIG_PATH.precommit) + expected_yaml.insert(1, "rev", PlainScalarString(existing_rev)) + repos[idx] = expected_yaml # type: ignore[assignment,call-overload] + repos_map = cast(CommentedMap, repos) + repos_map.yaml_set_comment_before_after_key(idx + 1, before="\n") + yaml.dump(config, CONFIG_PATH.precommit) msg = f"Updated {hook_id} hook in {CONFIG_PATH.precommit}" raise PrecommitError(msg) -def _determine_expected_repo_index(config: CommentedMap, hook_id: str) -> int: - repos: CommentedSeq = config["repos"] +def _determine_expected_repo_index(config: PrecommitConfig, hook_id: str) -> int: + repos = config["repos"] for i, repo_def in enumerate(repos): - hooks: CommentedSeq = repo_def["hooks"] + hooks = repo_def["hooks"] if len(hooks) != 1: continue if hook_id.lower() <= repo_def["hooks"][0]["id"].lower(): @@ -133,16 +140,16 @@ def _determine_expected_repo_index(config: CommentedMap, hook_id: str) -> int: return len(repos) -def _is_equivalent_repo(expected: CommentedMap, existing: CommentedMap) -> bool: - def remove_rev(config: CommentedMap) -> dict: - config_copy = dict(config) - config_copy.pop("rev", None) - return config_copy +def _is_equivalent_repo(expected: Repo, existing: Repo) -> bool: + def remove_rev(repo: Repo) -> dict: + repo_copy = dict(repo) + repo_copy.pop("rev", None) + return repo_copy return remove_rev(expected) == remove_rev(existing) -def update_precommit_hook(repo_url: str, expected_hook: CommentedMap) -> None: +def update_precommit_hook(repo_url: str, expected_hook: Hook) -> None: """Update the pre-commit hook definition of a specific pre-commit repo. This function updates the :code:`.pre-commit-config.yaml` file, but does this only @@ -150,32 +157,32 @@ def update_precommit_hook(repo_url: str, expected_hook: CommentedMap) -> None: """ if not CONFIG_PATH.precommit.exists(): return - config, yaml_parser = load_round_trip_precommit_config() + config, yaml = load_round_trip_precommit_config() idx_and_repo = find_repo(config, repo_url) if idx_and_repo is None: return repo_idx, repo = idx_and_repo repo_name = repo_url.split("/")[-1] - hooks: CommentedSeq = repo["hooks"] + hooks = repo["hooks"] hook_idx = __find_hook_idx(hooks, expected_hook["id"]) if hook_idx is None: hook_idx = __determine_expected_hook_idx(hooks, expected_hook["id"]) hooks.insert(hook_idx, expected_hook) if hook_idx == len(hooks) - 1: - repos: CommentedMap = config["repos"] + repos = cast(CommentedMap, config["repos"]) repos.yaml_set_comment_before_after_key(repo_idx + 1, before="\n") - yaml_parser.dump(config, CONFIG_PATH.precommit) + yaml.dump(config, CONFIG_PATH.precommit) msg = f"Added {expected_hook['id']!r} to {repo_name} pre-commit config" raise PrecommitError(msg) if hooks[hook_idx] != expected_hook: hooks[hook_idx] = expected_hook - yaml_parser.dump(config, CONFIG_PATH.precommit) + yaml.dump(config, CONFIG_PATH.precommit) msg = f"Updated args of {expected_hook['id']!r} {repo_name} pre-commit hook" raise PrecommitError(msg) -def __find_hook_idx(hooks: CommentedSeq, hook_id: str) -> int | None: +def __find_hook_idx(hooks: list[Hook], hook_id: str) -> int | None: msg = "" for i, hook in enumerate(hooks): msg += " " + hook["id"] @@ -184,127 +191,13 @@ def __find_hook_idx(hooks: CommentedSeq, hook_id: str) -> int | None: return None -def __determine_expected_hook_idx(hooks: CommentedSeq, hook_id: str) -> int: +def __determine_expected_hook_idx(hooks: list[Hook], hook_id: str) -> int: for i, hook in enumerate(hooks): if hook["id"] > hook_id: return i return len(hooks) -@define -class PrecommitCi: - """https://pre-commit.ci/#configuration.""" - - autofix_commit_msg: str = "[pre-commit.ci] auto fixes [...]" - autofix_prs: bool = True - autoupdate_commit_msg: str = "[pre-commit.ci] pre-commit autoupdate" - autoupdate_schedule: str = "weekly" - skip: list[str] | None = None - submodules: bool = False - - -@define -class Hook: - """https://pre-commit.com/#pre-commit-configyaml---hooks.""" - - id: str # noqa: A003 - name: str | None = None - description: str | None = None - entry: str | None = None - alias: str | None = None - additional_dependencies: list[str] = field(factory=list) - args: list[str] = field(factory=list) - files: str | None = None - exclude: str | None = None - types: list[str] | None = None - require_serial: bool = False - language: str | None = None - always_run: bool | None = None - pass_filenames: bool | None = None - types_or: list[str] | None = None - - -@define -class Repo: - """https://pre-commit.com/#pre-commit-configyaml---repos.""" - - repo: str - hooks: list[Hook] - rev: str | None = None - - def get_hook_index(self, hook_id: str) -> int | None: - for i, hook in enumerate(self.hooks): - if hook.id == hook_id: - return i - return None - - -@define -class PrecommitConfig: - """https://pre-commit.com/#pre-commit-configyaml---top-level.""" - - repos: list[Repo] - ci: PrecommitCi | None = None - files: str = "" - exclude: str = "^$" - fail_fast: bool = False - - @classmethod - def load(cls, path: Path | str = CONFIG_PATH.precommit) -> PrecommitConfig: - if not os.path.exists(path): - msg = f"This repository contains no {path}" - raise PrecommitError(msg) - with open(path) as stream: - definition = yaml.safe_load(stream) - return fromdict(definition, PrecommitConfig) - - def find_repo(self, search_pattern: str) -> Repo | None: - for repo in self.repos: - url = repo.repo - if re.search(search_pattern, url): - return repo - return None - - def get_repo_index(self, search_pattern: str) -> int | None: - for i, repo in enumerate(self.repos): - url = repo.repo - if re.search(search_pattern, url): - return i - return None - - -T = TypeVar("T", Hook, PrecommitCi, PrecommitConfig, Repo) - - -def asdict(inst: Any) -> dict: - return attrs.asdict( - inst, - recurse=True, - filter=lambda a, v: a.init and a.default != v, - ) - - -def fromdict(definition: dict, typ: type[T]) -> T: - if typ in {Hook, PrecommitCi}: - return typ(**definition) # type: ignore[return-value] - if typ is Repo: - definition = { - **definition, - "hooks": [fromdict(i, Hook) for i in definition["hooks"]], - } - return Repo(**definition) # type: ignore[return-value] - if typ is PrecommitConfig: - definition = { - **definition, - "repos": [fromdict(i, Repo) for i in definition["repos"]], - } - if "ci" in definition: - definition["ci"] = fromdict(definition["ci"], PrecommitCi) - return PrecommitConfig(**definition) # type: ignore[return-value] - msg = f"No implementation for type {typ.__name__}" - raise NotImplementedError(msg) - - @lru_cache(maxsize=None) def has_internet_connection( host: str = "8.8.8.8", port: int = 53, timeout: float = 0.5 @@ -317,3 +210,55 @@ def has_internet_connection( return False else: return True + + +class PrecommitConfig(TypedDict): + """https://pre-commit.com/#pre-commit-configyaml---top-level.""" + + ci: NotRequired[PrecommitCi] + repos: list[Repo] + default_stages: NotRequired[list[str]] + files: NotRequired[str] + exclude: NotRequired[str] + fail_fast: NotRequired[bool] + minimum_pre_commit_version: NotRequired[str] + + +class PrecommitCi(TypedDict): + """https://pre-commit.ci/#configuration.""" + + autofix_commit_msg: NotRequired[str] + autofix_prs: NotRequired[bool] + autoupdate_branch: NotRequired[str] + autoupdate_commit_msg: NotRequired[str] + autoupdate_schedule: NotRequired[Literal["weekly", "monthly", "quarterly"]] + skip: NotRequired[list[str]] + submodules: NotRequired[bool] + + +class Repo(TypedDict): + """https://pre-commit.com/#pre-commit-configyaml---repos.""" + + repo: str + rev: str + hooks: list[Hook] + + +class Hook(TypedDict): + """https://pre-commit.com/#pre-commit-configyaml---hooks.""" + + id: str + alias: NotRequired[str] + name: NotRequired[str] + language_version: NotRequired[str] + files: NotRequired[str] + exclude: NotRequired[str] + types: NotRequired[list[str]] + types_or: NotRequired[list[str]] + exclude_types: NotRequired[list[str]] + args: NotRequired[list[str]] + stages: NotRequired[list[str]] + additional_dependencies: NotRequired[list[str]] + always_run: NotRequired[bool] + verbose: NotRequired[bool] + log_file: NotRequired[str] diff --git a/tests/.gitignore b/tests/.gitignore deleted file mode 100644 index eed5c11d..00000000 --- a/tests/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!dummy-pre-commit-config.yml diff --git a/tests/check_dev_files/test_cspell.py b/tests/check_dev_files/test_cspell.py index 0e17a337..8d2f86da 100644 --- a/tests/check_dev_files/test_cspell.py +++ b/tests/check_dev_files/test_cspell.py @@ -5,7 +5,7 @@ from compwa_policy.check_dev_files.cspell import _update_cspell_repo_url from compwa_policy.errors import PrecommitError -from compwa_policy.utilities.precommit import PrecommitConfig, fromdict +from compwa_policy.utilities.precommit import PrecommitConfig @pytest.fixture(scope="session") @@ -16,8 +16,7 @@ def test_config_dir(test_dir: Path) -> Path: @pytest.fixture(scope="session") def good_config(test_config_dir: Path) -> PrecommitConfig: with open(test_config_dir / ".pre-commit-config-good.yaml") as stream: - definition = yaml.safe_load(stream) - return fromdict(definition, PrecommitConfig) + return yaml.safe_load(stream) @pytest.mark.parametrize( @@ -48,7 +47,8 @@ def test_update_cspell_repo_url( _update_cspell_repo_url(config_path) with open(config_path) as stream: - definition = yaml.safe_load(stream) - updated_config = fromdict(definition, PrecommitConfig) + definition: PrecommitConfig = yaml.safe_load(stream) - assert updated_config.repos[0].repo == good_config.repos[0].repo + imported = definition["repos"][0]["repo"] + expected = good_config["repos"][0]["repo"] + assert imported == expected diff --git a/tests/utilities/dummy-pre-commit-config.yml b/tests/utilities/dummy-pre-commit-config.yml deleted file mode 100644 index 27f4e242..00000000 --- a/tests/utilities/dummy-pre-commit-config.yml +++ /dev/null @@ -1,38 +0,0 @@ -ci: - autoupdate_commit_msg: "MAINT: autoupdate pre-commit hooks" - autoupdate_schedule: monthly - skip: [flake8, mypy] - -repos: - - repo: meta - hooks: - - id: check-hooks-apply - - id: check-useless-excludes - - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 - hooks: - - id: check-ast - - id: check-case-conflict - - id: check-json - - - repo: https://github.com/psf/black - rev: 21.10b0 - hooks: - - id: black - - - repo: local - hooks: - - id: flake8 - name: flake8 - entry: flake8 - language: system - types: - - python - - - id: mypy - name: mypy - entry: mypy - language: system - types: - - python diff --git a/tests/utilities/test_precommit.py b/tests/utilities/test_precommit.py deleted file mode 100644 index 3c7d84f3..00000000 --- a/tests/utilities/test_precommit.py +++ /dev/null @@ -1,82 +0,0 @@ -from pathlib import Path - -import pytest - -from compwa_policy.utilities.precommit import ( - Hook, - PrecommitCi, - PrecommitConfig, - Repo, - fromdict, -) - - -@pytest.fixture(scope="session") -def dummy_config() -> PrecommitConfig: - this_dir = Path(__file__).parent - return PrecommitConfig.load(this_dir / "dummy-pre-commit-config.yml") - - -class TestPrecommitConfig: - @pytest.fixture(scope="session") - def dummy_config(self) -> PrecommitConfig: - this_dir = Path(__file__).parent - return PrecommitConfig.load(this_dir / "dummy-pre-commit-config.yml") - - def test_find_repo(self, dummy_config: PrecommitConfig): - repo = dummy_config.find_repo("non-existent") - assert repo is None - repo = dummy_config.find_repo("meta") - assert repo is not None - assert repo.repo == "meta" - assert len(repo.hooks) == 2 - repo = dummy_config.find_repo("black") - assert repo is not None - assert repo.repo == "https://github.com/psf/black" - - def test_get_repo_index(self, dummy_config: PrecommitConfig): - assert dummy_config.get_repo_index("non-existent") is None - assert dummy_config.get_repo_index("meta") == 0 - assert dummy_config.get_repo_index("pre-commit-hooks") == 1 - assert dummy_config.get_repo_index(r"^.*/pre-commit-hooks$") == 1 - assert dummy_config.get_repo_index("https://github.com/psf/black") == 2 - - def test_load(self, dummy_config: PrecommitConfig): - assert dummy_config.ci is not None - assert dummy_config.ci.autoupdate_schedule == "monthly" - assert dummy_config.ci.skip == ["flake8", "mypy"] - assert len(dummy_config.repos) == 4 - - def test_load_default(self): - config = PrecommitConfig.load() - repo_names = {repo.repo for repo in config.repos} - assert repo_names >= { - "https://github.com/pre-commit/pre-commit-hooks", - "https://github.com/psf/black-pre-commit-mirror", - } - - -class TestRepo: - def test_get_hook_index(self, dummy_config: PrecommitConfig): - repo = dummy_config.find_repo("local") - assert repo is not None - assert repo.get_hook_index("non-existent") is None - assert repo.get_hook_index("mypy") == 1 - - -def test_fromdict(): - hook_def = {"id": "test"} - hook = Hook(id="test") - assert fromdict(hook_def, Hook) == hook - - repo_def = {"repo": "url", "hooks": [hook_def]} - repo = Repo(repo="url", hooks=[hook]) - assert fromdict(repo_def, Repo) == repo - - ci_def = {"autofix_prs": False} - ci = PrecommitCi(autofix_prs=False) - assert fromdict(ci_def, PrecommitCi) == ci - - config_def = {"repos": [repo_def], "ci": ci_def} - config = PrecommitConfig(repos=[repo], ci=ci) - assert fromdict(config_def, PrecommitConfig) == config From 27535684f1edcc756c9a54d7a65c2b06a272123a Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:33:49 +0100 Subject: [PATCH 2/4] BREAK: define `load_precommit_config()` function without roundtrip --- src/compwa_policy/check_dev_files/citation.py | 4 ++-- src/compwa_policy/check_dev_files/cspell.py | 7 ++++--- .../check_dev_files/github_workflows.py | 6 +++--- src/compwa_policy/check_dev_files/nbstripout.py | 4 ++-- src/compwa_policy/check_dev_files/precommit.py | 9 +++++---- src/compwa_policy/check_dev_files/prettier.py | 7 ++----- .../check_dev_files/update_pip_constraints.py | 4 ++-- src/compwa_policy/self_check.py | 4 ++-- src/compwa_policy/utilities/precommit.py | 15 +++++++++++---- src/compwa_policy/utilities/pyproject.py | 7 ++----- 10 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/compwa_policy/check_dev_files/citation.py b/src/compwa_policy/check_dev_files/citation.py index d799a762..5106b5f8 100644 --- a/src/compwa_policy/check_dev_files/citation.py +++ b/src/compwa_policy/check_dev_files/citation.py @@ -22,7 +22,7 @@ Hook, Repo, find_repo, - load_round_trip_precommit_config, + load_roundtrip_precommit_config, update_single_hook_precommit_repo, ) @@ -185,7 +185,7 @@ def add_json_schema_precommit() -> None: "CITATION.cff", ], ) - config, yaml = load_round_trip_precommit_config() + config, yaml = load_roundtrip_precommit_config() repo_url = "https://github.com/python-jsonschema/check-jsonschema" idx_and_repo = find_repo(config, repo_url) existing_repos = config["repos"] diff --git a/src/compwa_policy/check_dev_files/cspell.py b/src/compwa_policy/check_dev_files/cspell.py index ffd5f796..de7d80f1 100644 --- a/src/compwa_policy/check_dev_files/cspell.py +++ b/src/compwa_policy/check_dev_files/cspell.py @@ -18,7 +18,8 @@ Hook, Repo, find_repo, - load_round_trip_precommit_config, + load_precommit_config, + load_roundtrip_precommit_config, update_single_hook_precommit_repo, ) from compwa_policy.utilities.readme import add_badge, remove_badge @@ -49,7 +50,7 @@ def main(no_cspell_update: bool) -> None: executor(_update_cspell_repo_url) has_cspell_hook = False if CONFIG_PATH.cspell.exists(): - config, _ = load_round_trip_precommit_config() + config = load_precommit_config() has_cspell_hook = find_repo(config, __REPO_URL) is not None if not has_cspell_hook: executor(_remove_configuration) @@ -67,7 +68,7 @@ def _update_cspell_repo_url(path: Path = CONFIG_PATH.precommit) -> None: old_url_patters = [ r".*/mirrors-cspell(.git)?$", ] - config, yaml = load_round_trip_precommit_config(path) + config, yaml = load_roundtrip_precommit_config(path) for pattern in old_url_patters: repo_and_idx = find_repo(config, pattern) if repo_and_idx is None: diff --git a/src/compwa_policy/check_dev_files/github_workflows.py b/src/compwa_policy/check_dev_files/github_workflows.py index 1679b2db..508ac989 100644 --- a/src/compwa_policy/check_dev_files/github_workflows.py +++ b/src/compwa_policy/check_dev_files/github_workflows.py @@ -18,7 +18,7 @@ write, ) from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import load_round_trip_precommit_config +from compwa_policy.utilities.precommit import load_precommit_config from compwa_policy.utilities.project_info import PythonVersion, get_pypi_name from compwa_policy.utilities.yaml import create_prettier_round_trip_yaml @@ -181,8 +181,8 @@ def __update_style_section(config: CommentedMap, python_version: PythonVersion) def __is_remove_style_job() -> bool: if not CONFIG_PATH.precommit.exists(): return True - cfg, _ = load_round_trip_precommit_config() - precommit_ci = cfg.get("ci") + config = load_precommit_config() + precommit_ci = config.get("ci") if precommit_ci is not None and "skip" not in precommit_ci: return True return False diff --git a/src/compwa_policy/check_dev_files/nbstripout.py b/src/compwa_policy/check_dev_files/nbstripout.py index 1b2084cf..172ac0bb 100644 --- a/src/compwa_policy/check_dev_files/nbstripout.py +++ b/src/compwa_policy/check_dev_files/nbstripout.py @@ -7,7 +7,7 @@ Hook, Repo, find_repo, - load_round_trip_precommit_config, + load_precommit_config, update_single_hook_precommit_repo, ) @@ -16,7 +16,7 @@ def main() -> None: # cspell:ignore nbconvert showmarkdowntxt if not CONFIG_PATH.precommit.exists(): return - config, _ = load_round_trip_precommit_config() + config = load_precommit_config() repo_url = "https://github.com/kynan/nbstripout" idx_and_repo = find_repo(config, repo_url) if idx_and_repo is None: diff --git a/src/compwa_policy/check_dev_files/precommit.py b/src/compwa_policy/check_dev_files/precommit.py index ce0103b2..84b93890 100644 --- a/src/compwa_policy/check_dev_files/precommit.py +++ b/src/compwa_policy/check_dev_files/precommit.py @@ -14,7 +14,8 @@ from compwa_policy.utilities.precommit import ( PrecommitConfig, find_repo, - load_round_trip_precommit_config, + load_precommit_config, + load_roundtrip_precommit_config, ) from compwa_policy.utilities.yaml import create_prettier_round_trip_yaml @@ -54,7 +55,7 @@ def __repo_def_sorting(repo_def: CommentedMap) -> tuple[int, str]: def _update_precommit_ci_commit_msg() -> None: if not CONFIG_PATH.precommit.exists(): return - config, yaml = load_round_trip_precommit_config() + config, yaml = load_roundtrip_precommit_config() precommit_ci = config.get("ci") if precommit_ci is None: return @@ -72,7 +73,7 @@ def _update_precommit_ci_commit_msg() -> None: def _update_precommit_ci_skip() -> None: - config, _ = load_round_trip_precommit_config() + config = load_precommit_config() if config.get("ci") is None: return local_hooks = get_local_hooks(config) @@ -142,7 +143,7 @@ def _update_conda_environment() -> None: conda_env: CommentedMap = yaml.load(path) variables: CommentedMap = conda_env.get("variables", {}) key = "PRETTIER_LEGACY_CLI" - precommit_config, _ = load_round_trip_precommit_config() + precommit_config = load_precommit_config() if __has_prettier_v4alpha(precommit_config): if key not in variables: variables[key] = DoubleQuotedScalarString("1") diff --git a/src/compwa_policy/check_dev_files/prettier.py b/src/compwa_policy/check_dev_files/prettier.py index f41eb7a9..04fdd669 100644 --- a/src/compwa_policy/check_dev_files/prettier.py +++ b/src/compwa_policy/check_dev_files/prettier.py @@ -8,10 +8,7 @@ from compwa_policy.errors import PrecommitError from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH, vscode from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import ( - find_repo, - load_round_trip_precommit_config, -) +from compwa_policy.utilities.precommit import find_repo, load_precommit_config from compwa_policy.utilities.readme import add_badge, remove_badge # cspell:ignore esbenp rettier @@ -27,7 +24,7 @@ def main(no_prettierrc: bool) -> None: - config, _ = load_round_trip_precommit_config() + config = load_precommit_config() if find_repo(config, r".*/mirrors-prettier") is None: _remove_configuration() else: diff --git a/src/compwa_policy/check_dev_files/update_pip_constraints.py b/src/compwa_policy/check_dev_files/update_pip_constraints.py index 601265b1..623f9dd3 100644 --- a/src/compwa_policy/check_dev_files/update_pip_constraints.py +++ b/src/compwa_policy/check_dev_files/update_pip_constraints.py @@ -16,7 +16,7 @@ from compwa_policy.errors import PrecommitError from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import load_round_trip_precommit_config +from compwa_policy.utilities.precommit import load_precommit_config from compwa_policy.utilities.yaml import create_prettier_round_trip_yaml if sys.version_info < (3, 8): @@ -101,7 +101,7 @@ def _check_precommit_schedule() -> None: ) if not CONFIG_PATH.precommit.exists(): raise PrecommitError(msg) - config, _ = load_round_trip_precommit_config() + config = load_precommit_config() schedule = config.get("ci", {}).get("autoupdate_schedule") if schedule is None: raise PrecommitError(msg) diff --git a/src/compwa_policy/self_check.py b/src/compwa_policy/self_check.py index 8d2d65c4..23f69ac5 100644 --- a/src/compwa_policy/self_check.py +++ b/src/compwa_policy/self_check.py @@ -9,13 +9,13 @@ from compwa_policy.errors import PrecommitError from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import Hook, load_round_trip_precommit_config +from compwa_policy.utilities.precommit import Hook, load_precommit_config __HOOK_DEFINITION_FILE = ".pre-commit-hooks.yaml" def main() -> int: - cfg, _ = load_round_trip_precommit_config() + cfg = load_precommit_config() local_repos = [repo for repo in cfg["repos"] if repo["repo"] == "local"] hook_definitions = _load_precommit_hook_definitions() executor = Executor() diff --git a/src/compwa_policy/utilities/precommit.py b/src/compwa_policy/utilities/precommit.py index 68e439dd..fc8989e5 100644 --- a/src/compwa_policy/utilities/precommit.py +++ b/src/compwa_policy/utilities/precommit.py @@ -31,9 +31,16 @@ from ruamel.yaml import YAML -def load_round_trip_precommit_config( +def load_precommit_config(path: Path = CONFIG_PATH.precommit) -> PrecommitConfig: + """Load a **read-only** pre-commit config.""" + config, _ = load_roundtrip_precommit_config(path) + return config + + +def load_roundtrip_precommit_config( path: Path = CONFIG_PATH.precommit, ) -> tuple[PrecommitConfig, YAML]: + """Load the pre-commit config as a round-trip YAML object.""" yaml_parser = create_prettier_round_trip_yaml() config = yaml_parser.load(path) return config, yaml_parser @@ -50,7 +57,7 @@ def find_repo(config: PrecommitConfig, search_pattern: str) -> tuple[int, Repo] def remove_precommit_hook(hook_id: str, repo_url: str | None = None) -> None: - config, yaml = load_round_trip_precommit_config() + config, yaml = load_roundtrip_precommit_config() repo_and_hook_idx = __find_repo_and_hook_idx(config, hook_id, repo_url) if repo_and_hook_idx is None: return @@ -90,7 +97,7 @@ def update_single_hook_precommit_repo(expected: Repo) -> None: if not CONFIG_PATH.precommit.exists(): return expected_yaml = CommentedMap(expected) - config, yaml = load_round_trip_precommit_config() + config, yaml = load_roundtrip_precommit_config() repos = config.get("repos", []) repo_url = expected["repo"] idx_and_repo = find_repo(config, repo_url) @@ -157,7 +164,7 @@ def update_precommit_hook(repo_url: str, expected_hook: Hook) -> None: """ if not CONFIG_PATH.precommit.exists(): return - config, yaml = load_round_trip_precommit_config() + config, yaml = load_roundtrip_precommit_config() idx_and_repo = find_repo(config, repo_url) if idx_and_repo is None: return diff --git a/src/compwa_policy/utilities/pyproject.py b/src/compwa_policy/utilities/pyproject.py index 483cabcb..2175a1ac 100644 --- a/src/compwa_policy/utilities/pyproject.py +++ b/src/compwa_policy/utilities/pyproject.py @@ -13,10 +13,7 @@ from compwa_policy.errors import PrecommitError from compwa_policy.utilities import CONFIG_PATH from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.precommit import ( - find_repo, - load_round_trip_precommit_config, -) +from compwa_policy.utilities.precommit import find_repo, load_precommit_config if TYPE_CHECKING: from tomlkit.container import Container @@ -184,7 +181,7 @@ def update_nbqa_settings(key: str, expected: Any) -> None: def __has_nbqa_precommit_repo() -> bool: - config, _ = load_round_trip_precommit_config() + config = load_precommit_config() nbqa_repo = find_repo(config, "https://github.com/nbQA-dev/nbQA") if nbqa_repo is None: return False From 3fb040790678983b74df882e49d9f2a25d4a5bdd Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:42:33 +0100 Subject: [PATCH 3/4] BREAK: do not return index in `find_repo()` function --- src/compwa_policy/check_dev_files/citation.py | 4 ++-- src/compwa_policy/check_dev_files/cspell.py | 7 +++---- .../check_dev_files/nbstripout.py | 7 +++---- .../check_dev_files/precommit.py | 5 ++--- src/compwa_policy/utilities/precommit.py | 20 +++++++++++++++---- src/compwa_policy/utilities/pyproject.py | 5 +---- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/compwa_policy/check_dev_files/citation.py b/src/compwa_policy/check_dev_files/citation.py index 5106b5f8..cb064e51 100644 --- a/src/compwa_policy/check_dev_files/citation.py +++ b/src/compwa_policy/check_dev_files/citation.py @@ -21,7 +21,7 @@ from compwa_policy.utilities.precommit import ( Hook, Repo, - find_repo, + find_repo_with_index, load_roundtrip_precommit_config, update_single_hook_precommit_repo, ) @@ -187,7 +187,7 @@ def add_json_schema_precommit() -> None: ) config, yaml = load_roundtrip_precommit_config() repo_url = "https://github.com/python-jsonschema/check-jsonschema" - idx_and_repo = find_repo(config, repo_url) + idx_and_repo = find_repo_with_index(config, repo_url) existing_repos = config["repos"] if idx_and_repo is None: repo = Repo( diff --git a/src/compwa_policy/check_dev_files/cspell.py b/src/compwa_policy/check_dev_files/cspell.py index de7d80f1..99e12e56 100644 --- a/src/compwa_policy/check_dev_files/cspell.py +++ b/src/compwa_policy/check_dev_files/cspell.py @@ -70,11 +70,10 @@ def _update_cspell_repo_url(path: Path = CONFIG_PATH.precommit) -> None: ] config, yaml = load_roundtrip_precommit_config(path) for pattern in old_url_patters: - repo_and_idx = find_repo(config, pattern) - if repo_and_idx is None: + repo = find_repo(config, pattern) + if repo is None: continue - _, repo_def = repo_and_idx - repo_def["repo"] = __REPO_URL + repo["repo"] = __REPO_URL yaml.dump(config, path) msg = f"Updated cSpell pre-commit repo URL to {__REPO_URL} in {path}" raise PrecommitError(msg) diff --git a/src/compwa_policy/check_dev_files/nbstripout.py b/src/compwa_policy/check_dev_files/nbstripout.py index 172ac0bb..9bf1cc7d 100644 --- a/src/compwa_policy/check_dev_files/nbstripout.py +++ b/src/compwa_policy/check_dev_files/nbstripout.py @@ -18,8 +18,7 @@ def main() -> None: return config = load_precommit_config() repo_url = "https://github.com/kynan/nbstripout" - idx_and_repo = find_repo(config, repo_url) - if idx_and_repo is None: + if find_repo(config, repo_url) is None: return extra_keys_argument = [ "cell.attachments", @@ -40,7 +39,7 @@ def main() -> None: "metadata.varInspector", "metadata.vscode", ] - expected_hook = Repo( + expected_repo = Repo( repo=repo_url, rev="", hooks=[ @@ -53,4 +52,4 @@ def main() -> None: ) ], ) - update_single_hook_precommit_repo(expected_hook) + update_single_hook_precommit_repo(expected_repo) diff --git a/src/compwa_policy/check_dev_files/precommit.py b/src/compwa_policy/check_dev_files/precommit.py index 84b93890..1dd15426 100644 --- a/src/compwa_policy/check_dev_files/precommit.py +++ b/src/compwa_policy/check_dev_files/precommit.py @@ -172,9 +172,8 @@ def __get_skipped_hooks(config: PrecommitConfig) -> set[str]: def __has_prettier_v4alpha(config: PrecommitConfig) -> bool: - idx_and_repo = find_repo(config, search_pattern=r"^.*/mirrors-prettier$") - if idx_and_repo is None: + repo = find_repo(config, search_pattern=r"^.*/mirrors-prettier$") + if repo is None: return False - _, repo = idx_and_repo rev = repo.get("rev", "") return rev.startswith("v4") and "alpha" in rev diff --git a/src/compwa_policy/utilities/precommit.py b/src/compwa_policy/utilities/precommit.py index fc8989e5..7684706a 100644 --- a/src/compwa_policy/utilities/precommit.py +++ b/src/compwa_policy/utilities/precommit.py @@ -46,8 +46,20 @@ def load_roundtrip_precommit_config( return config, yaml_parser -def find_repo(config: PrecommitConfig, search_pattern: str) -> tuple[int, Repo] | None: - """Find pre-commit hook definition and its index in pre-commit config.""" +def find_repo(config: PrecommitConfig, search_pattern: str) -> Repo | None: + """Find pre-commit repo definition in pre-commit config.""" + repos = config.get("repos", []) + for repo in repos: + url = repo.get("repo", "") + if re.search(search_pattern, url): + return repo + return None + + +def find_repo_with_index( + config: PrecommitConfig, search_pattern: str +) -> tuple[int, Repo] | None: + """Find pre-commit repo definition and its index in pre-commit config.""" repos = config.get("repos", []) for i, repo in enumerate(repos): url = repo.get("repo", "") @@ -100,7 +112,7 @@ def update_single_hook_precommit_repo(expected: Repo) -> None: config, yaml = load_roundtrip_precommit_config() repos = config.get("repos", []) repo_url = expected["repo"] - idx_and_repo = find_repo(config, repo_url) + idx_and_repo = find_repo_with_index(config, repo_url) hook_id = expected["hooks"][0]["id"] if idx_and_repo is None: if not expected_yaml.get("rev"): @@ -165,7 +177,7 @@ def update_precommit_hook(repo_url: str, expected_hook: Hook) -> None: if not CONFIG_PATH.precommit.exists(): return config, yaml = load_roundtrip_precommit_config() - idx_and_repo = find_repo(config, repo_url) + idx_and_repo = find_repo_with_index(config, repo_url) if idx_and_repo is None: return repo_idx, repo = idx_and_repo diff --git a/src/compwa_policy/utilities/pyproject.py b/src/compwa_policy/utilities/pyproject.py index 2175a1ac..1a53d407 100644 --- a/src/compwa_policy/utilities/pyproject.py +++ b/src/compwa_policy/utilities/pyproject.py @@ -182,7 +182,4 @@ def update_nbqa_settings(key: str, expected: Any) -> None: def __has_nbqa_precommit_repo() -> bool: config = load_precommit_config() - nbqa_repo = find_repo(config, "https://github.com/nbQA-dev/nbQA") - if nbqa_repo is None: - return False - return True + return find_repo(config, "https://github.com/nbQA-dev/nbQA") is not None From af272a012d897757c39eeb50abe54ae647d1ee6b Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Fri, 12 Jan 2024 15:20:41 +0100 Subject: [PATCH 4/4] FIX: install `typing-extensions` in Python 3.10 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b7af7581..a7a74a7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "pre-commit", "ruamel.yaml", # better YAML dumping "tomlkit", - 'typing-extensions; python_version <"3.10.0"', + 'typing-extensions; python_version <"3.11.0"', ] description = "Pre-commit hooks that ensure that ComPWA repositories have a similar developer set-up" dynamic = ["version"]