From 9784efe8b3e0c8567ee6636a0f08d9ed597e727a Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Tue, 11 Jun 2024 15:21:59 -0500 Subject: [PATCH 1/9] make filter actually work --- ni_python_styleguide/_fix.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 178c519..78d00ad 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -101,9 +101,9 @@ def fix( all_files.extend(file_path.rglob("*.py")) else: all_files.append(file_path) - all_files = filter( - lambda o: not any([fnmatch(o, exclude_) for exclude_ in exclude.split(",")]), all_files - ) + all_files = list(filter( + lambda o: not any([fnmatch(pathlib.Path(o).resolve().relative_to(pathlib.Path.cwd()).as_posix(), exclude_ + "/*") for exclude_ in exclude.split(",")]), all_files + )) for file in all_files: if not file.is_file(): # doesn't really exist... continue From b6805bc8a3bb8194a412b8ba162fc8731dbbfd13 Mon Sep 17 00:00:00 2001 From: mshafer-NI Date: Tue, 11 Jun 2024 15:48:01 -0500 Subject: [PATCH 2/9] log when we fail to read a file, what file it was --- .../_acknowledge_existing_errors/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 2a693dc..112e2a3 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -134,7 +134,11 @@ def _handle_emergent_violations(exclude, app_import_names, extend_ignore, file_o def remove_auto_suppressions_from_file(file: pathlib.Path): """Removes auto-suppressions from file.""" - lines = file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() + try: + lines = file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() + except: + _module_logger.warning("Failed to read %s with encoding %s", file, _utils.DEFAULT_ENCODING) + raise stripped_lines = [_filter_suppresion_from_line(line) for line in lines] file.write_text("\n".join(stripped_lines) + "\n", encoding=_utils.DEFAULT_ENCODING) From 734486f18affd2bebddb5b7668c84bc7ed94d131 Mon Sep 17 00:00:00 2001 From: mshafer-NI Date: Tue, 11 Jun 2024 15:55:41 -0500 Subject: [PATCH 3/9] make a failing test --- tests/test_cli/test_fix.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py index 9b14b74..8bd2142 100644 --- a/tests/test_cli/test_fix.py +++ b/tests/test_cli/test_fix.py @@ -51,6 +51,9 @@ def test_given_bad_input__fix__produces_expected_output_aggressive( in_file = test_dir / "input.py" test_file = tmp_path / "input.py" shutil.copyfile(in_file, test_file) + test_loading_file = tmp_path / ".venv/file_with_non_unicode_chars.py" + test_loading_file.parent.mkdir(parents=True, exist_ok=True) + test_loading_file.write_text("Förward to victory!", encoding="ISO 8859-1") output = styleguide_command(command="fix", command_args=["--aggressive"]) From dcab57b48b238f724d3d614c51fdd16cfd7ed2bb Mon Sep 17 00:00:00 2001 From: mshafer-NI Date: Tue, 11 Jun 2024 16:03:37 -0500 Subject: [PATCH 4/9] make it pass --- ni_python_styleguide/_fix.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 78d00ad..767444c 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -101,9 +101,20 @@ def fix( all_files.extend(file_path.rglob("*.py")) else: all_files.append(file_path) - all_files = list(filter( - lambda o: not any([fnmatch(pathlib.Path(o).resolve().relative_to(pathlib.Path.cwd()).as_posix(), exclude_ + "/*") for exclude_ in exclude.split(",")]), all_files - )) + all_files = list( + filter( + lambda o: not any( + [ + fnmatch(pathlib.Path(o).as_posix(), (exclude_ + "/*")) + or fnmatch(pathlib.Path(o).as_posix(), (exclude_)) + or (not exclude.startswith("/") and fnmatch(pathlib.Path(o).as_posix(), ("*/" + exclude_))) + or (not exclude.startswith("/") and fnmatch(pathlib.Path(o).as_posix(), ("*/" + exclude_ + "/*"))) + for exclude_ in exclude.split(",") + ] + ), + all_files, + ) + ) for file in all_files: if not file.is_file(): # doesn't really exist... continue From 2900ac288beaeb28b78ed32e8296a40e54cd6d05 Mon Sep 17 00:00:00 2001 From: mshafer-NI Date: Tue, 11 Jun 2024 16:56:51 -0500 Subject: [PATCH 5/9] print the exception in the test --- tests/test_cli/test_fix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py index 8bd2142..92deaec 100644 --- a/tests/test_cli/test_fix.py +++ b/tests/test_cli/test_fix.py @@ -57,6 +57,7 @@ def test_given_bad_input__fix__produces_expected_output_aggressive( output = styleguide_command(command="fix", command_args=["--aggressive"]) + assert not output.exception, f"Error in running:\n{output.exception}" assert output.exit_code in (True, 0), f"Error in running:\n{output}" result = test_file.read_text(encoding="UTF-8") snapshot.snapshot_dir = test_dir From 32594eb740b22a9c41b10706f320355e16117522 Mon Sep 17 00:00:00 2001 From: mshafer-NI Date: Tue, 11 Jun 2024 16:57:30 -0500 Subject: [PATCH 6/9] switch to pathspec for globbing/matching in fix --- ni_python_styleguide/_fix.py | 27 ++++++++++++--------------- poetry.lock | 2 +- pyproject.toml | 1 + 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 767444c..05fa2a8 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -5,6 +5,7 @@ from typing import Iterable import isort +import pathspec from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _config_constants @@ -93,28 +94,24 @@ def fix( ): """Fix basic linter errors and format.""" file_or_dir = file_or_dir or ["."] + extend_ignore = extend_ignore or "" + exclude = exclude or "" if aggressive: + glob_spec = pathspec.PathSpec.from_lines( + "gitwildmatch", + ["*.py"] + + [f"!{exclude_}" for exclude_ in exclude.split(",") if exclude_] + + [f"!{exclude_}/*" for exclude_ in exclude.split(",") if exclude_] + + [f"!{ignore_}" for ignore_ in extend_ignore.split(",") if ignore_] + + [f"!{ignore_}/*" for ignore_ in extend_ignore.split(",") if ignore_], + ) all_files = [] for file_or_dir_ in file_or_dir: file_path = pathlib.Path(file_or_dir_) if file_path.is_dir(): - all_files.extend(file_path.rglob("*.py")) + all_files.extend([pathlib.Path(o) for o in glob_spec.match_tree(str(file_path), negate=False)]) else: all_files.append(file_path) - all_files = list( - filter( - lambda o: not any( - [ - fnmatch(pathlib.Path(o).as_posix(), (exclude_ + "/*")) - or fnmatch(pathlib.Path(o).as_posix(), (exclude_)) - or (not exclude.startswith("/") and fnmatch(pathlib.Path(o).as_posix(), ("*/" + exclude_))) - or (not exclude.startswith("/") and fnmatch(pathlib.Path(o).as_posix(), ("*/" + exclude_ + "/*"))) - for exclude_ in exclude.split(",") - ] - ), - all_files, - ) - ) for file in all_files: if not file.is_file(): # doesn't really exist... continue diff --git a/poetry.lock b/poetry.lock index f73217a..f2b1908 100644 --- a/poetry.lock +++ b/poetry.lock @@ -406,7 +406,7 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "8c3f0d8fe73c715b00b20afd4b8334b143754e3899fd2fa54125708d79190a5a" +content-hash = "fcfd1ca17126ff6de77487c045b7a1d61119c3e525c452df9ad8916945cc51c5" [metadata.files] black = [ diff --git a/pyproject.toml b/pyproject.toml index d88e988..304f3af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ pep8-naming = ">=0.11.1" # Tooling that we're locking so our tool can run importlib-metadata = {version= "<5.0", python="<3.8"} +pathspec = ">=0.11.1" [tool.poetry.dev-dependencies] From c84c1e0636d5ef7dc08264e3270466bbab9af1a2 Mon Sep 17 00:00:00 2001 From: mshafer-NI Date: Tue, 11 Jun 2024 16:59:52 -0500 Subject: [PATCH 7/9] move pathspec to core tooling section --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 304f3af..4e439e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ include = ["ni_python_styleguide/config.toml"] [tool.poetry.dependencies] python = "^3.7" +pathspec = ">=0.11.1" # Tools we aggregate flake8 = [ @@ -46,7 +47,6 @@ pep8-naming = ">=0.11.1" # Tooling that we're locking so our tool can run importlib-metadata = {version= "<5.0", python="<3.8"} -pathspec = ">=0.11.1" [tool.poetry.dev-dependencies] From 6ae83b82166d58f4044f7212cba8a10da382cc46 Mon Sep 17 00:00:00 2001 From: mshafer-NI Date: Tue, 11 Jun 2024 17:04:16 -0500 Subject: [PATCH 8/9] format / linter issues --- .../_acknowledge_existing_errors/__init__.py | 2 +- ni_python_styleguide/_fix.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 112e2a3..fa60e58 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -136,7 +136,7 @@ def remove_auto_suppressions_from_file(file: pathlib.Path): """Removes auto-suppressions from file.""" try: lines = file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() - except: + except Exception: _module_logger.warning("Failed to read %s with encoding %s", file, _utils.DEFAULT_ENCODING) raise stripped_lines = [_filter_suppresion_from_line(line) for line in lines] diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 05fa2a8..a62f945 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -1,16 +1,17 @@ import logging import pathlib from collections import defaultdict -from fnmatch import fnmatch from typing import Iterable import isort import pathspec -from ni_python_styleguide import _acknowledge_existing_errors -from ni_python_styleguide import _config_constants -from ni_python_styleguide import _format -from ni_python_styleguide import _utils +from ni_python_styleguide import ( + _acknowledge_existing_errors, + _config_constants, + _format, + _utils, +) _module_logger = logging.getLogger(__name__) @@ -99,8 +100,8 @@ def fix( if aggressive: glob_spec = pathspec.PathSpec.from_lines( "gitwildmatch", - ["*.py"] + - [f"!{exclude_}" for exclude_ in exclude.split(",") if exclude_] + ["*.py"] + + [f"!{exclude_}" for exclude_ in exclude.split(",") if exclude_] + [f"!{exclude_}/*" for exclude_ in exclude.split(",") if exclude_] + [f"!{ignore_}" for ignore_ in extend_ignore.split(",") if ignore_] + [f"!{ignore_}/*" for ignore_ in extend_ignore.split(",") if ignore_], @@ -109,7 +110,9 @@ def fix( for file_or_dir_ in file_or_dir: file_path = pathlib.Path(file_or_dir_) if file_path.is_dir(): - all_files.extend([pathlib.Path(o) for o in glob_spec.match_tree(str(file_path), negate=False)]) + all_files.extend( + [pathlib.Path(o) for o in glob_spec.match_tree(str(file_path), negate=False)] + ) else: all_files.append(file_path) for file in all_files: From 14039bfb43165325a20071c0484d58ef7571251e Mon Sep 17 00:00:00 2001 From: mshafer-NI Date: Tue, 11 Jun 2024 17:11:27 -0500 Subject: [PATCH 9/9] simplify --- ni_python_styleguide/_fix.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index a62f945..b1f3858 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -102,9 +102,7 @@ def fix( "gitwildmatch", ["*.py"] + [f"!{exclude_}" for exclude_ in exclude.split(",") if exclude_] - + [f"!{exclude_}/*" for exclude_ in exclude.split(",") if exclude_] - + [f"!{ignore_}" for ignore_ in extend_ignore.split(",") if ignore_] - + [f"!{ignore_}/*" for ignore_ in extend_ignore.split(",") if ignore_], + + [f"!{ignore_}" for ignore_ in extend_ignore.split(",") if ignore_], ) all_files = [] for file_or_dir_ in file_or_dir: