From 38a4300ff4a8396a642c892b7af22b5e11299410 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Fri, 20 Sep 2024 17:39:25 -0400 Subject: [PATCH] Add multi-package delete capabilities Query-only mode no longer requires pulling safeties Add `--delete-project` safety for when all versions of the project would be deleted Update documentation Release 0.1.8 closes #23 --- README.md | 276 +++++++++++++++++++++-- build.py | 2 +- src/main/python/pypi_cleanup/__init__.py | 206 +++++++++-------- 3 files changed, 368 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index d716873..dabe022 100644 --- a/README.md +++ b/README.md @@ -38,15 +38,15 @@ Authentication with TOTP is supported. ```bash $ pypi-cleanup --help -usage: pypi-cleanup [-h] [-u USERNAME] -p PACKAGE [-t URL] [-r PATTERNS | --leave-most-recent-only] [--query-only] [--do-it] [-y] [-d DAYS] [-v] +usage: pypi-cleanup [-h] [-u USERNAME] -p PACKAGES [-t URL] [-r PATTERNS | --leave-most-recent-only] [--query-only] [--do-it] [--delete-project] [-y] [-d DAYS] [-v] -PyPi Package Cleanup Utility v0.1.7.dev20240624230606 +PyPi Package Cleanup Utility v0.1.8 options: -h, --help show this help message and exit -u USERNAME, --username USERNAME authentication username (default: None) - -p PACKAGE, --package PACKAGE + -p PACKAGES, --package PACKAGES PyPI package name (default: None) -t URL, --host URL PyPI :// prefix (default: https://pypi.org/) -r PATTERNS, --version-regex PATTERNS @@ -55,25 +55,78 @@ options: delete all releases except the *most recent* one, i.e. the one containing the most recently created files (default: False) --query-only only queries and processes the package, no login required (default: False) --do-it actually perform the destructive delete (default: False) + --delete-project actually perform the destructive delete that will remove all versions of the project (default: False) -y, --yes confirm extremely dangerous destructive delete (default: False) -d DAYS, --days DAYS only delete releases **matching specified patterns** where all files are older than X days (default: 0) -v, --verbose be verbose (default: 0) ``` +#### Query-Only Mode + +Query-only mode is a safe mode that simply displays all package versions matches and exits, without requiring authentication or removing safeties: + +```bash +$ pypi-cleanup -p karellen-llvm-core -p karellen-llvm-clang -r '.*rc\d.*' --query-only +INFO:root:Running in QUERY-ONLY mode +INFO:root:Will use the following patterns [re.compile('.*rc\\d.*')] on package 'karellen-llvm-core' +INFO:root:Found the following releases of package 'karellen-llvm-core' to delete: +INFO:root: 19.1.0.0rc1.post62 +INFO:root: 19.1.0.0rc2 +INFO:root: 19.1.0.0rc2.post43 +INFO:root: 19.1.0.0rc2.post45 +INFO:root: 19.1.0.0rc2.post52 +INFO:root: 19.1.0.0rc2.post59 +INFO:root: 19.1.0.0rc2.post69 +INFO:root: 19.1.0.0rc3 +INFO:root: 19.1.0.0rc3.post12 +INFO:root: 19.1.0.0rc3.post50 +INFO:root: 19.1.0.0rc3.post53 +INFO:root: 19.1.0.0rc4 +INFO:root: 19.1.0.0rc4.post6 +INFO:root: 19.1.0.0rc4.post13 +INFO:root: 19.1.0.0rc4.post18 +INFO:root:Query-only mode - exiting +``` + #### Regular Cleanup of Development Artifacts + +First without the `--do-it` confirmation, i.e. in DRY RUN mode, including authentication and getting as close as possible to deleting without actually doing it: + ```bash $ pypi-cleanup -u arcivanov -p pybuilder +INFO:root:Running in DRY RUN mode +INFO:root:Will use the following patterns [re.compile('.*\\.dev\\d+$')] on package 'pybuilder' +INFO:root:Found the following releases of package 'pybuilder' to delete: +INFO:root: 0.13.13.dev20240604074936 +INFO:root: 0.13.14.dev20240814015648 +Password: +Authentication code: 933344 +INFO:root:Would be deleting 'pybuilder' version 0.13.13.dev20240604074936, but not doing it! +INFO:root:Would be deleting 'pybuilder' version 0.13.14.dev20240814015648, but not doing it! +``` + +Now to actually delete the specificed packages +```bash +$ pypi-cleanup -u arcivanov -p pybuilder --do-it +WARNING:root:!!! POSSIBLE DESTRUCTIVE OPERATION !!! +INFO:root:Will use the following patterns [re.compile('.*\\.dev\\d+$')] on package 'pybuilder' +INFO:root:Found the following releases of package 'pybuilder' to delete: +INFO:root: 0.13.13.dev20240604074936 +INFO:root: 0.13.14.dev20240814015648 Password: Authentication code: 123456 -INFO:root:Deleting pybuilder version 0.12.3.dev20200421010849 -INFO:root:Deleted pybuilder version 0.12.3.dev20200421010849 -INFO:root:Deleting pybuilder version 0.12.3.dev20200421010857 -INFO:root:Deleted pybuilder version 0.12.3.dev20200421010857 +WARNING:root:!!! WILL ACTUALLY DELETE THINGS - LAST CHANCE TO CHANGE YOUR MIND !!! +WARNING:root:Sleeping for 5 seconds - Ctrl-C to abort! +INFO:root:Deleting 'pybuilder' version 0.13.13.dev20240604074936 +INFO:root:Deleted 'pybuilder' version 0.13.13.dev20240604074936 +INFO:root:Deleting 'pybuilder' version 0.13.14.dev20240814015648 +INFO:root:Deleted 'pybuilder' version 0.13.14.dev20240814015648 ``` #### Using Custom Regex Pattern + ```bash -$ pypi-cleanup -u arcivanov -p geventmp -r '.*\\.dev1$' +$ pypi-cleanup -u arcivanov -p geventmp -r '.*\\.dev1$' WARNING:root: WARNING: You're using custom patterns: [re.compile('.*\\\\.dev1$')]. @@ -81,6 +134,9 @@ WARNING: Make sure to test your patterns before running the destructive cleanup. Once you're satisfied the patterns are correct re-run with `-y`/`--yes` to confirm you know what you're doing. Goodbye. +``` + +```bash $ pypi-cleanup -u arcivanov -p geventmp -r '.*\\.dev1$' -y Password: WARNING:root:RUNNING IN DRY-RUN MODE @@ -91,24 +147,14 @@ INFO:root:Deleting geventmp version 0.0.1.dev1 #### Deleting All Versions Except The Most Recent One -```bash -$ pypi-cleanup -p pypi-cleanup --leave-most-recent-only -WARNING:root: -WARNING: - You're trying to delete ALL versions of the package EXCEPT for the *most recent one*, i.e. - the one with the most recent (by the wall clock) files, disregarding the actual version numbers - or versioning schemes! +List all versions that are going to be deleted except for the most recent one: - You can potentially wipe critical versions irrecoverably. - Make sure this is what you really want before running the destructive cleanup. - Once you're sure you want to delete all versions except the most recent one, - re-run with `-y`/`--yes` to confirm you know what you're doing. - Goodbye. -$ pypi-cleanup -p pypi-cleanup --leave-most-recent-only -y --query-only -INFO:root:Running in DRY RUN mode +```bash +$ pypi-cleanup -p pypi-cleanup --leave-most-recent-only --query-only +INFO:root:Running in QUERY-ONLY mode INFO:root:Will only leave the MOST RECENT version of the package 'pypi-cleanup' -INFO:root:Leaving the MOST RECENT package version: 0.1.7.dev20240624221535 - 2024-06-24T22:15:52.778775+0000 -INFO:root:Found the following releases to delete: +INFO:root:Leaving the MOST RECENT version for 'pypi-cleanup': 0.1.7 - 2024-06-25T05:53:47.930884+0000 +INFO:root:Found the following releases of package 'pypi-cleanup' to delete: INFO:root: 0.0.1 INFO:root: 0.0.2 INFO:root: 0.0.3 @@ -121,3 +167,185 @@ INFO:root: 0.1.5 INFO:root: 0.1.6 INFO:root:Query-only mode - exiting ``` + +Proceeding with deletion of everything except the most recent version requires pulling safeties: + +```bash +$ pypi-cleanup -p pypi-cleanup --leave-most-recent-only +WARNING:root: +WARNING: + You're trying to delete ALL versions of the package EXCEPT for the *most recent one*, i.e. + the one with the most recent (by the wall clock) files, disregarding the actual version numbers + or versioning schemes! + + You can potentially wipe critical versions irrecoverably. + Make sure this is what you really want before running the destructive cleanup. + Once you're sure you want to delete all versions except the most recent one, + re-run with `-y`/`--yes` to confirm you know what you're doing. + Goodbye. +``` + +#### Deleting Multiple Packages + +Specify multiple packages by adding additional `-p` arguments as follows: + +```bash +$ pypi-cleanup -p karellen-llvm-core -p karellen-llvm-clang -p karellen-llvm-lldb -p karellen-llvm-toolchain-tools -r '.*rc\d.*' -y -u karellen --do-it +WARNING:root:!!! POSSIBLE DESTRUCTIVE OPERATION !!! +INFO:root:Will use the following patterns [re.compile('.*rc\\d.*')] on package 'karellen-llvm-core' +INFO:root:Will use the following patterns [re.compile('.*rc\\d.*')] on package 'karellen-llvm-clang' +INFO:root:Will use the following patterns [re.compile('.*rc\\d.*')] on package 'karellen-llvm-lldb' +INFO:root:Will use the following patterns [re.compile('.*rc\\d.*')] on package 'karellen-llvm-toolchain-tools' +INFO:root:Found the following releases of package 'karellen-llvm-core' to delete: +INFO:root: 19.1.0.0rc1.post62 +INFO:root: 19.1.0.0rc2 +INFO:root: 19.1.0.0rc2.post43 +INFO:root: 19.1.0.0rc2.post45 +INFO:root: 19.1.0.0rc2.post52 +INFO:root: 19.1.0.0rc2.post59 +INFO:root: 19.1.0.0rc2.post69 +INFO:root: 19.1.0.0rc3 +INFO:root: 19.1.0.0rc3.post12 +INFO:root: 19.1.0.0rc3.post50 +INFO:root: 19.1.0.0rc3.post53 +INFO:root: 19.1.0.0rc4 +INFO:root: 19.1.0.0rc4.post6 +INFO:root: 19.1.0.0rc4.post13 +INFO:root: 19.1.0.0rc4.post18 +INFO:root:Found the following releases of package 'karellen-llvm-clang' to delete: +INFO:root: 19.1.0.0rc1.post62 +INFO:root: 19.1.0.0rc2 +INFO:root: 19.1.0.0rc2.post43 +INFO:root: 19.1.0.0rc2.post45 +INFO:root: 19.1.0.0rc2.post52 +INFO:root: 19.1.0.0rc2.post59 +INFO:root: 19.1.0.0rc2.post69 +INFO:root: 19.1.0.0rc3 +INFO:root: 19.1.0.0rc3.post12 +INFO:root: 19.1.0.0rc3.post50 +INFO:root: 19.1.0.0rc3.post53 +INFO:root: 19.1.0.0rc4 +INFO:root: 19.1.0.0rc4.post6 +INFO:root: 19.1.0.0rc4.post13 +INFO:root: 19.1.0.0rc4.post18 +INFO:root:Found the following releases of package 'karellen-llvm-lldb' to delete: +INFO:root: 19.1.0.0rc4.post6 +INFO:root: 19.1.0.0rc4.post13 +INFO:root: 19.1.0.0rc4.post18 +INFO:root:Found the following releases of package 'karellen-llvm-toolchain-tools' to delete: +INFO:root: 19.1.0.0rc1.post62 +INFO:root: 19.1.0.0rc2 +INFO:root: 19.1.0.0rc2.post43 +INFO:root: 19.1.0.0rc2.post45 +INFO:root: 19.1.0.0rc2.post52 +INFO:root: 19.1.0.0rc2.post59 +INFO:root: 19.1.0.0rc2.post69 +INFO:root: 19.1.0.0rc3 +INFO:root: 19.1.0.0rc3.post12 +INFO:root: 19.1.0.0rc3.post50 +INFO:root: 19.1.0.0rc3.post53 +INFO:root: 19.1.0.0rc4 +INFO:root: 19.1.0.0rc4.post6 +INFO:root: 19.1.0.0rc4.post13 +INFO:root: 19.1.0.0rc4.post18 +Password: +Authentication code: 123456 +WARNING:root:!!! WILL ACTUALLY DELETE THINGS - LAST CHANCE TO CHANGE YOUR MIND !!! +WARNING:root:Sleeping for 5 seconds - Ctrl-C to abort! +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc1.post62 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc1.post62 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc2 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc2 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc2.post43 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc2.post43 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc2.post45 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc2.post45 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc2.post52 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc2.post52 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc2.post59 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc2.post59 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc2.post69 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc2.post69 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc3 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc3 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc3.post12 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc3.post12 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc3.post50 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc3.post50 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc3.post53 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc3.post53 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc4 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc4 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc4.post6 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc4.post6 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc4.post13 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc4.post13 +INFO:root:Deleting 'karellen-llvm-core' version 19.1.0.0rc4.post18 +INFO:root:Deleted 'karellen-llvm-core' version 19.1.0.0rc4.post18 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc1.post62 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc1.post62 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc2 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc2 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc2.post43 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc2.post43 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc2.post45 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc2.post45 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc2.post52 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc2.post52 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc2.post59 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc2.post59 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc2.post69 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc2.post69 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc3 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc3 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc3.post12 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc3.post12 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc3.post50 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc3.post50 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc3.post53 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc3.post53 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc4 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc4 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc4.post6 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc4.post6 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc4.post13 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc4.post13 +INFO:root:Deleting 'karellen-llvm-clang' version 19.1.0.0rc4.post18 +INFO:root:Deleted 'karellen-llvm-clang' version 19.1.0.0rc4.post18 +INFO:root:Deleting 'karellen-llvm-lldb' version 19.1.0.0rc4.post6 +INFO:root:Deleted 'karellen-llvm-lldb' version 19.1.0.0rc4.post6 +INFO:root:Deleting 'karellen-llvm-lldb' version 19.1.0.0rc4.post13 +INFO:root:Deleted 'karellen-llvm-lldb' version 19.1.0.0rc4.post13 +INFO:root:Deleting 'karellen-llvm-lldb' version 19.1.0.0rc4.post18 +INFO:root:Deleted 'karellen-llvm-lldb' version 19.1.0.0rc4.post18 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc1.post62 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc1.post62 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post43 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post43 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post45 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post45 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post52 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post52 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post59 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post59 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post69 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc2.post69 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc3 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc3 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc3.post12 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc3.post12 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc3.post50 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc3.post50 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc3.post53 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc3.post53 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc4 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc4 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc4.post6 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc4.post6 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc4.post13 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc4.post13 +INFO:root:Deleting 'karellen-llvm-toolchain-tools' version 19.1.0.0rc4.post18 +INFO:root:Deleted 'karellen-llvm-toolchain-tools' version 19.1.0.0rc4.post18 +``` diff --git a/build.py b/build.py index 96eeb45..84b5048 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ name = "pypi-cleanup" -version = "0.1.8.dev" +version = "0.1.8" summary = "PyPI Bulk Release Version Cleanup Utility" authors = [Author("Arcadiy Ivanov", "arcadiy@ivanov.biz")] diff --git a/src/main/python/pypi_cleanup/__init__.py b/src/main/python/pypi_cleanup/__init__.py index d49727b..049db0e 100644 --- a/src/main/python/pypi_cleanup/__init__.py +++ b/src/main/python/pypi_cleanup/__init__.py @@ -74,13 +74,16 @@ def handle_endtag(self, tag): class PypiCleanup: - def __init__(self, url, username, package, do_it, patterns, verbose, days, query_only, leave_most_recent_only, **_): + def __init__(self, url, username, packages, do_it, patterns, verbose, days, query_only, leave_most_recent_only, + confirm, delete_project, **_): self.url = urlparse(url).geturl() if self.url[-1] == "/": self.url = self.url[:-1] self.username = username self.do_it = do_it - self.package = package + self.confirm = confirm + self.delete_project = delete_project + self.packages = packages self.patterns = patterns or DEFAULT_PATTERNS self.verbose = verbose self.query_only = query_only @@ -93,85 +96,102 @@ def run(self): if self.verbose: logging.root.setLevel(logging.DEBUG) - if self.do_it: - logging.warning("!!! POSSIBLE DESTRUCTIVE OPERATION !!!") + if not self.query_only: + if self.do_it: + logging.warning("!!! POSSIBLE DESTRUCTIVE OPERATION !!!") + else: + logging.info("Running in DRY RUN mode") else: - logging.info("Running in DRY RUN mode") + logging.info("Running in QUERY-ONLY mode") - if not self.leave_most_recent_only: - logging.info(f"Will use the following patterns {self.patterns} on package {self.package!r}") - else: - logging.info(f"Will only leave the MOST RECENT version of the package {self.package!r}") + for package in self.packages: + if not self.leave_most_recent_only: + logging.info(f"Will use the following patterns {self.patterns} on package {package!r}") + else: + logging.info(f"Will only leave the MOST RECENT version of the package {package!r}") with requests.Session() as s: s.headers.update({"User-Agent": f"pypi-cleanup/{__version__} (requests/{requests_version})"}) - with s.get(f"{self.url}/simple/{self.package}/", - headers={"Accept": "application/vnd.pypi.simple.v1+json"}) as r: - try: - r.raise_for_status() - except RequestException as e: - logging.error(f"Unable to find package {self.package!r}", exc_info=e) - return 1 - - project_info = r.json() - releases_by_date = {} - - def package_matches_file(p, v, f): - filename = f["filename"].lower() - if filename.endswith(".whl") or filename.endswith(".egg") or filename.endswith(".src.rpm"): - return filename.startswith(f"{p.replace('-', '_')}-{v}-") + pkg_to_pkg_vers = {} + for package in self.packages: + with s.get(f"{self.url}/simple/{package}/", + headers={"Accept": "application/vnd.pypi.simple.v1+json"}) as r: + try: + r.raise_for_status() + except RequestException as e: + logging.error(f"Unable to find package {package!r}", exc_info=e) + return 1 - return filename in (f"{p}-{v}.tar.gz", f"{p}-{v}.zip") + project_info = r.json() + releases_by_date = {} - for version in project_info["versions"]: - releases_by_date[version] = max( - [datetime.datetime.strptime(f["upload-time"], "%Y-%m-%dT%H:%M:%S.%f%z") - for f in project_info["files"] - if package_matches_file(self.package, version, f)]) + def package_matches_file(p, v, f): + filename = f["filename"].lower() + if filename.endswith(".whl") or filename.endswith(".egg") or filename.endswith(".src.rpm"): + return filename.startswith(f"{p.replace('-', '_')}-{v}-") - if not releases_by_date: - logging.info(f"No releases for package {self.package!r} have been found") - return + return filename in (f"{p}-{v}.tar.gz", f"{p}-{v}.zip") - if self.leave_most_recent_only: - leave_release = max(releases_by_date, key=releases_by_date.get) - logging.info( - f"Leaving the MOST RECENT package version: {leave_release} - " - f"{releases_by_date[leave_release].strftime('%Y-%m-%dT%H:%M:%S.%f%z')}") - pkg_vers = list(r for r in releases_by_date if r != leave_release) - else: - pkg_vers = list(filter(lambda k: - any(filter(lambda rex: rex.match(k), - self.patterns)) and releases_by_date[k] < self.date, - releases_by_date.keys())) - - if not pkg_vers: - logging.info(f"No releases were found matching specified patterns " - f"and dates in package {self.package!r}") - else: - logging.info("Found the following releases to delete:") - for pkg_ver in pkg_vers: - logging.info(f" {pkg_ver}") + for version in project_info["versions"]: + releases_by_date[version] = max( + [datetime.datetime.strptime(f["upload-time"], "%Y-%m-%dT%H:%M:%S.%f%z") + for f in project_info["files"] + if package_matches_file(package, version, f)]) - if pkg_vers and set(pkg_vers) == set(releases_by_date.keys()): - print(dedent(f""" - WARNING: - \tYou have selected the following patterns: {self.patterns} - \tThese patterns would delete all available released versions of {self.package!r}. - \tThis will render your project/package permanently inaccessible. - \tSince the costs of an error are too high I'm refusing to do this. - \tGoodbye. - """), file=sys.stderr) + if not releases_by_date: + logging.info(f"No releases for package {package!r} have been found") + continue - if not self.do_it: - return 3 + if self.leave_most_recent_only: + leave_release = max(releases_by_date, key=releases_by_date.get) + logging.info( + f"Leaving the MOST RECENT version for {package!r}: {leave_release} - " + f"{releases_by_date[leave_release].strftime('%Y-%m-%dT%H:%M:%S.%f%z')}") + pkg_vers = list(r for r in releases_by_date if r != leave_release) + else: + pkg_vers = list(filter(lambda k: + any(filter(lambda rex: rex.match(k), + self.patterns)) and releases_by_date[k] < self.date, + releases_by_date.keys())) + + if not pkg_vers: + logging.info(f"No releases were found matching specified patterns " + f"and dates in package {package!r}") + else: + logging.info(f"Found the following releases of package {package!r} to delete:") + for pkg_ver in pkg_vers: + logging.info(f" {pkg_ver}") + + if pkg_vers and set(pkg_vers) == set(releases_by_date.keys()): + msg = f""" + WARNING: + \tYou have selected the following patterns: {self.patterns} + \tThese patterns would delete ALL AVAILABLE RELEASED VERSIONS of {package!r}. + \tThis will render your project/package permanently inaccessible. + """ + + if not self.delete_project: + print(dedent(f""" + {msg} + \tSince the costs of an error are too high I'm refusing to do this. + \tGoodbye. + """), file=sys.stderr) + return 3 + else: + print(dedent(f""" + {msg} + \tSince you've specified "--delete-project", I will proceed anyway. + """), file=sys.stderr) + + if pkg_vers: + pkg_to_pkg_vers[package] = pkg_vers if self.query_only: logging.info("Query-only mode - exiting") return - if not pkg_vers: + if not pkg_to_pkg_vers: return password = os.getenv("PYPI_CLEANUP_PASSWORD") @@ -244,30 +264,31 @@ def package_matches_file(p, v, f): logging.warning("Sleeping for 5 seconds - Ctrl-C to abort!") time.sleep(5.0) - for pkg_ver in pkg_vers: - if self.do_it: - logging.info(f"Deleting {self.package!r} version {pkg_ver}") - form_action = f"/manage/project/{self.package}/release/{pkg_ver}/" - form_url = f"{self.url}{form_action}" - with s.get(form_url) as r: - r.raise_for_status() - parser = CsfrParser(form_action, "confirm_delete_version") - parser.feed(r.text) - if not parser.csrf: - raise ValueError(f"No CSFR found in {form_action}") - csrf = parser.csrf - referer = r.url - - with s.post(form_url, - data={"csrf_token": csrf, - "confirm_delete_version": pkg_ver, - }, - headers={"referer": referer}) as r: - r.raise_for_status() - - logging.info(f"Deleted {self.package!r} version {pkg_ver}") - else: - logging.info(f"Would be deleting {self.package!r} version {pkg_ver}, but not doing it!") + for package, pkg_vers in pkg_to_pkg_vers.items(): + for pkg_ver in pkg_vers: + if self.do_it: + logging.info(f"Deleting {package!r} version {pkg_ver}") + form_action = f"/manage/project/{package}/release/{pkg_ver}/" + form_url = f"{self.url}{form_action}" + with s.get(form_url) as r: + r.raise_for_status() + parser = CsfrParser(form_action, "confirm_delete_version") + parser.feed(r.text) + if not parser.csrf: + raise ValueError(f"No CSFR found in {form_action}") + csrf = parser.csrf + referer = r.url + + with s.post(form_url, + data={"csrf_token": csrf, + "confirm_delete_version": pkg_ver, + }, + headers={"referer": referer}) as r: + r.raise_for_status() + + logging.info(f"Deleted {package!r} version {pkg_ver}") + else: + logging.info(f"Would be deleting {package!r} version {pkg_ver}, but not doing it!") def main(): @@ -277,7 +298,8 @@ def main(): parser = argparse.ArgumentParser(description=f"PyPi Package Cleanup Utility v{__version__}", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-u", "--username", help="authentication username") - parser.add_argument("-p", "--package", required=True, help="PyPI package name") + parser.add_argument("-p", "--package", dest="packages", action="append", required=True, + help="PyPI package name") parser.add_argument("-t", "--host", default="https://pypi.org/", dest="url", help="PyPI :// prefix") g = parser.add_mutually_exclusive_group() @@ -290,6 +312,8 @@ def main(): help="only queries and processes the package, no login required") parser.add_argument("--do-it", action="store_true", default=False, help="actually perform the destructive delete") + parser.add_argument("--delete-project", action="store_true", default=False, + help="actually perform the destructive delete that will remove all versions of the project") parser.add_argument("-y", "--yes", action="store_true", default=False, dest="confirm", help="confirm extremely dangerous destructive delete") parser.add_argument("-d", "--days", type=int, default=0, @@ -298,7 +322,7 @@ def main(): parser.add_argument("-v", "--verbose", action="store_const", const=1, default=0, help="be verbose") args = parser.parse_args() - if args.patterns and not args.confirm and not args.do_it: + if args.patterns and not args.confirm and not args.do_it and not args.query_only: logging.warning(dedent(f""" WARNING: \tYou're using custom patterns: {args.patterns}. @@ -309,7 +333,7 @@ def main(): \t""")) return 3 - if args.leave_most_recent_only and not args.confirm and not args.do_it: + if args.leave_most_recent_only and not args.confirm and not args.do_it and not args.query_only: logging.warning(dedent(""" WARNING: \tYou're trying to delete ALL versions of the package EXCEPT for the *most recent one*, i.e.