Skip to content

Commit

Permalink
[WIP] Migrate cleanup workflow to GitHub Action
Browse files Browse the repository at this point in the history
The `clean-old-packages` script was adapted from the
`securedrop-builder` repository and originally written by
Kunal Mehta.
  • Loading branch information
eloquence committed Apr 9, 2024
1 parent 0446421 commit 4f7d413
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 0 deletions.
29 changes: 29 additions & 0 deletions .github/workflows/cleanup.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: Cleanup
on: pull_request

defaults:
run:
shell: bash

jobs:
clean-old-packages:
runs-on: ubuntu-latest
container: fedora:39
steps:
- name: Install dependencies
run: |
dnf install -y rpmdevtools git git-lfs
- uses: actions/checkout@v4
with:
lfs: true
token: ${{ secrets.PUSH_TOKEN }}
- name: Clean old packages
run: |
git config --global --add safe.directory '*'
git config user.email "[email protected]"
git config user.name "sdcibot"
find workstation -mindepth 1 -maxdepth 2 -type d | xargs -I '{}' ./scripts/clean-old-packages '{}' 4
git add .
# Index will be clean if there are no changes
git diff-index --quiet HEAD || git commit -m "Removing old packages"
# git push origin main
113 changes: 113 additions & 0 deletions scripts/clean-old-packages
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""
Clean up old packages, specifying how many to keep
Example:
./clean-old-packages securedrop-yum-test/workstation/buster-nightlies 7
This script is run in CI in a Fedora container. You can spin up a similar
container locally using podman or docker, e.g.:
podman run -it --rm -v $(pwd):/workspace:Z fedora:39 bash
The `rpm` module is provided by `python3-rpm`, and `rpmdev-vercmp`
is provided by `rpmdevtools`.
"""
import argparse
import functools
import subprocess
from collections import defaultdict
from pathlib import Path
from typing import Tuple

import rpm


def sort_rpm_versions(one: Tuple[str, Path], two: Tuple[str, Path]):
"""sort two RPM package versions"""
status = subprocess.run(['rpmdev-vercmp', one[0], two[0]], stdout=subprocess.DEVNULL)
if status.returncode == 11:
# false, one is bigger
return 1
else: # status.returncode == 12
# true, two is bigger
return -1


def fix_name(name: str) -> str:
"""
Linux packages embed the version in the name, so we'd never have multiple
packages meet the deletion threshold. Silly string manipulation to drop
the version.
E.g. "linux-image-5.15.26-grsec-securedrop" -> "linux-image-securedrop"
"""
if name.endswith(('-securedrop', '-workstation')):
suffix = name.split('-')[-1]
else:
return name
if name.startswith('linux-image-'):
return f'linux-image-{suffix}'
elif name.startswith('linux-headers-'):
return f'linux-headers-{suffix}'
return name


def rpm_info(path: Path) -> Tuple[str, str]:
"""
learned this incantation from <https://web.archive.org/web/20120911204323/http://docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s05.html>
and help(headers)
"""
ts = rpm.ts()
with path.open() as f:
headers = ts.hdrFromFdno(f)
print(headers[rpm.RPMTAG_VERSION])

return headers[rpm.RPMTAG_NAME], headers[rpm.RPMTAG_VERSION] + '-' + headers[rpm.RPMTAG_RELEASE]


def cleanup(data, to_keep: int, sorter):
for name, versions in sorted(data.items()):
if len(versions) <= to_keep:
# Nothing to delete
continue
print(f'### {name}')
items = sorted(versions.items(), key=functools.cmp_to_key(sorter), reverse=True)
keeps = items[:to_keep]
print('Keeping:')
for _, keep in keeps:
print(f'* {keep.name}')
delete = items[to_keep:]
print('Deleting:')
for _, path in delete:
print(f'* {path.name}')
path.unlink()


def main():
parser = argparse.ArgumentParser(
description="Cleans up old packages"
)
parser.add_argument(
"directory",
type=Path,
help="Directory to clean up",
)
parser.add_argument(
"keep",
type=int,
help="Number of packages to keep"
)
args = parser.parse_args()
if not args.directory.is_dir():
raise RuntimeError(f"Directory, {args.directory}, doesn't exist")
print(f'Only keeping the latest {args.keep} packages')
rpms = defaultdict(dict)
for rpm in args.directory.glob('*.rpm'):
name, version = rpm_info(rpm)
rpms[name][version] = rpm

cleanup(rpms, args.keep, sort_rpm_versions)


if __name__ == '__main__':
main()

0 comments on commit 4f7d413

Please sign in to comment.