diff --git a/src/cmd-buildprep b/src/cmd-buildprep index 8606576638..ef28359085 100755 --- a/src/cmd-buildprep +++ b/src/cmd-buildprep @@ -38,6 +38,8 @@ def main(): print("Remote has no builds!") return + # NB: We only buildprep for the arch we're on for now. Could probably make + # this a switch in the future. buildid = builds.get_latest() builddir = builds.get_build_dir(buildid) diff --git a/src/cmd-buildupload b/src/cmd-buildupload index f171426373..ddc78d8eed 100755 --- a/src/cmd-buildupload +++ b/src/cmd-buildupload @@ -48,7 +48,12 @@ def cmd_upload_s3(args): if args.build == 'latest': args.build = builds.get_latest() print(f"Targeting build: {args.build}") - s3_upload_build(args, builds.get_build_dir(args.build), args.build) + if builds.is_legacy(): + s3_upload_build(args, builds.get_build_dir(args.build), args.build) + else: + for arch in builds.get_build_archs(args.build): + s3_upload_build(args, builds.get_build_dir(args.build, arch), + f'{args.build}/{arch}') s3_cp(args, 'builds/builds.json', 'builds.json', '--cache-control=max-age=60') diff --git a/src/cmd-compress b/src/cmd-compress index 0f8f033303..94de0c2ca0 100755 --- a/src/cmd-compress +++ b/src/cmd-compress @@ -103,7 +103,13 @@ def compress_one_builddir(builddir): return at_least_one -changed = compress_one_builddir(builds.get_build_dir(build)) +changed = [] +if builds.is_legacy(): + changed.append(compress_one_builddir(builds.get_build_dir(build))) +else: + for arch in builds.get_build_archs(build): + builddir = builds.get_build_dir(build, arch) + changed.append(compress_one_builddir(builddir)) -if not changed: +if not any(changed): print(f"All builds already compressed") diff --git a/src/cmdlib.py b/src/cmdlib.py index 1fb131e14c..234ee16093 100755 --- a/src/cmdlib.py +++ b/src/cmdlib.py @@ -10,6 +10,7 @@ import sys import tempfile import gi +import semver gi.require_version("RpmOstree", "1.0") from gi.repository import RpmOstree @@ -181,11 +182,20 @@ def __init__(self, workdir=None): elif os.path.isfile(self._fn): self._data = load_json(self._fn) else: - # must be a new workdir + # must be a new workdir; use new schema self._data = { + 'schema-version': "1.0.0", 'builds': [] } self.flush() + self._version = semver.parse_version_info( + self._data.get('schema-version', "0.0.1")) + # we understand < 2.0.0 only + if self._version._major >= 2: + raise Exception("Builds schema too new; please update cosa") + # for now, since we essentially just support "1.0.0" and "0.0.1", + # just dillute to a bool + self._legacy = (self._version._major < 1) def _path(self, path): if not self._workdir: @@ -193,27 +203,65 @@ def _path(self, path): return os.path.join(self._workdir, path) def has(self, build_id): - return build_id in self._data['builds'] + if self._legacy: + return build_id in self._data['builds'] + return any([b['id'] == build_id for b in self._data['builds']]) def is_empty(self): return len(self._data['builds']) == 0 def get_latest(self): # just let throw if there are none - return self._data['builds'][0] - - def get_build_dir(self, build_id): + if self._legacy: + return self._data['builds'][0] + return self._data['builds'][0]['id'] + + def get_build_archs(self, build_id): + assert not self._legacy + for build in self._data['builds']: + if build['id'] == build_id: + return build['archs'] + assert False, "Build not found!" + + def get_build_dir(self, build_id, basearch=None): if build_id == 'latest': build_id = self.get_latest() - return self._path(f"builds/{build_id}") - - def insert_build(self, build_id): - self._data['builds'].insert(0, build_id) + if self._legacy: + return self._path(f"builds/{build_id}") + if not basearch: + # just assume caller wants build dir for current arch + basearch = get_basearch() + return self._path(f"builds/{build_id}/{basearch}") + + def insert_build(self, build_id, basearch=None): + if self._legacy: + self._data['builds'].insert(0, build_id) + else: + if not basearch: + basearch = get_basearch() + # for future tooling: allow inserting in an existing build for a + # separate arch + for build in self._data['builds']: + if build['id'] == build_id: + if basearch in build['archs']: + raise "Build {build_id} for {basearch} already exists" + build['archs'] += [basearch] + break + else: + self._data['builds'].insert(0, { + 'id': build_id, + 'archs': [ + basearch + ] + }) def bump_timestamp(self): self._data['timestamp'] = rfc3339_time() self.flush() + def is_legacy(self): + return self._legacy + def raw(self): return self._data diff --git a/src/deps.txt b/src/deps.txt index f503c9a346..5f13551bf2 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -43,6 +43,9 @@ jq # For interacting with AWS/HTTP awscli python3-boto3 python3-requests +# For metadata versioning +python3-semver + # For ignition file validation in cmd-run /usr/bin/ignition-validate diff --git a/src/prune_builds b/src/prune_builds index c77ad8ccac..907f6cca7e 100755 --- a/src/prune_builds +++ b/src/prune_builds @@ -16,7 +16,7 @@ import collections from datetime import timedelta, datetime, timezone sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from cmdlib import Builds +from cmdlib import get_basearch, Builds def parse_date_string(date_string): @@ -34,7 +34,7 @@ def parse_date_string(date_string): return dt.replace(tzinfo=timezone.utc) -Build = collections.namedtuple('Build', ['id', 'timestamp']) +Build = collections.namedtuple('Build', ['id', 'timestamp', 'basearchs']) # Let's just hardcode this here for now DEFAULT_KEEP_LAST_N = 3 @@ -109,9 +109,34 @@ with os.scandir(builds_dir) as it: print(f"Ignoring non-directory {entry.path}") continue - ts = get_timestamp(entry) - if ts: - scanned_builds.append(Build(id=entry.name, timestamp=ts)) + if builds.is_legacy(): + ts = get_timestamp(entry) + if ts: + scanned_builds.append(Build(id=entry.name, timestamp=ts, + basearchs=[get_basearch()])) + continue + + # scan all per-arch builds, pick up the most recent build of those as + # the overall "build" timestamp for pruning purposes + with os.scandir(entry.path) as basearch_it: + multiarch_build = None + for basearch_entry in basearch_it: + # ignore non-dirs + if not basearch_entry.is_dir(follow_symlinks=False): + print(f"Ignoring non-directory {basearch_entry.path}") + continue + ts = get_timestamp(basearch_entry) + if not ts: + continue + if not multiarch_build: + multiarch_build = Build(id=entry.name, timestamp=ts, + basearchs=[basearch_entry.name]) + else: + multiarch_build.basearchs += [basearch_entry.name] + multiarch_build.timestamp = max( + multiarch_build.timestamp, ts) + if multiarch_build: + scanned_builds.append(multiarch_build) # just get the trivial case out of the way @@ -159,7 +184,8 @@ else: builds.raw()['builds'] = [] for build in reversed(new_builds): - builds.insert_build(build.id) + for basearch in build.basearchs: + builds.insert_build(build.id, basearch) builds.bump_timestamp() # if we're not pruning, then we're done!