Add new multi-arch build layout mode

Add a new mode which allows cosa to manipulate multi-arch build layouts:

```
$ find builds
builds
builds/builds.json
builds/30.1
builds/30.1/x86_64
builds/30.1/x86_64/coreos-assembler-config.tar.gz
builds/30.1/x86_64/coreos-assembler-config-git.json
builds/30.1/x86_64/fedora-coreos-30.1-qemu.qcow2
...
```

A pipeline could e.g. dispatch builds for each architecture on different
nodes, then group them back into a single workdir and have it
manipulated by e.g. `buildupload` seamlessly.

This new layout also matches the bucket layout for FCOS (see
coreos/fedora-coreos-tracker#189).

The basic idea is to add a `schema-version` to `builds.json` and denote
the legacy behaviour as "pre-1.0.0", while `1.0.0` contains a different
schema: each element in the `builds` array is now an object, which has
an `id`, and a list of `archs` for which that build has been built:

```
$ cat builds/builds.json
{
    "schema-version": "1.0.0",
    "builds": [
        {
            "id": "30.1",
            "archs": [
                "x86_64"
            ]
        }
    ],
    "timestamp": "2019-06-28T20:50:54Z"
}
```

We retain backwards-compatibility by simply checking the schema version.
Right now, only new workdirs will have this layout. Pipelines which use
`buildprep` will fetch `builds.json` as is and key off of its contents
to determine the bucket layout as well. We can write new code in the
future to convert previously single-arch buckets into the new layout to
then enable multi-arch.

src/cmd-buildprep

@@ -38,6 +38,8 @@ def main():
         print("Remote has no builds!")
         return
 
+    # NB: We only buildprep for the arch we're on for now. Could probably make
+    # this a switch in the future.
 
     buildid = builds.get_latest()
     builddir = builds.get_build_dir(buildid)

src/cmd-buildupload

@@ -48,7 +48,12 @@ def cmd_upload_s3(args):
         if args.build == 'latest':
             args.build = builds.get_latest()
         print(f"Targeting build: {args.build}")
-        s3_upload_build(args, builds.get_build_dir(args.build), args.build)
+        if builds.is_legacy():
+            s3_upload_build(args, builds.get_build_dir(args.build), args.build)
+        else:
+            for arch in builds.get_build_archs(args.build):
+                s3_upload_build(args, builds.get_build_dir(args.build, arch),
+                                f'{args.build}/{arch}')
     s3_cp(args, 'builds/builds.json', 'builds.json',
           '--cache-control=max-age=60')
 

src/cmd-compress

@@ -90,7 +90,13 @@ def compress_one_builddir(builddir):
     return at_least_one
 
 
-changed = compress_one_builddir(builds.get_build_dir(build))
+changed = []
+if builds.is_legacy():
+    changed.append(compress_one_builddir(builds.get_build_dir(build)))
+else:
+    for arch in builds.get_build_archs(build):
+        builddir = builds.get_build_dir(build, arch)
+        changed.append(compress_one_builddir(builddir))
 
-if not changed:
+if not any(changed):
     print(f"All builds already compressed")

src/cmdlib.py

@@ -10,6 +10,7 @@
 import sys
 import tempfile
 import gi
+import semver
 
 gi.require_version("RpmOstree", "1.0")
 from gi.repository import RpmOstree
@@ -181,39 +182,86 @@ def __init__(self, workdir=None):
         elif not os.path.isdir(self._path("builds")):
             raise Exception("No builds/ dir found!")
         else:
-            # must be a new workdir
+            # must be a new workdir; use new schema
             self._data = {
+                'schema-version': "1.0.0",
                 'builds': []
             }
             self.flush()
+        self._version = semver.parse_version_info(
+            self._data.get('schema-version', "0.0.1"))
+        # we understand < 2.0.0 only
+        if self._version._major >= 2:
+            raise Exception("Builds schema too new; please update cosa")
+        # for now, since we essentially just support "1.0.0" and "0.0.1",
+        # just dillute to a bool
+        self._legacy = (self._version._major < 1)
 
     def _path(self, path):
         if not self._workdir:
             return path
         return os.path.join(self._workdir, path)
 
     def has(self, build_id):
-        return build_id in self._data['builds']
+        if self._legacy:
+            return build_id in self._data['builds']
+        return any([b['id'] == build_id for b in self._data['builds']])
 
     def is_empty(self):
         return len(self._data['builds']) == 0
 
     def get_latest(self):
         # just let throw if there are none
-        return self._data['builds'][0]
-
-    def get_build_dir(self, build_id):
+        if self._legacy:
+            return self._data['builds'][0]
+        return self._data['builds'][0]['id']
+
+    def get_build_archs(self, build_id):
+        assert not self._legacy
+        for build in self._data['builds']:
+            if build['id'] == build_id:
+                return build['archs']
+        assert False, "Build not found!"
+
+    def get_build_dir(self, build_id, basearch=None):
         if build_id == 'latest':
             build_id = self.get_latest()
-        return self._path(f"builds/{build_id}")
-
-    def insert_build(self, build_id):
-        self._data['builds'].insert(0, build_id)
+        if self._legacy:
+            return self._path(f"builds/{build_id}")
+        if not basearch:
+            # just assume caller wants build dir for current arch
+            basearch = get_basearch()
+        return self._path(f"builds/{build_id}/{basearch}")
+
+    def insert_build(self, build_id, basearch=None):
+        if self._legacy:
+            self._data['builds'].insert(0, build_id)
+        else:
+            if not basearch:
+                basearch = get_basearch()
+            # for future tooling: allow inserting in an existing build for a
+            # separate arch
+            for build in self._data['builds']:
+                if build['id'] == build_id:
+                    if basearch in build['archs']:
+                        raise "Build {build_id} for {basearch} already exists"
+                    build['archs'] += [basearch]
+                    break
+            else:
+                self._data['builds'].insert(0, {
+                    'id': build_id,
+                    'archs': [
+                        basearch
+                    ]
+                })
 
     def bump_timestamp(self):
         self._data['timestamp'] = rfc3339_time()
         self.flush()
 
+    def is_legacy(self):
+        return self._legacy
+
     def raw(self):
         return self._data
 

src/deps.txt

@@ -43,6 +43,9 @@ jq
 # For interacting with AWS/HTTP
 awscli python3-boto3 python3-requests
 
+# For metadata versioning
+python3-semver
+
 # For ignition file validation in cmd-run
 /usr/bin/ignition-validate
 

src/prune_builds

@@ -16,7 +16,7 @@ import collections
 from datetime import timedelta, datetime, timezone
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from cmdlib import Builds
+from cmdlib import get_basearch, Builds
 
 
 def parse_date_string(date_string):
@@ -34,7 +34,7 @@ def parse_date_string(date_string):
     return dt.replace(tzinfo=timezone.utc)
 
 
-Build = collections.namedtuple('Build', ['id', 'timestamp'])
+Build = collections.namedtuple('Build', ['id', 'timestamp', 'basearchs'])
 
 # Let's just hardcode this here for now
 DEFAULT_KEEP_LAST_N = 3
@@ -109,9 +109,34 @@ with os.scandir(builds_dir) as it:
                 print(f"Ignoring non-directory {entry.path}")
             continue
 
-        ts = get_timestamp(entry)
-        if ts:
-            scanned_builds.append(Build(id=entry.name, timestamp=ts))
+        if builds.is_legacy():
+            ts = get_timestamp(entry)
+            if ts:
+                scanned_builds.append(Build(id=entry.name, timestamp=ts,
+                                            basearchs=[get_basearch()]))
+            continue
+
+        # scan all per-arch builds, pick up the most recent build of those as
+        # the overall "build" timestamp for pruning purposes
+        with os.scandir(entry.path) as basearch_it:
+            multiarch_build = None
+            for basearch_entry in basearch_it:
+                # ignore non-dirs
+                if not basearch_entry.is_dir(follow_symlinks=False):
+                    print(f"Ignoring non-directory {basearch_entry.path}")
+                    continue
+                ts = get_timestamp(basearch_entry)
+                if not ts:
+                    continue
+                if not multiarch_build:
+                    multiarch_build = Build(id=entry.name, timestamp=ts,
+                                            basearchs=[basearch_entry.name])
+                else:
+                    multiarch_build.basearchs += [basearch_entry.name]
+                    multiarch_build.timestamp = max(
+                        multiarch_build.timestamp, ts)
+            if multiarch_build:
+                scanned_builds.append(multiarch_build)
 
 
 # just get the trivial case out of the way
@@ -159,7 +184,8 @@ else:
 
 builds.raw()['builds'] = []
 for build in reversed(new_builds):
-    builds.insert_build(build.id)
+    for basearch in build.basearchs:
+        builds.insert_build(build.id, basearch)
 builds.bump_timestamp()
 
 # if we're not pruning, then we're done!