Multi-arch build layout prep patches

src/cmd-build

@@ -145,7 +145,8 @@ commitmeta_input_json=${PWD}/tmp/commit-metadata-input.json
 cat > "${commitmeta_input_json}" <<EOF
 {
   "coreos-assembler.config-gitrev": "${config_gitrev}",
-  "coreos-assembler.config-dirty": "${config_dirty}"
+  "coreos-assembler.config-dirty": "${config_dirty}",
+  "coreos-assembler.basearch": "${basearch}"
 }
 EOF
 

src/cmd-buildextend-metal

@@ -128,7 +128,7 @@ size="$(jq '."estimate-mb".final' "$PWD/tmp/ostree-size.json")"
 # extra size is the non-ostree partitions, see create_disk.sh
 size="$(( size + 513 ))M"
 echo "Disk size estimated to $size"
-kargs="$(python3 -c 'import sys, yaml; args = yaml.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "$configdir/image.yaml")"
+kargs="$(python3 -c 'import sys, yaml; args = yaml.safe_load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "$configdir/image.yaml")"
 kargs="$kargs console=tty0 console=${VM_TERMINAL},115200n8 ignition.platform.id=metal"
 
 qemu-img create -f qcow2 "${path}.qcow2" "$size"

src/cmd-buildprep

@@ -79,6 +79,12 @@ class Fetcher(object):
         self.fetch_impl(url, dest)
         return dest
 
+    def fetch_impl(self, url, dest):
+        raise NotImplementedError
+
+    def exists_impl(self, url):
+        raise NotImplementedError
+
     def fetch_json(self, path):
         return load_json(self.fetch(path))
 

src/cmd-buildupload

@@ -22,6 +22,8 @@ def main():
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--build", help="Build ID", default='latest')
+    parser.add_argument("--dry-run", help="Just print and exit",
+                        action='store_true')
     parser.add_argument("--freshen", help="Only push builds.json",
                         action='store_true')
 
@@ -96,9 +98,11 @@ def s3_upload_build(args):
 def s3_cp(args, src, dest, *s3_args):
     acl = f'--acl={args.acl}'
     dest = f's3://{args.url}/{dest}'
-    print(f"Uploading: {dest}")
-    subprocess.check_call(['aws', 's3', 'cp', acl, src, dest, *s3_args],
-                          stdout=subprocess.DEVNULL)
+    s3_args = ['aws', 's3', 'cp', acl, src, dest, *s3_args]
+    print("%s: %s" % ("Would run" if args.dry_run else "Running",
+                      subprocess.list2cmdline(s3_args)))
+    if not args.dry_run:
+        subprocess.check_call(s3_args, stdout=subprocess.DEVNULL)
 
 
 if __name__ == '__main__':

src/cmd-clean

@@ -51,6 +51,5 @@ prepare_build
 
 # But go back to the toplevel
 cd "${workdir:?}"
-# Note we don't prune the cache.qcow2 or the objects
-# in the repo.  If you want that, just rm -rf them.
+# Note we don't prune the cache.  If you want that, just rm -rf them.
 rm -rf builds/* tmp/*

src/cmd-koji-upload

@@ -128,8 +128,8 @@ class Build(_Build):
         """
         # locate all the build artifacts in the build directory.
         files = []
-        for ffile in os.listdir(self.build_root):
-            files.append(os.path.join(self.build_root, ffile))
+        for ffile in os.listdir(self.build_dir):
+            files.append(os.path.join(self.build_dir, ffile))
             if os.path.islink(ffile):
                 log.debug(" * EXCLUDING symlink '%s'", ffile)
                 log.debug(" *    target '%s'", os.path.realpath(ffile))

src/cmdlib.py

@@ -9,6 +9,10 @@
 import subprocess
 import sys
 import tempfile
+import gi
+
+gi.require_version("RpmOstree", "1.0")
+from gi.repository import RpmOstree
 
 from datetime import datetime
 
@@ -158,3 +162,11 @@ def import_ostree_commit(repo, commit, tarfile):
         subprocess.check_call(['tar', '-C', d, '-xf', tarfile])
         subprocess.check_call(['ostree', 'pull-local', '--repo', repo,
                                d, commit])
+
+
+def get_basearch():
+    try:
+        return get_basearch.saved
+    except AttributeError:
+        get_basearch.saved = RpmOstree.get_basearch()
+        return get_basearch.saved

src/cmdlib.sh

@@ -30,6 +30,14 @@ fatal() {
     echo "fatal: $*" 1>&2; exit 1
 }
 
+# Get target base architecture
+basearch=$(python3 -c '
+import gi
+gi.require_version("RpmOstree", "1.0")
+from gi.repository import RpmOstree
+print(RpmOstree.get_basearch())')
+export basearch
+
 # Get target architecture
 arch=$(uname -m)
 export arch
@@ -274,7 +282,7 @@ ostree-layers:
 EOF
     fi
     if [[ -n $(ls "${overridesdir}/rpm/"*.rpm 2> /dev/null) ]]; then
-        (cd "${overridesdir}"/rpm && createrepo_c .)
+        (cd "${overridesdir}"/rpm && rm -rf .repodata && createrepo_c .)
         echo "Using RPM overrides from: ${overridesdir}/rpm"
         cat >> "${override_manifest}" <<EOF
 repos:

src/cosalib/build.py

@@ -55,6 +55,7 @@ def write_json(path, data):
     os.fchmod(f.file.fileno(), 0o644)
     os.rename(f.name, path)
 
+
 class _Build:
     """
     The Build Class handles the reading in and return of build JSON emitted
@@ -64,13 +65,13 @@ class _Build:
       - _build_artifacts(*args, **kwargs)
     """
 
-    def __init__(self, build_dir, build="latest", workdir=None):
+    def __init__(self, builds_dir, build="latest", workdir=None):
         """
         init loads the builds.json which lists the builds, loads the relevant
         meta-data from JSON and finally, locates the build artifacts.
 
-        :param build_dir: name of directory to find the builds
-        :type build_dir: str
+        :param builds_dir: name of directory to find the builds
+        :type builds_dir: str
         :param build: build id or "latest" to parse
         :type build: str
         :param workdir: Temporary directory to ensure exists and is clean
@@ -83,15 +84,15 @@ def __init__(self, build_dir, build="latest", workdir=None):
         If workdir is None then no temporary work directory is created.
         """
         log.info('Evaluating builds.json')
-        builds = load_json('%s/builds.json' % build_dir)
+        builds = load_json('%s/builds.json' % builds_dir)
         if build != "latest":
             if build not in builds['builds']:
                 raise BuildError("Build was not found in builds.json")
         else:
             build = builds['builds'][0]
 
         log.info("Targeting build: %s", build)
-        self._build_root = os.path.abspath("%s/%s" % (build_dir, build))
+        self._build_dir = os.path.abspath("%s/%s" % (builds_dir, build))
 
         self._build_json = {
             "commit": None,
@@ -155,9 +156,9 @@ def build_id(self):
         return self.get_meta_key("meta", "buildid")
 
     @property
-    def build_root(self):
+    def build_dir(self):
         """ return the actual path for the build root """
-        return self._build_root
+        return self._build_dir
 
     @property
     def build_name(self):
@@ -220,11 +221,10 @@ def __file(self, var):
         :raises: KeyError
         """
         lookup = {
-            "commit": "%s/commitmeta.json" % self.build_root,
-            "config": ("%s/coreos-assembler-config-git.json" %
-                       self.build_root),
+            "commit": "%s/commitmeta.json" % self.build_dir,
+            "config": ("%s/coreos-assembler-config-git.json" % self.build_dir),
             "image": "/cosa/coreos-assembler-git.json",
-            "meta": "%s/meta.json" % self.build_root,
+            "meta": "%s/meta.json" % self.build_dir,
         }
         return lookup[var]
 

src/prune_builds

@@ -11,7 +11,6 @@ import sys
 import json
 import shutil
 import argparse
-import subprocess
 import collections
 
 from datetime import timedelta, datetime, timezone
@@ -35,8 +34,7 @@ def parse_date_string(date_string):
     return dt.replace(tzinfo=timezone.utc)
 
 
-Build = collections.namedtuple('Build', ['id', 'timestamp',
-                                         'ostree_timestamp'])
+Build = collections.namedtuple('Build', ['id', 'timestamp'])
 
 # Let's just hardcode this here for now
 DEFAULT_KEEP_LAST_N = 3
@@ -115,10 +113,7 @@ with os.scandir(builds_dir) as it:
         # Older versions only had ostree-timestamp
         ts = j.get('coreos-assembler.build-timestamp') or j['ostree-timestamp']
         t = parse_date_string(ts)
-        ostree_ts = j['ostree-timestamp']
-        ostree_t = parse_date_string(ostree_ts)
-        builds.append(Build(id=entry.name, timestamp=t,
-                            ostree_timestamp=ostree_t))
+        builds.append(Build(id=entry.name, timestamp=t))
 
 # just get the trivial case out of the way
 if len(builds) == 0: