Use new builds/ dir helpers

Prep for switching those over to the new build layout.

Most commands have pretty straightforward changes. Some of the more
complex ones that deal with the build layout a lot were refactored a bit
as prep for multi-arch.

src/cmd-build

@@ -80,10 +80,9 @@ prepare_build
 ostree --version
 rpm-ostree --version
 
-previous_build=
-if [ -L "${workdir}"/builds/latest ]; then
-    previous_build=$(readlink "${workdir}"/builds/latest)
-    previous_builddir="${workdir}/builds/${previous_build}"
+previous_build=$(get_latest_build)
+if [ -n "${previous_build}" ]; then
+    previous_builddir=$(get_build_dir "${previous_build}")
     echo "Previous build: ${previous_build}"
 fi
 
@@ -363,18 +362,20 @@ saved_build_tmpdir="${workdir}/tmp/last-build-tmp"
 rm -rf "${saved_build_tmpdir}"
 mv -T tmp "${saved_build_tmpdir}"
 ostree prune --repo="${tmprepo}" --refs-only
-# Back to the toplevel build directory, so we can rename this one
-cd "${workdir}"/builds
+# Back to the toplevel work directory, so we can rename this one
+cd "${workdir}"
 # We create a .build-commit file to note that we're in the
 # middle of a "commit".  This may be useful in the future
 # for having things be transactional.  If for example we
 # were interrupted between the rename() and linkat() below,
 # things would be inconsistent and future builds would fail
 # on the `mv`.
-touch .build-commit
-mv -T "${tmp_builddir}" "${buildid}"
+touch builds/.build-commit
+builddir=$(get_build_dir "${buildid}")
+mkdir -p "${builddir}"
+mv -T "${tmp_builddir}" "${builddir}"
 # Replace the latest link
-ln -Tsfr "${buildid}" latest
+ln -Tsf "${buildid}" builds/latest
 # Update builds.json
 # the variables passed to `prune_builds` end up single quoted and
 # python treats them as literals, so we workaround this by duplicating
@@ -384,4 +385,4 @@ if [ "${SKIP_PRUNE}" == 1 ]; then
 else
     "${dn}"/prune_builds --workdir "${workdir}"
 fi
-rm .build-commit
+rm builds/.build-commit

src/cmd-buildextend-aws

@@ -8,7 +8,7 @@ import os,sys,json,yaml,shutil,argparse,subprocess,re,collections
 import tempfile,hashlib,gzip
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from cmdlib import run_verbose, write_json, sha256sum_file
+from cmdlib import run_verbose, write_json, sha256sum_file, Builds
 
 # Parse args and dispatch
 parser = argparse.ArgumentParser()
@@ -23,8 +23,8 @@ parser.add_argument("--grant-user", help="Grant user launch permission",
                     nargs="*", default=[])
 args = parser.parse_args()
 
-
-builddir = os.path.join('builds', args.build)
+builds = Builds()
+builddir = builds.get_build_dir(args.build)
 buildmeta_path = os.path.join(builddir, 'meta.json')
 with open(buildmeta_path) as f:
     buildmeta = json.load(f)

src/cmd-buildextend-gcp

@@ -18,7 +18,8 @@ sys.path.insert(0, cosa_dir)
 from cmdlib import (
         run_verbose,
         sha256sum_file,
-        write_json)
+        write_json,
+        Builds)
 
 # Parse args and dispatch
 parser = argparse.ArgumentParser()
@@ -46,7 +47,8 @@ if args.project is None:
     raise Exception(arg_exp_str.format("project", "GCP_PROJECT"))
 
 # Identify the builds
-builddir = os.path.join('builds', args.build)
+builds = Builds()
+builddir = builds.get_build_dir(args.build)
 buildmeta_path = os.path.join(builddir, 'meta.json')
 with open(buildmeta_path) as f:
     buildmeta = json.load(f)

src/cmd-buildextend-installer

@@ -14,7 +14,7 @@ import tempfile
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from cmdlib import run_verbose, write_json, sha256sum_file
-from cmdlib import import_ostree_commit
+from cmdlib import import_ostree_commit, Builds
 
 # Parse args and dispatch
 parser = argparse.ArgumentParser()
@@ -23,16 +23,16 @@ parser.add_argument("--force", action='store_true', default=False,
                     help="Overwrite previously generated installer")
 args = parser.parse_args()
 
+builds = Builds()
+
 # default to latest build if not specified
 if not args.build:
-    with open('builds/builds.json') as f:
-        j = json.load(f)
-        args.build = j['builds'][0]
+    args.build = builds.get_latest()
 
 print(f"Targeting build: {args.build}")
 
 workdir = os.path.abspath(os.getcwd())
-builddir = os.path.join(workdir, 'builds', args.build)
+builddir = builds.get_build_dir(args.build)
 buildmeta_path = os.path.join(builddir, 'meta.json')
 with open(buildmeta_path) as f:
     buildmeta = json.load(f)

src/cmd-buildextend-metal

@@ -68,14 +68,13 @@ export LIBGUESTFS_BACKEND=direct
 prepare_build
 
 if [ -z "${build}" ]; then
-    if [ -L "${workdir}"/builds/latest ]; then
-        build=$(readlink "${workdir}"/builds/latest)
-    else
+    build=$(get_latest_build)
+    if [ -z "${build}" ]; then
         fatal "No build found."
     fi
 fi
 
-builddir="${workdir}/builds/${build}"
+builddir=$(get_build_dir "$build")
 if [ ! -d "${builddir}" ]; then
     fatal "Build dir ${builddir} does not exist."
 fi

src/cmd-buildextend-openstack

@@ -11,22 +11,22 @@ import shutil
 import argparse
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from cmdlib import run_verbose, write_json, sha256sum_file
+from cmdlib import run_verbose, write_json, sha256sum_file, Builds
 
 # Parse args and dispatch
 parser = argparse.ArgumentParser()
 parser.add_argument("--build", help="Build ID")
 args = parser.parse_args()
 
+builds = Builds()
+
 # default to latest build if not specified
 if not args.build:
-    with open('builds/builds.json') as f:
-        j = json.load(f)
-        args.build = j['builds'][0]
+    args.build = builds.get_latest()
 
 print(f"Targeting build: {args.build}")
 
-builddir = os.path.join('builds', args.build)
+builddir = builds.get_build_dir(args.build)
 buildmeta_path = os.path.join(builddir, 'meta.json')
 with open(buildmeta_path) as f:
     buildmeta = json.load(f)

src/cmd-buildextend-vmware

@@ -11,19 +11,20 @@ import argparse
 import tarfile
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from cmdlib import run_verbose, load_json, write_json, sha256sum_file
+from cmdlib import run_verbose, load_json, write_json, sha256sum_file, Builds
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--build", help="Build ID",
                     required=False)
 args = parser.parse_args()
 
+builds = Builds()
+
 # default to latest build if not specified
 if not args.build:
-    j = load_json('builds/builds.json')
-    args.build = j['builds'][0]
+    args.build = builds.get_latest()
 
-builddir = os.path.join('builds', args.build)
+builddir = builds.get_build_dir(args.build)
 buildmeta_path = os.path.join(builddir, 'meta.json')
 buildmeta = load_json(buildmeta_path)
 base_name = buildmeta['name']

src/cmd-buildprep

@@ -14,7 +14,7 @@ import boto3
 import shutil
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from cmdlib import load_json, rm_allow_noent  # noqa: E402
+from cmdlib import load_json, rm_allow_noent, Builds  # noqa: E402
 
 
 def main():
@@ -29,26 +29,34 @@ def main():
     else:
         raise Exception("Invalid scheme: only file://, s3://, and http(s):// supported")
 
-    builds = []
+    builds = None
     if fetcher.exists('builds.json'):
-        builds = fetcher.fetch_json('builds.json')['builds']
+        fetcher.fetch_json('builds.json')
+        builds = Builds()
 
-    if len(builds) == 0:
+    if not builds or builds.is_empty():
         print("Remote has no builds!")
         return
 
-    buildid = builds[0]
-    os.makedirs(f'builds/{buildid}', exist_ok=True)
+
+    buildid = builds.get_latest()
+    builddir = builds.get_build_dir(buildid)
+    os.makedirs(builddir, exist_ok=True)
+
+    # trim out the leading builds/
+    assert builddir.startswith("builds/")
+    builddir = builddir[len("builds/"):]
+
     for f in ['meta.json', 'ostree-commit-object']:
-        fetcher.fetch(f'{buildid}/{f}')
+        fetcher.fetch(f'{builddir}/{f}')
 
     # and finally the symlink
     rm_allow_noent('builds/latest')
     os.symlink(buildid, 'builds/latest')
 
     # also nuke the any local matching OStree ref, since we want to build on
     # top of this new one
-    buildmeta = load_json('builds/latest/meta.json')
+    buildmeta = load_json(f'builds/{builddir}/meta.json')
     if 'ref' in buildmeta and os.path.isdir('tmp/repo'):
         subprocess.check_call(['ostree', 'refs', '--repo', 'tmp/repo',
                                '--delete', buildmeta['ref']],

src/cmd-buildupload

@@ -11,7 +11,7 @@ import sys
 import tempfile
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from cmdlib import load_json  # noqa: E402
+from cmdlib import load_json, Builds  # noqa: E402
 
 
 def main():
@@ -44,18 +44,17 @@ def parse_args():
 
 def cmd_upload_s3(args):
     if not args.freshen:
-        s3_upload_build(args)
+        builds = Builds()
+        if args.build == 'latest':
+            args.build = builds.get_latest()
+        print(f"Targeting build: {args.build}")
+        s3_upload_build(args, builds.get_build_dir(args.build), args.build)
     s3_cp(args, 'builds/builds.json', 'builds.json',
           '--cache-control=max-age=60')
 
 
-def s3_upload_build(args):
-    builddir = f'builds/{args.build}'
+def s3_upload_build(args, builddir, dest):
     build = load_json(f'{builddir}/meta.json')
-    buildid = build['buildid']
-
-    print(f"Targeting build: {buildid}")
-    print(f"  OSTree commit: {build['ostree-commit']}")
 
     # Upload images with special handling for gzipped data.
     uploaded = set()
@@ -71,27 +70,27 @@ def s3_upload_build(args):
                 args.enable_gz_peel):
             nogz = bn[:-3]
             img['path'] = nogz
-            s3_cp(args, path, f'{buildid}/{nogz}',
+            s3_cp(args, path, f'{dest}/{nogz}',
                   '--content-encoding=gzip',
                   f'--content-disposition=inline; filename={nogz}')
         else:
-            s3_cp(args, path, f'{buildid}/{bn}')
+            s3_cp(args, path, f'{dest}/{bn}')
         uploaded.add(bn)
 
     for f in os.listdir(builddir):
         # we do meta.json right after
         if f in uploaded or f == 'meta.json':
             continue
         path = os.path.join(builddir, f)
-        s3_cp(args, path, f'{buildid}/{f}')
+        s3_cp(args, path, f'{dest}/{f}')
 
     # Now upload a modified version of the meta.json which has the fixed
     # filenames without the .gz suffixes. We don't want to modify the local
     # build dir.
     with tempfile.NamedTemporaryFile('w') as f:
         json.dump(build, f, indent=4)
         f.flush()
-        s3_cp(args, f.name, f'{buildid}/meta.json',
+        s3_cp(args, f.name, f'{dest}/meta.json',
               '--content-type=application/json')