From 2b28b72e59dfd2bbfbb7af6925c4d6b7a193444c Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Sun, 11 Aug 2024 20:34:10 -0400 Subject: [PATCH 1/2] deps.txt: move some deps to vmdeps.txt We run osbuild inside of supermin, so for building the live ISO we'll need all the tools it needs in there. This includes `genisoimage` and `syslinux-nonlinux`. We don't currently name `squashfs-tools` as a toplevel dependency. It's currently pulled in by libguestfs, but we need it in supermin too, so explicitly list it there. --- src/deps-x86_64.txt | 3 --- src/deps.txt | 5 +---- src/vmdeps-x86_64.txt | 3 +++ src/vmdeps.txt | 3 +++ 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/deps-x86_64.txt b/src/deps-x86_64.txt index fcfcf71566..62c2063f37 100644 --- a/src/deps-x86_64.txt +++ b/src/deps-x86_64.txt @@ -1,6 +1,3 @@ -# For generating ISO images -syslinux-nonlinux - # For the pipeline and developers to interact with AWS. It's not needed by cosa # itself. This isn't available on s390x at least, so make it x86_64-only. awscli2 diff --git a/src/deps.txt b/src/deps.txt index 1f3d59b37b..e013ff6ec7 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -16,9 +16,6 @@ dumb-init rpm-ostree createrepo_c openssh-clients python3-createrepo_c composefs dnf-utils -# For generating ISO images -genisoimage - # Standard build tools make git rpm-build @@ -100,4 +97,4 @@ bsdtar fedora-repos-ostree # For graphing manifest includes using `manifest_graph` -python-anytree \ No newline at end of file +python-anytree diff --git a/src/vmdeps-x86_64.txt b/src/vmdeps-x86_64.txt index 5289937a8b..f81b67b450 100644 --- a/src/vmdeps-x86_64.txt +++ b/src/vmdeps-x86_64.txt @@ -2,6 +2,9 @@ grub2 bootupd +# For generating ISO images +syslinux-nonlinux + # For creating bootable UEFI media on x86_64 shim-x64 grub2-efi-x64 diff --git a/src/vmdeps.txt b/src/vmdeps.txt index eaa9e95fad..7809b35451 100644 --- a/src/vmdeps.txt +++ b/src/vmdeps.txt @@ -5,6 +5,9 @@ # bare essentials bash vim-minimal coreutils util-linux procps-ng kmod kernel-modules +# For generating ISO images +genisoimage squashfs-tools + # for composes rpm-ostree distribution-gpg-keys jq From d29a0656fc4c96d797f76079b2506e55df9d7fe4 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Sun, 11 Aug 2024 20:36:20 -0400 Subject: [PATCH 2/2] Build live ISO using osbuild This adds a new osbuild pipeline for building the live ISO using the new `org.osbuild.coreos.live-iso` mega stage. Still missing is changing `cmd-buildextend-live` to call osbuild instead. I think we'll need to ratchet this in using e.g. an env var like we did for the other osbuild artifacts. E.g. `COSA_OSBUILD_LIVE` would control whether to use the old logic or the new one. And then turn it on in rawhide, etc... The new `cmd-buildextend-live` would also take care of extracting the live artifacts from the live ISO to insert into `meta.json`. --- src/cmdlib.sh | 2 + .../coreos.osbuild.x86_64.mpp.yaml | 6 ++ .../platform.live-iso.ipp.yaml | 96 +++++++++++++++++++ src/runvm-osbuild | 12 ++- 4 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 src/osbuild-manifests/platform.live-iso.ipp.yaml diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 11f6983fea..2113c06497 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -730,6 +730,8 @@ runvm() { # include COSA in the image find /usr/lib/coreos-assembler/ -type f > "${vmpreparedir}/hostfiles" echo /usr/lib/osbuild/stages/org.osbuild.dmverity >> "${vmpreparedir}/hostfiles" + echo /usr/lib/osbuild/stages/org.osbuild.coreos.live-iso >> "${vmpreparedir}/hostfiles" + echo /usr/lib/osbuild/stages/org.osbuild.coreos.live-iso.meta.json >> "${vmpreparedir}/hostfiles" # and include all GPG keys find /etc/pki/rpm-gpg/ -type f >> "${vmpreparedir}/hostfiles" diff --git a/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml index bb4f633a19..f6a5c799fa 100644 --- a/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml @@ -11,6 +11,7 @@ mpp-vars: extra_kargs: $extra_kargs metal_image_size_mb: $metal_image_size_mb cloud_image_size_mb: $cloud_image_size_mb + squashfs_compression: $squashfs_compression bios_boot_size_mb: 1 ppc_prep_size_mb: 4 reserved_part_size_mb: 1 @@ -18,6 +19,9 @@ mpp-vars: boot_size_mb: 384 sector_size: 512 four_k_sector_size: 4096 + metal_filename: $metal_filename + metal4k_filename: $metal4k_filename + live_efiboot_img_size_mb: $live_efiboot_img_size_mb # Filesystem UUID and label definitions. These UUIDs # are looked for on boot and if found replaced with # a new random UUID to make each install unique. @@ -587,3 +591,5 @@ pipelines: path: platform.metal.ipp.yaml - mpp-import-pipelines: path: platform.qemu.ipp.yaml + - mpp-import-pipelines: + path: platform.live-iso.ipp.yaml diff --git a/src/osbuild-manifests/platform.live-iso.ipp.yaml b/src/osbuild-manifests/platform.live-iso.ipp.yaml new file mode 100644 index 0000000000..b08c232191 --- /dev/null +++ b/src/osbuild-manifests/platform.live-iso.ipp.yaml @@ -0,0 +1,96 @@ +# This file defines the pipeline for building the live ISO. +version: '2' +pipelines: + - name: live-iso + stages: + # Yuck: here we copy the metal and metal4k images into the tree from the + # outside instead of referencing the metal/metal4k osbuild pipelines. + # The reason for that is that our tooling and pipeline is currently + # very oriented towards building the metal images as separate artifacts + # in separate invocations of osbuild, so we need to support the metal + # images pre-existing. Also, the invocations in which we currently build + # those images are done with -snapshot on the cache qcow so it can be + # parallelized, so we can't just checkpoint those stage outputs. And even + # if we did, it doesn't feel quite right to rely on whatever is cached + # vs the canonical images. I think to improve this, we should move to + # building the metal, metal4k and live artifacts in one invocation. + # + # The mpp-if here is so osbuild-mpp doesn't try to embed the (possibly + # non-existent metal images) when building other stuff... We really should + # improve how we interact with osbuild. + - mpp-if: metal_filename != '' + then: + type: org.osbuild.copy + inputs: + metal: + type: org.osbuild.files + origin: org.osbuild.source + mpp-embed: + id: metal + url: + mpp-format-string: 'file://{metal_filename}' + options: + paths: + - from: + mpp-format-string: input://metal/{embedded['metal']} + to: tree:///metal.raw + - mpp-if: metal4k_filename != '' + then: + type: org.osbuild.copy + inputs: + metal4k: + type: org.osbuild.files + origin: org.osbuild.source + mpp-embed: + id: metal4k + url: + mpp-format-string: 'file://{metal4k_filename}' + options: + paths: + - from: + mpp-format-string: input://metal4k/{embedded['metal4k']} + to: tree:///metal4k.raw + # We need to be able to create efiboot.img, a FAT filesystem image. It's + # hard to setup loopback devices from within the stage, so just do it + # here. This should normally be conditional on the architecture but it + # doesn't hurt either since it's a tiny file. + - type: org.osbuild.truncate + options: + filename: efiboot.img + size: + mpp-format-string: '{live_efiboot_img_size_mb * 1024 * 1024}' + - type: org.osbuild.coreos.live-iso + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + # XXX: we should rename this pipeline now that we don't use it as + # a buildroot + - name:build + devices: + metal: + type: org.osbuild.loopback + options: + filename: metal.raw + partscan: true + read-only: true + sector-size: 512 + metal4k: + type: org.osbuild.loopback + options: + filename: metal4k.raw + partscan: true + read-only: true + sector-size: 4096 + efiboot_img: + type: org.osbuild.loopback + options: + filename: efiboot.img + options: + efiboot_img_filename: + mpp-format-string: 'efiboot.img' + filename: + mpp-format-string: '{filename}' + squashfs_compression: + mpp-format-string: '{squashfs_compression}' diff --git a/src/runvm-osbuild b/src/runvm-osbuild index ceb12ba755..7a7f7f02ec 100755 --- a/src/runvm-osbuild +++ b/src/runvm-osbuild @@ -3,8 +3,8 @@ set -eux -o pipefail usage() { cat <