From 8462388936675474de7ec74cf9a1984668b3dc19 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 22 Oct 2024 14:50:10 +1100 Subject: [PATCH] implement update.py --- recipe/conda_build_config.yaml | 19 ++++ recipe/meta.yaml | 11 +- recipe/update.py | 107 +++++++++++++++++++ recipe/update.sh | 184 --------------------------------- 4 files changed, 133 insertions(+), 188 deletions(-) create mode 100644 recipe/update.py delete mode 100644 recipe/update.sh diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml index 3491aa7..0c256de 100644 --- a/recipe/conda_build_config.yaml +++ b/recipe/conda_build_config.yaml @@ -14,6 +14,25 @@ cross_target_platform: - linux-ppc64le - linux-s390x +alma_version: + - "8.9" +glibc_version: + - "2.28" +kh_version: + - "4.18.0" +# glibc artefacts have two build numbers plus the alma version, e.g. +# 2.28-236.el8_9.13.x86_64.rpm +# ↑ ↑ ↑ ↑ +# └glibc_ver └alma_version +# └build1 └build2 +glibc_build1: + - "236" +glibc_build2: + - "13" +kh_build: + - "513.24.1" +# nss-* versions need to be set in meta.yaml (and/or update.py) + ctng_vendor: - conda diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 26fdbad..cd54904 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,22 +1,25 @@ -{% set glibc_version = "2.28" %} -{% set kh_version = "4.18.0" %} {% set build_number = "4" %} +{% if alma_version is undefined %} {% set alma_version = "8.9" %} +{% endif %} + {% set rpm_url = "https://repo.almalinux.org/vault/" ~ alma_version ~ "/BaseOS/" ~ centos_machine ~ "/os/Packages" %} {% set appstream_rpm_url = "https://repo.almalinux.org/vault/" ~ alma_version ~ "/AppStream/" ~ centos_machine ~ "/os/Packages" %} {% set powertools_rpm_url = "https://repo.almalinux.org/vault/" ~ alma_version ~ "/PowerTools/" ~ centos_machine ~ "/os/Packages" %} # artefact contains a first build number, the alma version, and another build number -{% set glibc_build = "236.el" ~ alma_version.replace(".", "_") ~ ".13" %} +{% set glibc_build = glibc_build1 ~ ".el" ~ alma_version.replace(".", "_") ~ "." ~ glibc_build2 %} {% set glibc_string = glibc_version ~ "-" ~ glibc_build %} -{% set kh_string = kh_version ~ "-513.24.1.el" ~ alma_version.replace(".", "_") %} +{% set kh_string = kh_version ~ "-" ~ kh_build ~ ".el" ~ alma_version.replace(".", "_") %} package: name: linux-sysroot version: {{ glibc_version }} source: + # for calculating the hashes below, you can run `python update.py` + # in the recipe folder (after ensuring the versions in CBC are correct) - folder: binary-glibc url: {{ rpm_url }}/glibc-{{ glibc_string }}.{{ centos_machine }}.rpm sha256: a82cc18a623f63f5e3370188543930fbc44cbb668407c3fe2b89fbb717e07279 # [cross_target_platform == "linux-64"] diff --git a/recipe/update.py b/recipe/update.py new file mode 100644 index 0000000..8f816de --- /dev/null +++ b/recipe/update.py @@ -0,0 +1,107 @@ +""" +run this in recipe/ folder like +python update.py +or +python update.py -log=INFO +""" +import argparse +import hashlib +import logging +import requests +import os +from ruamel_yaml import BaseLoader, load + +parser = argparse.ArgumentParser() +parser.add_argument('-log', '--loglevel', default='warning') + +args = parser.parse_args() +logging.basicConfig(level=args.loglevel.upper()) + +cbc = os.path.join(".", "conda_build_config.yaml") +if not os.path.exists(cbc): + raise ValueError("cannot load conda_build_config.yaml; execute script in recipe folder!") + +with open(cbc, "r", encoding="utf-8") as f: + cbc_content = "".join(f.readlines()) + +config = load(cbc_content, Loader=BaseLoader) +rpm_arches = config["centos_machine"] +conda_arches = config["cross_target_platform"] + +url_template = ( + f"https://repo.almalinux.org/vault/{config['alma_version'][0]}" + # second part intententionally not filled yet + "/{subfolder}/{arch}/os/Packages" +) + +el_ver = "el" + config["alma_version"][0].replace(".", "_") +glibc_string = f"{config['glibc_version'][0]}-{config['glibc_build1'][0]}.{el_ver}.{config['glibc_build2'][0]}" +kh_string = f"{config['kh_version'][0]}-{config['kh_build'][0]}.{el_ver}" + +out_lines = [] + +name2string = { + # package name to build string + "glibc": glibc_string, + "glibc-all-langpacks": glibc_string, + "glibc-common": glibc_string, + "glibc-devel": glibc_string, + "glibc-gconv-extra": glibc_string, + "glibc-headers": glibc_string, + "glibc-nss-devel": glibc_string, + "glibc-static": glibc_string, + "kernel-headers": kh_string, + "nss_db": glibc_string, + # manual override + "nss_nis": "3.0-8.el8", + "nss-softokn-freebl": "3.90.0-6.el8_9", +} + +def get_subfolder(pkg, string): + # find in which subfolder the rpm lives on the alma vault; + # we assume that the layout for x86_64 works for all arches + pkg_template = url_template + f"/{pkg}-{string}.x86_64.rpm" + url = pkg_template.format(arch="x86_64", subfolder="BaseOS") + logging.info(f"Testing if {url} exists by downloading") + if requests.get(url).status_code == 200: + return "BaseOS" + url = pkg_template.format(arch="x86_64", subfolder="PowerTools") + logging.info(f"Testing if {url} exists by downloading") + if requests.get(url).status_code == 200: + return "PowerTools" + url = pkg_template.format(arch="x86_64", subfolder="AppStream") + logging.info(f"Testing if {url} exists by downloading") + if requests.get(url).status_code == 200: + return "AppStream" + raise ValueError(f"could not find valid artefact for {pkg}-{string}!") + +for pkg, string in name2string.items(): + out_lines.append(f" - folder: binary-{pkg}") + subfolder = get_subfolder(pkg, string) + url_jinja = ( + "{{ rpm_url }}" if subfolder == "BaseOS" else + "{{ powertools_rpm_url }}" if subfolder == "PowerTools" else + "{{ appstream_rpm_url }}" + ) + string_jinja = ( + "{{ glibc_string }}" if string == glibc_string else + "{{ kh_string }}" if string == kh_string else string + ) + # quadruple curly braces to keep {{ }} jinja templates + out_lines.append(f" url: {url_jinja}/{pkg}-{string_jinja}.{{{{ centos_machine }}}}.rpm") + + for rpm_arch, conda_arch in zip(rpm_arches, conda_arches): + rpm_url = ( + url_template.format(arch=rpm_arch, subfolder=subfolder) + + f"/{pkg}-{string}.{rpm_arch}.rpm" + ) + logging.info(f"Downloading {rpm_url}") + r = requests.get(rpm_url) + if r.status_code != 200: + logging.warning(f"Could not download rpm for {pkg} from {rpm_url}!") + continue + sha = hashlib.sha256(r.content).hexdigest(); + out_lines.append(f' sha256: {sha} # [cross_target_platform == "{conda_arch}"]') + out_lines.append("") + +print("\n".join(out_lines)) diff --git a/recipe/update.sh b/recipe/update.sh deleted file mode 100644 index 6d02da9..0000000 --- a/recipe/update.sh +++ /dev/null @@ -1,184 +0,0 @@ -rm asd.txt - -echo " - folder: binary" >> asd.txt -echo " url: {{ rpm_url }}/glibc-2.17-317.el7.{{ centos_machine }}.rpm" >> asd.txt - -wget http://mirror.centos.org/centos/7.9.2009/os/x86_64/Packages/glibc-2.17-317.el7.x86_64.rpm -sha=$(sha256sum glibc-2.17-317.el7.x86_64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-64\"]" >> asd.txt -rm glibc-2.17-317.el7.x86_64.rpm - -wget http://mirror.centos.org/altarch/7/os/aarch64/Packages/glibc-2.17-317.el7.aarch64.rpm -sha=$(sha256sum glibc-2.17-317.el7.aarch64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-aarch64\"]" >> asd.txt -rm glibc-2.17-317.el7.aarch64.rpm - -wget http://mirror.centos.org/altarch/7/os/ppc64le/Packages/glibc-2.17-317.el7.ppc64le.rpm -sha=$(sha256sum glibc-2.17-317.el7.ppc64le.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-ppc64le\"]" >> asd.txt -rm glibc-2.17-317.el7.ppc64le.rpm - -wget http://download.sinenomine.net/clefos/7/os/s390x/glibc-2.17-317.el7.s390x.rpm -sha=$(sha256sum glibc-2.17-317.el7.s390x.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-s390x\"]" >> asd.txt -rm glibc-2.17-317.el7.s390x.rpm - -echo >> asd.txt - -echo " - folder: binary-glibc-common" >> asd.txt -echo " url: {{ rpm_url }}/glibc-common-2.17-317.el7.{{ centos_machine }}.rpm" >> asd.txt - -wget http://mirror.centos.org/centos/7.9.2009/os/x86_64/Packages/glibc-common-2.17-317.el7.x86_64.rpm -sha=$(sha256sum glibc-common-2.17-317.el7.x86_64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-64\"]" >> asd.txt -rm glibc-common-2.17-317.el7.x86_64.rpm - -wget http://mirror.centos.org/altarch/7/os/aarch64/Packages/glibc-common-2.17-317.el7.aarch64.rpm -sha=$(sha256sum glibc-common-2.17-317.el7.aarch64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-aarch64\"]" >> asd.txt -rm glibc-common-2.17-317.el7.aarch64.rpm - -wget http://mirror.centos.org/altarch/7/os/ppc64le/Packages/glibc-common-2.17-317.el7.ppc64le.rpm -sha=$(sha256sum glibc-common-2.17-317.el7.ppc64le.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-ppc64le\"]" >> asd.txt -rm glibc-common-2.17-317.el7.ppc64le.rpm - -wget http://download.sinenomine.net/clefos/7/os/s390x/glibc-common-2.17-317.el7.s390x.rpm -sha=$(sha256sum glibc-common-2.17-317.el7.s390x.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-s390x\"]" >> asd.txt -rm glibc-common-2.17-317.el7.s390x.rpm - -echo >> asd.txt - -echo " - folder: binary-glibc-devel" >> asd.txt -echo " url: {{ rpm_url }}/glibc-devel-2.17-317.el7.{{ centos_machine }}.rpm" >> asd.txt - -wget http://mirror.centos.org/centos/7.9.2009/os/x86_64/Packages/glibc-devel-2.17-317.el7.x86_64.rpm -sha=$(sha256sum glibc-devel-2.17-317.el7.x86_64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-64\"]" >> asd.txt -rm glibc-devel-2.17-317.el7.x86_64.rpm - -wget http://mirror.centos.org/altarch/7/os/aarch64/Packages/glibc-devel-2.17-317.el7.aarch64.rpm -sha=$(sha256sum glibc-devel-2.17-317.el7.aarch64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-aarch64\"]" >> asd.txt -rm glibc-devel-2.17-317.el7.aarch64.rpm - -wget http://mirror.centos.org/altarch/7/os/ppc64le/Packages/glibc-devel-2.17-317.el7.ppc64le.rpm -sha=$(sha256sum glibc-devel-2.17-317.el7.ppc64le.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-ppc64le\"]" >> asd.txt -rm glibc-devel-2.17-317.el7.ppc64le.rpm - -wget http://download.sinenomine.net/clefos/7/os/s390x/glibc-devel-2.17-317.el7.s390x.rpm -sha=$(sha256sum glibc-devel-2.17-317.el7.s390x.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-s390x\"]" >> asd.txt -rm glibc-devel-2.17-317.el7.s390x.rpm - -echo >> asd.txt - - -echo " - folder: binary-glibc-headers" >> asd.txt -echo " url: {{ rpm_url }}/glibc-headers-2.17-317.el7.{{ centos_machine }}.rpm" >> asd.txt - -wget http://mirror.centos.org/centos/7.9.2009/os/x86_64/Packages/glibc-headers-2.17-317.el7.x86_64.rpm -sha=$(sha256sum glibc-headers-2.17-317.el7.x86_64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-64\"]" >> asd.txt -rm glibc-headers-2.17-317.el7.x86_64.rpm - -wget http://mirror.centos.org/altarch/7/os/aarch64/Packages/glibc-headers-2.17-317.el7.aarch64.rpm -sha=$(sha256sum glibc-headers-2.17-317.el7.aarch64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-aarch64\"]" >> asd.txt -rm glibc-headers-2.17-317.el7.aarch64.rpm - -wget http://mirror.centos.org/altarch/7/os/ppc64le/Packages/glibc-headers-2.17-317.el7.ppc64le.rpm -sha=$(sha256sum glibc-headers-2.17-317.el7.ppc64le.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-ppc64le\"]" >> asd.txt -rm glibc-headers-2.17-317.el7.ppc64le.rpm - -wget http://download.sinenomine.net/clefos/7/os/s390x/glibc-headers-2.17-317.el7.s390x.rpm -sha=$(sha256sum glibc-headers-2.17-317.el7.s390x.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-s390x\"]" >> asd.txt -rm glibc-headers-2.17-317.el7.s390x.rpm - -echo >> asd.txt - - -echo " - folder: binary-glibc-static" >> asd.txt -echo " url: {{ rpm_url }}/glibc-static-2.17-317.el7.{{ centos_machine }}.rpm" >> asd.txt - -wget http://mirror.centos.org/centos/7.9.2009/os/x86_64/Packages/glibc-static-2.17-317.el7.x86_64.rpm -sha=$(sha256sum glibc-static-2.17-317.el7.x86_64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-64\"]" >> asd.txt -rm glibc-static-2.17-317.el7.x86_64.rpm - -wget http://mirror.centos.org/altarch/7/os/aarch64/Packages/glibc-static-2.17-317.el7.aarch64.rpm -sha=$(sha256sum glibc-static-2.17-317.el7.aarch64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-aarch64\"]" >> asd.txt -rm glibc-static-2.17-317.el7.aarch64.rpm - -wget http://mirror.centos.org/altarch/7/os/ppc64le/Packages/glibc-static-2.17-317.el7.ppc64le.rpm -sha=$(sha256sum glibc-static-2.17-317.el7.ppc64le.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-ppc64le\"]" >> asd.txt -rm glibc-static-2.17-317.el7.ppc64le.rpm - -wget http://download.sinenomine.net/clefos/7/os/s390x/glibc-static-2.17-317.el7.s390x.rpm -sha=$(sha256sum glibc-static-2.17-317.el7.s390x.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-s390x\"]" >> asd.txt -rm glibc-static-2.17-317.el7.s390x.rpm - -echo >> asd.txt - - -echo " - folder: binary-kernel-headers" >> asd.txt -echo " url: {{ rpm_url }}/kernel-headers-{{ kernel_headers_version }}-1160.el7.{{ centos_machine }}.rpm # [cross_target_platform == \"linux-64\"]" >> asd.txt -echo " url: {{ rpm_url }}/kernel-headers-{{ kernel_headers_version }}-193.28.1.el7.{{ centos_machine }}.rpm # [cross_target_platform == \"linux-aarch64\"]" >> asd.txt -echo " url: {{ rpm_url }}/kernel-headers-{{ kernel_headers_version }}-1160.el7.{{ centos_machine }}.rpm # [cross_target_platform == \"linux-ppc64le\"]" >> asd.txt -echo " url: {{ rpm_url }}/kernel-headers-{{ kernel_headers_version }}-1160.el7.{{ centos_machine }}.rpm # [cross_target_platform == \"linux-s390x\"]" >> asd.txt - -wget http://mirror.centos.org/centos/7.9.2009/os/x86_64/Packages/kernel-headers-3.10.0-1160.el7.x86_64.rpm -sha=$(sha256sum kernel-headers-3.10.0-1160.el7.x86_64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-64\"]" >> asd.txt -rm kernel-headers-3.10.0-1160.el7.x86_64.rpm - -wget http://mirror.centos.org/altarch/7/os/aarch64/Packages/kernel-headers-4.18.0-193.28.1.el7.aarch64.rpm -sha=$(sha256sum kernel-headers-4.18.0-193.28.1.el7.aarch64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-aarch64\"]" >> asd.txt -rm kernel-headers-4.18.0-193.28.1.el7.aarch64.rpm - -wget http://mirror.centos.org/altarch/7/os/ppc64le/Packages/kernel-headers-3.10.0-1160.el7.ppc64le.rpm -sha=$(sha256sum kernel-headers-3.10.0-1160.el7.ppc64le.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-ppc64le\"]" >> asd.txt -rm kernel-headers-3.10.0-1160.el7.ppc64le.rpm - -wget http://download.sinenomine.net/clefos/7/os/s390x/kernel-headers-3.10.0-1160.el7.s390x.rpm -sha=$(sha256sum kernel-headers-3.10.0-1160.el7.s390x.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-s390x\"]" >> asd.txt -rm kernel-headers-3.10.0-1160.el7.s390x.rpm - -echo >> asd.txt - - -echo " - folder: binary-freebl" >> asd.txt -echo " url: {{ rpm_url }}/nss-softokn-freebl-3.44.0-8.el7_7.{{ centos_machine }}.rpm" >> asd.txt - -wget http://mirror.centos.org/centos/7.9.2009/os/x86_64/Packages/nss-softokn-freebl-3.44.0-8.el7_7.x86_64.rpm -sha=$(sha256sum nss-softokn-freebl-3.44.0-8.el7_7.x86_64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-64\"]" >> asd.txt -rm nss-softokn-freebl-3.44.0-8.el7_7.x86_64.rpm - -wget http://mirror.centos.org/altarch/7/os/aarch64/Packages/nss-softokn-freebl-3.44.0-8.el7_7.aarch64.rpm -sha=$(sha256sum nss-softokn-freebl-3.44.0-8.el7_7.aarch64.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-aarch64\"]" >> asd.txt -rm nss-softokn-freebl-3.44.0-8.el7_7.aarch64.rpm - -wget http://mirror.centos.org/altarch/7/os/ppc64le/Packages/nss-softokn-freebl-3.44.0-8.el7_7.ppc64le.rpm -sha=$(sha256sum nss-softokn-freebl-3.44.0-8.el7_7.ppc64le.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-ppc64le\"]" >> asd.txt -rm nss-softokn-freebl-3.44.0-8.el7_7.ppc64le.rpm - -wget http://download.sinenomine.net/clefos/7/os/s390x/nss-softokn-freebl-3.44.0-8.el7_7.s390x.rpm -sha=$(sha256sum nss-softokn-freebl-3.44.0-8.el7_7.s390x.rpm | cut -b -64) -echo " sha256: $sha # [cross_target_platform == \"linux-s390x\"]" >> asd.txt -rm nss-softokn-freebl-3.44.0-8.el7_7.s390x.rpm - -echo >> asd.txt -