implement update.py

recipe/conda_build_config.yaml

@@ -14,6 +14,25 @@ cross_target_platform:
   - linux-ppc64le
   - linux-s390x
 
+alma_version:
+  - "8.9"
+glibc_version:
+  - "2.28"
+kh_version:
+  - "4.18.0"
+# glibc artefacts have two build numbers plus the alma version, e.g.
+#   2.28-236.el8_9.13.x86_64.rpm
+#   ↑    ↑     ↑   ↑
+#   └glibc_ver └alma_version
+#        └build1   └build2
+glibc_build1:
+  - "236"
+glibc_build2:
+  - "13"
+kh_build:
+  - "513.24.1"
+# nss-* versions need to be set in meta.yaml (and/or update.py)
+
 ctng_vendor:
   - conda
 

recipe/meta.yaml

@@ -1,22 +1,25 @@
-{% set glibc_version = "2.28" %}
-{% set kh_version = "4.18.0" %}
 {% set build_number = "4" %}
 
+{% if alma_version is undefined %}
 {% set alma_version = "8.9" %}
+{% endif %}
+
 {% set rpm_url = "https://repo.almalinux.org/vault/" ~ alma_version ~ "/BaseOS/" ~ centos_machine ~ "/os/Packages" %}
 {% set appstream_rpm_url = "https://repo.almalinux.org/vault/" ~ alma_version ~ "/AppStream/" ~ centos_machine ~ "/os/Packages" %}
 {% set powertools_rpm_url = "https://repo.almalinux.org/vault/" ~ alma_version ~ "/PowerTools/" ~ centos_machine ~ "/os/Packages" %}
 
 # artefact contains a first build number, the alma version, and another build number
-{% set glibc_build = "236.el" ~ alma_version.replace(".", "_") ~ ".13" %}
+{% set glibc_build = glibc_build1 ~ ".el" ~ alma_version.replace(".", "_") ~ "." ~ glibc_build2 %}
 {% set glibc_string = glibc_version ~ "-" ~ glibc_build %}
-{% set kh_string = kh_version ~ "-513.24.1.el" ~ alma_version.replace(".", "_") %}
+{% set kh_string = kh_version ~ "-" ~ kh_build ~ ".el" ~ alma_version.replace(".", "_") %}
 
 package:
   name: linux-sysroot
   version: {{ glibc_version }}
 
 source:
+  # for calculating the hashes below, you can run `python update.py`
+  # in the recipe folder (after ensuring the versions in CBC are correct)
   - folder: binary-glibc
     url: {{ rpm_url }}/glibc-{{ glibc_string }}.{{ centos_machine }}.rpm
     sha256: a82cc18a623f63f5e3370188543930fbc44cbb668407c3fe2b89fbb717e07279  # [cross_target_platform == "linux-64"]

recipe/update.py

@@ -0,0 +1,107 @@
+"""
+run this in recipe/ folder like
+python update.py
+or
+python update.py -log=INFO
+"""
+import argparse
+import hashlib
+import logging
+import requests
+import os
+from ruamel_yaml import BaseLoader, load
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-log', '--loglevel', default='warning')
+
+args = parser.parse_args()
+logging.basicConfig(level=args.loglevel.upper())
+
+cbc = os.path.join(".", "conda_build_config.yaml")
+if not os.path.exists(cbc):
+    raise ValueError("cannot load conda_build_config.yaml; execute script in recipe folder!")
+
+with open(cbc, "r", encoding="utf-8") as f:
+    cbc_content = "".join(f.readlines())
+
+config = load(cbc_content, Loader=BaseLoader)
+rpm_arches = config["centos_machine"]
+conda_arches = config["cross_target_platform"]
+
+url_template = (
+    f"https://repo.almalinux.org/vault/{config['alma_version'][0]}"
+    # second part intententionally not filled yet
+    "/{subfolder}/{arch}/os/Packages"
+)
+
+el_ver = "el" + config["alma_version"][0].replace(".", "_")
+glibc_string = f"{config['glibc_version'][0]}-{config['glibc_build1'][0]}.{el_ver}.{config['glibc_build2'][0]}"
+kh_string = f"{config['kh_version'][0]}-{config['kh_build'][0]}.{el_ver}"
+
+out_lines = []
+
+name2string = {
+    # package name to build string
+    "glibc": glibc_string,
+    "glibc-all-langpacks": glibc_string,
+    "glibc-common": glibc_string,
+    "glibc-devel": glibc_string,
+    "glibc-gconv-extra": glibc_string,
+    "glibc-headers": glibc_string,
+    "glibc-nss-devel": glibc_string,
+    "glibc-static": glibc_string,
+    "kernel-headers": kh_string,
+    "nss_db": glibc_string,
+    # manual override
+    "nss_nis": "3.0-8.el8",
+    "nss-softokn-freebl": "3.90.0-6.el8_9",
+}
+
+def get_subfolder(pkg, string):
+    # find in which subfolder the rpm lives on the alma vault;
+    # we assume that the layout for x86_64 works for all arches
+    pkg_template = url_template + f"/{pkg}-{string}.x86_64.rpm"
+    url = pkg_template.format(arch="x86_64", subfolder="BaseOS")
+    logging.info(f"Testing if {url} exists by downloading")
+    if requests.get(url).status_code == 200:
+        return "BaseOS"
+    url = pkg_template.format(arch="x86_64", subfolder="PowerTools")
+    logging.info(f"Testing if {url} exists by downloading")
+    if requests.get(url).status_code == 200:
+        return "PowerTools"
+    url = pkg_template.format(arch="x86_64", subfolder="AppStream")
+    logging.info(f"Testing if {url} exists by downloading")
+    if requests.get(url).status_code == 200:
+        return "AppStream"
+    raise ValueError(f"could not find valid artefact for {pkg}-{string}!")
+
+for pkg, string in name2string.items():
+    out_lines.append(f"  - folder: binary-{pkg}")
+    subfolder = get_subfolder(pkg, string)
+    url_jinja = (
+        "{{ rpm_url }}" if subfolder == "BaseOS" else
+        "{{ powertools_rpm_url }}" if subfolder == "PowerTools" else
+        "{{ appstream_rpm_url }}"
+    )
+    string_jinja = (
+        "{{ glibc_string }}" if string == glibc_string else
+        "{{ kh_string }}" if string == kh_string else string
+    )
+    # quadruple curly braces to keep {{ }} jinja templates
+    out_lines.append(f"    url: {url_jinja}/{pkg}-{string_jinja}.{{{{ centos_machine }}}}.rpm")
+
+    for rpm_arch, conda_arch in zip(rpm_arches, conda_arches):
+        rpm_url = (
+            url_template.format(arch=rpm_arch, subfolder=subfolder)
+            + f"/{pkg}-{string}.{rpm_arch}.rpm"
+        )
+        logging.info(f"Downloading {rpm_url}")
+        r = requests.get(rpm_url)
+        if r.status_code != 200:
+            logging.warning(f"Could not download rpm for {pkg} from {rpm_url}!")
+            continue
+        sha = hashlib.sha256(r.content).hexdigest();
+        out_lines.append(f'    sha256: {sha}  # [cross_target_platform == "{conda_arch}"]')
+    out_lines.append("")
+
+print("\n".join(out_lines))