diff --git a/Dockerfile.template b/Dockerfile.template index db7781e..d69047f 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -1,73 +1,146 @@ -FROM {{INPUT}}{{TAG}} +FROM {{INPUT}} as base-image USER root SHELL ["/bin/bash", "-lc"] -# If we don't have locales set correctly, the pip install pieces can fail. -# Maybe they should be ARGs, but this seems like a reasonable thing to -# put in the environment by default. -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -# Runtime scripts use ${LOADRSPSTACK} but we need the distinction -# in case you want to create a separate environment for the JupyterLab- -# specific pieces. -ENV LOADSTACK=/opt/lsst/software/stack/loadLSST.bash -ENV LOADRSPSTACK=/opt/lsst/software/rspstack/loadrspstack.bash -RUN mkdir -p /opt/lsst/software/rspstack -# If you want the JupyterLab pieces in their own environment, do the -# COPY and add the environment clone to the python build stage. This -# increases container size by about 60%. If you want it in the same -# environment (the default), link the RSP loadstack instead. -#COPY loadrspstack.bash ${LOADRSPSTACK} -# In general: if an environment variable needs to be used across multiple -# stages, make it an ARG. Introduce it before the first stage that uses it. -RUN ln -s ${LOADSTACK} ${LOADRSPSTACK} -ARG srcdir=/opt/lsst/src -ARG BLD=${srcdir}/build -ARG jl=/opt/lsst/software/jupyterlab -ARG verdir="${jl}/versions.installed" -RUN mkdir -p ${BLD} ${verdir} -COPY stage1-rpm.sh ${BLD} -RUN ${BLD}/stage1-rpm.sh -COPY stage2-os.sh ${BLD} -RUN ${BLD}/stage2-os.sh -COPY stage3-py.sh ${BLD} -RUN ${BLD}/stage3-py.sh -# This should be exposed at runtime for JupyterLab, hence ENV -ENV NODE_OPTIONS="--max-old-space-size=7168 --max-http-header-size=16384" -RUN mkdir -p /usr/local/etc/jupyter -# We update the config during stage 4. -COPY jupyter_server_config.json jupyter_server_config.py \ - /usr/local/etc/jupyter/ -COPY stage4-jup.sh ${BLD} -RUN ${BLD}/stage4-jup.sh -COPY local01-nbstripjq.sh local02-hub.sh local03-showrspnotice.sh \ - local04-pythonrc.sh local05-path.sh local06-term.sh \ - local07-namespaceenv.sh \ + +RUN mkdir -p /tmp/build +WORKDIR /tmp/build + +COPY scripts/install-base-packages /tmp/build +RUN ./install-base-packages + +# Now we have a patched python container. Add system dependencies. + +FROM base-image as deps-image +COPY scripts/install-dependency-packages /tmp/build +RUN ./install-dependency-packages + +# Add other system-level files + +# /etc/profile.d parts + +RUN mkdir -p /etc/profile.d + +COPY profile.d/local01-nbstripjq.sh profile.d/local02-hub.sh \ + profile.d/local03-showrspnotice.sh profile.d/local04-pythonrc.sh \ + profile.d/local05-path.sh profile.d/local06-term.sh \ + profile.d/local07-namespaceenv.sh profile.d/local08-setupstack.sh \ /etc/profile.d/ -COPY lsst_kernel.json \ + +# /etc/skel + +RUN for i in notebooks WORK DATA; do mkdir -p /etc/skel/${i}; done + +COPY skel/pythonrc /etc/skel/.pythonrc +COPY skel/gitconfig /etc/skel/.gitconfig +COPY skel/git-credentials /etc/skel/.git-credentials +COPY skel/user_setups /etc/skel/notebooks/.user_setups + +# Might want to move these? Or make them owned by jupyter user? +# But for right now they need to live here as a compatibility layer if +# nothing else. + +COPY jupyter_server/jupyter_server_config.json \ + jupyter_server/jupyter_server_config.py \ + /usr/local/etc/jupyter/ + +COPY runtime/lsst_kernel.json \ /usr/local/share/jupyter/kernels/lsst/kernel.json -COPY rsp_notice /usr/local/etc -COPY pythonrc /etc/skel/.pythonrc -COPY gitconfig /etc/skel/.gitconfig -COPY git-credentials /etc/skel/.git-credentials -COPY user_setups /etc/skel/notebooks/.user_setups -COPY lsst_kernel.json lsstlaunch.bash runlab.sh 20-logging.py \ - ${jl}/ -# If running noninteractively, configuration configmap must be mounted at -# /opt/lsst/software/jupyterlab/noninteractive/command/command.json -# and env. var NONINTERACTIVE must be set -COPY stage5-ro.sh ${BLD} -RUN ${BLD}/stage5-ro.sh -# Overwrite Stack Container definitions with more-accurate-for-us ones -ENV DESCRIPTION="Rubin Science Platform Notebook Aspect" -ENV SUMMARY="Rubin Science Platform Notebook Aspect" -WORKDIR /tmp -# This needs to be numeric for k8s non-root contexts. We will -# replace it with the actual UID in the JupyterHub spawner, but 1000:1000 -# is the container underlying lsst user, here lsst_local (as explained in -# stage5-ro.sh). So just in case it's spawned by someone outside a JL -# context, and they manage to get all the setup env right, still not root. + +COPY etc/rsp_notice /usr/local/etc + +COPY scripts/install-system-files /tmp/build +RUN ./install-system-files + +# Add our new unprivileged user. + +FROM deps-image as user-image + +COPY scripts/make-user /tmp/build +RUN ./make-user + +# Give jupyterlab ownership to unprivileged user + +RUN mkdir -p /usr/local/share/jupyterlab +RUN chown lsst_local:lsst_local /usr/local/share/jupyterlab + +# Switch to unprivileged user + +USER lsst_local:lsst_local + +# Add the DM stack. + +FROM user-image as base-stack-image + +COPY scripts/install-dm-stack /tmp/build +RUN ./install-dm-stack {{TAG}} + +# Add RSP user-facing packages + +FROM base-stack-image as rsp-stack-image +COPY scripts/install-rsp-user /tmp/build +RUN ./install-rsp-user + +FROM rsp-stack-image as jupyterlab-image + +COPY scripts/install-jupyterlab /tmp/build +RUN ./install-jupyterlab + +FROM jupyterlab-image as base-rsp-image + +RUN mkdir -p /usr/local/share/jupyterlab/etc +COPY --chown=lsst_local:lsst_local etc/rsp_notice etc/20-logging.py \ + jupyter_server/jupyter_server_config.json \ + jupyter_server/jupyter_server_config.py \ + /usr/local/share/jupyterlab/etc/ + +COPY --chown=lsst_local:lsst_local runtime/lsst_kernel.json \ + runtime/lsstlaunch.bash runtime/runlab /usr/local/share/jupyterlab/ + +FROM base-rsp-image as notebooks-rsp-image + +# Check out notebooks-at-build-time +COPY scripts/install-notebooks /tmp/build +RUN ./install-notebooks + +FROM notebooks-rsp-image as compat-rsp-image + +# Add compatibility layer to allow for transition from old to new +# paths. + +COPY scripts/install-compat /tmp/build +RUN ./install-compat + +FROM compat-rsp-image as manifests-rsp-image + +# Get our manifests. This has always been really useful for debugging +# "what broke this week?" + +COPY scripts/generate-versions /tmp/build +RUN ./generate-versions + +FROM manifests-rsp-image as rsp-image + + +# Clean up. +# This needs to be numeric, since we will remove /etc/passwd and friends +# while we're running. +USER 0:0 +WORKDIR / + +COPY scripts/cleanup-files / +RUN ./cleanup-files +RUN rm ./cleanup-files + +# Back to unprivileged USER 1000:1000 -CMD [ "/opt/lsst/software/jupyterlab/runlab.sh" ] +WORKDIR /tmp + +CMD ["/usr/local/share/jupyterlab/runlab"] + +# Overwrite Stack Container definitions with more-accurate-for-us ones +ENV DESCRIPTION="Two-Python Rubin Science Platform Notebook Aspect" +ENV SUMMARY="Two-Python Rubin Science Platform Notebook Aspect" + LABEL description="Rubin Science Platform Notebook Aspect: {{IMAGE}}" \ name="{{IMAGE}}" \ version="{{VERSION}}" diff --git a/Makefile b/Makefile index 7fb2b5c..b999432 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ ifeq ($(image),) endif ifeq ($(input),) - input = docker.io/lsstsqre/centos:7-stack-lsst_distrib- + input = docker.io/library/python:3.12-slim # For one of the four build targets, you need to include the colon here, # and the input tag has to end with $(tag). For "retag" it's different # and is explained below. @@ -88,15 +88,6 @@ ifneq ($(supplementary),) version := exp_$(version)_$(supplementary) endif -# We don't have an arm64 build of the DM stack yet, so if you happen to be -# building on such a machine (e.g. Apple Silicon), cross-build to amd64 -# instead - -uname := $(shell uname -p) -ifeq ($(uname),arm) - platform := --platform amd64 -endif - # Experimentals do not get tagged as latest anything. Dailies, weeklies, and # releases get tagged as latest_. The "latest" tag for the lab # container should always point to the latest weekly or release, but not a @@ -159,11 +150,10 @@ push: image # I keep getting this wrong, so make it work either way. build: image -# Force DOCKER_BUILDKIT off, to appease GitHub Actions (6 Aug 2023) image: dockerfile img=$$(echo $(image) | cut -d ',' -f 1) && \ more=$$(echo $(image) | cut -d ',' -f 2- | tr ',' ' ') && \ - DOCKER_BUILDKIT=0 $(DOCKER) build ${platform} -t $${img}:$(version) . && \ + $(DOCKER) build ${platform} -t $${img}:$(version) . && \ for m in $${more}; do \ $(DOCKER) tag $${img}:$(version) $${m}:$(version) ; \ done diff --git a/bld b/bld deleted file mode 100755 index 346440e..0000000 --- a/bld +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# -# This is now obsolete. You should use the Makefile instead. Once our -# ci jobs are updated, this script will be deleted. -# - -set -e - -function usage() { - echo 1>&2 "Usage: $0 [-d] [-x] [-i image] [-s additional] TAG" - echo 1>&2 " -d enables dry run (no Docker build or push)" - echo 1>&2 " -x is 'do not push' (but does build)." - echo 1>&2 " -i image specifies target image location." - echo 1>&2 " -s additional adds '_additional' to end of exp. build tag." - echo 1>&2 " default image='docker.io/lsstsqre/sciplat-lab'" - echo 1>&2 " typical TAG='w_2021_03'" - exit 2 -} - -function cleanup { - if [ -n "${WORKDIR}" ]; then - rm -rf ${WORKDIR} - fi -} - -trap cleanup EXIT -WORKDIR="" -OPTIND=1 -DRY_RUN=0 -SUPPLEMENTARY=0 -NOPUSH=0 -IMAGE="docker.io/lsstsqre/sciplat-lab" -TARGET="all" -while getopts ':hvdxi:s:efpt:n:b:' opt; do - case $opt in - h) - usage - ;; - d) - TARGET="dockerfile" - ;; - x) - TARGET="image" - ;; - i) - IMAGE=${OPTARG} - ;; - s) - SUPPLEMENTARY=${OPTARG} - ;; - e | f | p| t | n | b | v) - echo "Obsolete option ${opt} no longer has any effect." - ;; - \?) - usage - ;; - esac -done -shift $((OPTIND - 1)) -TAG=${1} -if [ -z "${TAG}" ] || [ $# -gt 1 ]; then - usage -fi - -suparg="" -if [ -n "${SUPPLEMENTARY}" ]; then - suparg="supplementary=${SUPPLEMENTARY}" -fi - -make tag=${TAG} image=${IMAGE} ${suparg} ${TARGET} diff --git a/20-logging.py b/etc/20-logging.py similarity index 100% rename from 20-logging.py rename to etc/20-logging.py diff --git a/rsp_notice b/etc/rsp_notice similarity index 100% rename from rsp_notice rename to etc/rsp_notice diff --git a/jupyter_server_config.json b/jupyter_server/jupyter_server_config.json similarity index 100% rename from jupyter_server_config.json rename to jupyter_server/jupyter_server_config.json diff --git a/jupyter_server_config.py b/jupyter_server/jupyter_server_config.py similarity index 100% rename from jupyter_server_config.py rename to jupyter_server/jupyter_server_config.py diff --git a/loadrspstack.bash b/loadrspstack.bash deleted file mode 100644 index 58430b5..0000000 --- a/loadrspstack.bash +++ /dev/null @@ -1,7 +0,0 @@ -# This script is intended to be used with bash to load the RSP clone of -# the minimal LSST environment -# Usage: source loadrspstack.bash - -export LSST_CONDA_ENV_NAME="rsp-$(source ${LOADSTACK} && \ - echo "${LSST_CONDA_ENV_NAME}")" -source ${LOADSTACK} # Uses modified LSST_CONDA_ENV_NAME diff --git a/local01-nbstripjq.sh b/profile.d/local01-nbstripjq.sh similarity index 100% rename from local01-nbstripjq.sh rename to profile.d/local01-nbstripjq.sh diff --git a/local02-hub.sh b/profile.d/local02-hub.sh similarity index 100% rename from local02-hub.sh rename to profile.d/local02-hub.sh diff --git a/local03-showrspnotice.sh b/profile.d/local03-showrspnotice.sh similarity index 83% rename from local03-showrspnotice.sh rename to profile.d/local03-showrspnotice.sh index 724f2c4..c27f8ea 100644 --- a/local03-showrspnotice.sh +++ b/profile.d/local03-showrspnotice.sh @@ -1,5 +1,7 @@ #!/bin/sh +etc="/usr/local/share/jupyterlab/etc" + site_recommendation () { cat << EOF @@ -26,11 +28,11 @@ case "$-" in # Are we a login shell? if shopt -q login_shell; then # Yes. Display the notice(s) - if [ -e "/usr/local/etc/rsp_notice" ]; then - cat /usr/local/etc/rsp_notice + if [ -e "${etc}/rsp_notice" ]; then + cat ${etc}/rsp_notice site_recommendation fi - msgdir="/opt/lsst/software/jupyterlab/messages.d" + msgdir="${etc}/messages.d" if [ -e ${msgdir} ]; then any=$(ls ${msgdir}) if [ -n "${any}" ]; then diff --git a/local04-pythonrc.sh b/profile.d/local04-pythonrc.sh similarity index 100% rename from local04-pythonrc.sh rename to profile.d/local04-pythonrc.sh diff --git a/local05-path.sh b/profile.d/local05-path.sh similarity index 100% rename from local05-path.sh rename to profile.d/local05-path.sh diff --git a/local06-term.sh b/profile.d/local06-term.sh similarity index 100% rename from local06-term.sh rename to profile.d/local06-term.sh diff --git a/local07-namespaceenv.sh b/profile.d/local07-namespaceenv.sh similarity index 100% rename from local07-namespaceenv.sh rename to profile.d/local07-namespaceenv.sh diff --git a/profile.d/local08-setupstack.sh b/profile.d/local08-setupstack.sh new file mode 100644 index 0000000..582e834 --- /dev/null +++ b/profile.d/local08-setupstack.sh @@ -0,0 +1,4 @@ +#!/bin/sh +if [ -n "${RUNNING_INSIDE_JUPYTERLAB}" ]; then + . /opt/lsst/software/stack/loadLSST.bash +fi diff --git a/runlab.sh b/runlab.sh deleted file mode 100755 index c1cd193..0000000 --- a/runlab.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# This script is a stopgap until we separate the DM stack and the Python -# responsible for running JupyterLab, and can use the entrypoint directly, -# using reset_user_env() in the LabRunner class, and not requiring knowledge -# of the stack-loading magic. - -function reset_user_env() { - local now=$(date +%Y%m%d%H%M%S) - local reloc="${HOME}/.user_env.${now}" - mkdir -p "${reloc}" - local moved="" - # Dirs - for i in cache conda eups local jupyter; do - if [ -d "${HOME}/.${i}" ]; then - mv "${HOME}/.${i}" "${reloc}" - moved="yes" - fi - done - # Files; they're not necessarily at the top level - if [ -f "${HOME}/notebooks/.user_setups" ]; then - mkdir -p "${reloc}/notebooks" - mv "${HOME}/notebooks/.user_setups" "${reloc}/notebooks/user_setups" - moved="yes" - fi - # If nothing was actually relocated, then do not keep the reloc directory - if [ -z "${moved}" ]; then - rmdir "${reloc}" - fi -} - -# Start of mainline code -if [ -n "${RESET_USER_ENV}" ]; then - reset_user_env -fi -# LOADRSPSTACK should be set, but if not... -if [ -z "${LOADRSPSTACK}" ]; then - if [ -e "/opt/lsst/software/rspstack/loadrspstack.bash" ]; then - LOADRSPSTACK="/opt/lsst/software/rspstack/loadrspstack.bash" - else - LOADRSPSTACK="/opt/lsst/software/stack/loadLSST.bash" - fi -fi -export LOADRSPSTACK -source ${LOADRSPSTACK} -source /etc/profile.d/local05-path.sh - -# Now we transfer control to the Python entrypoint "launch-rubin-jupyterlab", -# defined as part of lsst.rsp (in the lsst-rsp package). - -exec launch-rubin-jupyterlab diff --git a/lsst_kernel.json b/runtime/lsst_kernel.json similarity index 62% rename from lsst_kernel.json rename to runtime/lsst_kernel.json index 64f55bc..efd777e 100644 --- a/lsst_kernel.json +++ b/runtime/lsst_kernel.json @@ -2,7 +2,7 @@ "display_name": "LSST", "language": "python", "argv": [ - "/opt/lsst/software/jupyterlab/lsstlaunch.bash", + "/usr/local/share/jupyterlab/lsstlaunch.bash", "{connection_file}" ] } diff --git a/lsstlaunch.bash b/runtime/lsstlaunch.bash similarity index 100% rename from lsstlaunch.bash rename to runtime/lsstlaunch.bash diff --git a/runtime/runlab b/runtime/runlab new file mode 100755 index 0000000..9d8e909 --- /dev/null +++ b/runtime/runlab @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# This script just activates the Jupyterlab virtualenv and then uses the +# lsst.rsp.startup class entrypoint directly. + +source /usr/local/share/jupyterlab/venv/bin/activate + +# Now we transfer control to the Python entrypoint "launch-rubin-jupyterlab", +# defined as part of lsst.rsp (in the lsst-rsp package). + +exec launch-rubin-jupyterlab diff --git a/scripts/cleanup-files b/scripts/cleanup-files new file mode 100755 index 0000000..b51ad65 --- /dev/null +++ b/scripts/cleanup-files @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -x +set -euo pipefail + +for f in passwd group shadow gshadow subuid subgid; do + rm /etc/${f} /etc/${f}- +done + +rm -rf /tmp/* /tmp/.* # Modern Unixes don't recursively remove ".." + diff --git a/scripts/generate-versions b/scripts/generate-versions new file mode 100755 index 0000000..c713c0a --- /dev/null +++ b/scripts/generate-versions @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -x + +set -eo pipefail # -u and mamba don't play well: ADDR2LINE: unbound variable + +# First system + +vi_dir="/usr/local/share/jupyterlab/versions-installed" +mkdir -p ${vi_dir} +cd ${vi_dir} +dpkg -l > "system-dpkgs" +python -m pip list > "system-python" + +# Now JupyterLab python + +source /usr/local/share/jupyterlab/venv/bin/activate +python -m uv pip list > "jupyterlab-python" +(jupyter labextension list 2>&1) > "lab-extensions" +deactivate + +# Now the stack +vi_dir="/opt/lsst/software/jupyterlab/versions-installed" +source /opt/lsst/software/stack/loadLSST.bash +mkdir -p ${vi_dir} +cd ${vi_dir} +mamba env export > conda-stack.yaml + diff --git a/scripts/install-base-packages b/scripts/install-base-packages new file mode 100755 index 0000000..b77e57e --- /dev/null +++ b/scripts/install-base-packages @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -x + +# This script updates packages in the base Docker image that's used by both +# the build and runtime images, and gives us a place to install additional +# system-level packages with apt-get. +# +# Based on the blog post: +# https://pythonspeed.com/articles/system-packages-docker/ + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +set -euo pipefail + +# Tell apt-get we're never going to be able to give manual feedback. +export DEBIAN_FRONTEND=noninteractive + +# Update the package listing, so we know what packages exist. +apt-get update + +# Install security updates. +apt-get -y upgrade + +# Delete cached files we don't need anymore. +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/scripts/install-compat b/scripts/install-compat new file mode 100755 index 0000000..67d6ec2 --- /dev/null +++ b/scripts/install-compat @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -x +# +# This can be pretty minimal. Effectively we just want to ensure that +# files once found in the old-style layout still exist, but point to their +# new equivalent. +# +# lsst_dask.yml and idds.cfg.client.template are copied in by nublado and +# put in /opt/lsst/software/jupyterlab, which seems fine. +# + +set -euo pipefail + +cd /opt/lsst/software +mkdir jupyterlab +cd jupyterlab + +realdir="../../../../usr/local/share/jupyterlab" +for file in lsst_kernel.json lsstlaunch.bash; do + ln -s "${realdir}/${file}" +done +ln -s "${realdir}/etc/20-logging.py" +ln -s "${realdir}/runlab" runlab.sh diff --git a/scripts/install-dependency-packages b/scripts/install-dependency-packages new file mode 100755 index 0000000..9a45b17 --- /dev/null +++ b/scripts/install-dependency-packages @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -x + +# This script installs additional packages used by the dependency image but +# not needed by the runtime image, such as additional packages required to +# build Python dependencies. +# +# Since the base image wipes all the apt caches to clean up the image that +# will be reused by the runtime image, we unfortunately have to do another +# apt-get update here, which wastes some time and network. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Tell apt-get we're never going to be able to give manual +# feedback: +export DEBIAN_FRONTEND=noninteractive + +# Update the package listing, so we know what packages exist: +apt-get update + +# Install various dependencies for runtime. + +# build-essential: sometimes needed to build Python modules +# git: required by setuptools_scm +# libffi-dev: sometimes needed to build cffi, a cryptography dependency +# libxss1 ... pandoc: needed for export of notebooks in various formats +# libdigest-md5-file-perl ... file: generally useful utilities +# man: it is an interactive system, after all +# curl: required to download the DM stack installer +# nano ... ed: enough editors to cover most people's habits +apt-get -y install --no-install-recommends \ + build-essential git libffi-dev \ + libxss1 libasound2 libcups2 libpango-1.0.0 libgtk-3-0 \ + libnss3 libnspr4 libdrm2 libgbm1 pandoc \ + libdigest-md5-file-perl jq gh hub unzip ack screen tmux tree file \ + man curl \ + nano vim emacs-nox ed + +# Delete cached files we don't need anymore: +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/scripts/install-dm-stack b/scripts/install-dm-stack new file mode 100755 index 0000000..a0930ab --- /dev/null +++ b/scripts/install-dm-stack @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -x + +# -u makes lsstinstall fail, via mamba ADDR2LINE: unbound variable. +set -eo pipefail + +TAG=$1 + +stackdir=/opt/lsst/software/stack +sbt=https://raw.githubusercontent.com/lsst/shebangtron/main/shebangtron + +cd $stackdir +curl -OL https://ls.st/lsstinstall +chmod u+x lsstinstall +./lsstinstall -T $TAG +source $stackdir/loadLSST.bash +for prod in $EUPS_PRODUCTS; do + eups distrib install --no-server-tags -vvv "$prod" -t "$EUPS_TAG" +done +find ${stackdir} -exec strip --strip-unneeded --preserve-dates {} + \ + > /dev/null 2>&1 || true +find ${stackdir} -maxdepth 5 -name tests -type d -exec rm -rf {} + \ + > /dev/null 2>&1 || true +find ${stackdir} -maxdepth 6 \ + \( -path "*doc/html" -o -path "*doc/xml" \) -type d -exec rm -rf {} + \ + > /dev/null 2>&1 || true +find ${stackdir} -maxdepth 5 -name src -type d -exec rm -rf {} + \ + > /dev/null 2>&1 || true + +eups distrib install -t $TAG lsst_distrib +curl -sSL $sbt | python +setup lsst_distrib + +mamba clean -a -f -y --no-banner # Not clear that -f is safe. diff --git a/scripts/install-jupyterlab b/scripts/install-jupyterlab new file mode 100755 index 0000000..8ded56e --- /dev/null +++ b/scripts/install-jupyterlab @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -x +set -euo pipefail + +# Install Jupyterlab and its runtime extensions under the system python +# (not the Stack python). This will let us iterate the UI without having +# to worry about stack compatibility. + +# First, install uv, because it's a whole lot faster than pip. + +python3 -m venv /usr/local/share/jupyterlab/venv +source /usr/local/share/jupyterlab/venv/bin/activate + +python3 -m pip install uv + +# We need to install datashader and its dependencies first, because otherwise +# we get stuck in a nasty llvm version dependency hole. + +python3 -m uv pip install --upgrade datashader + +# Here we go. Most of these are Jupyter-related. Some (like black and mypy) +# are just a good idea. + +python3 -m uv pip install --upgrade \ + astrowidgets \ + black \ + black-nb \ + bokeh \ + bqplot \ + firefly-client \ + geoviews \ + holoviews \ + ipydatawidgets \ + ipympl \ + ipyvolume \ + ipyvuetify \ + ipywebrtc \ + ipywidgets \ + jupyter \ + jupyter_bokeh \ + jupyter_firefly_extensions \ + jupyter-dash \ + jupyter-resource-usage \ + jupyter-server-proxy \ + jupyterhub \ + jupyterlab \ + jupyterlab_execute_time \ + jupyterlab_iframe \ + jupyterlab_widgets \ + lckr-jupyterlab-variableinspector \ + llvmlite \ + lsst-rsp \ + mypy \ + 'nbconvert[webpdf]' \ + nbdime \ + nbval \ + panel \ + playwright \ + plotly \ + pythreejs \ + pre-commit \ + rsp-jupyter-extensions \ + ruff \ + sidecar + +# Having done that, install chromium (for webpdf export) +playwright install chromium + +# File sharing doesn't work in the RSP environment; remove the extension. +jupyter labextension disable "@jupyterlab/filebrowser-extension:share-file" +# And Jupyter News is just obnoxious +jupyter labextension disable "@jupyterlab/apputils-extension:announcements" +# Our RSP menu supersedes the Hub menu items +jupyter labextension disable "@jupyterlab/hub-extension:menu" + +jupyter labextension lock "@jupyterlab/hub-extension:menu" \ + "@jupyterlab/apputils-extension:announcements" \ + "@jupyterlab/filebrowser-extension:share-file" + +# Clean up cache +uv cache clean diff --git a/scripts/install-notebooks b/scripts/install-notebooks new file mode 100755 index 0000000..3f317d7 --- /dev/null +++ b/scripts/install-notebooks @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -x +set -euo pipefail + +# Check out notebooks-at-build-time +# Do a shallow clone (important for the tutorials) +nbdir="/opt/lsst/software/notebooks-at-build-time" +mkdir -p "${nbdir}" +branch="prod" +notebooks="lsst-sqre/system-test rubin-dp0/tutorial-notebooks" +cd ${nbdir} +for n in ${notebooks}; do + git clone --depth 1 -b ${branch} "https://github.com/${n}" +done diff --git a/scripts/install-rsp-user b/scripts/install-rsp-user new file mode 100755 index 0000000..a4346de --- /dev/null +++ b/scripts/install-rsp-user @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -x +# -u causes failure in activate: ADDR2LINE: unbound variable +set -eo pipefail +stackdir=/opt/lsst/software/stack + +# Never allow the installation to upgrade rubin_env. Generally enforcing +# the pin is only needed for releases, where the current version may have +# moved ahead. + +source $stackdir/loadLSST.bash + +rubin_env_ver=$(mamba list rubin-env$ --no-banner --json | \ + jq -r '.[0].version') +# Refactor of rubin-env-rsp coming later to separate user packages from +# Lab environment packages. +mamba install --no-banner -y "rubin-env-rsp==${rubin_env_ver}" + +# As with conda->mamba, uv is compatible with pip but much faster. It +# matters less here, of course, because there are many fewer +# pip-installed packages. +pip install uv + +# These are the things that are not available on conda-forge. +# Note that we are not installing with `--upgrade`. That is so that if +# lower-level layers have already installed the package, pinned to a version +# they need, we won't upgrade it. But if it isn't already installed, we'll +# just take the latest available. `--no-build-isolation` ensures that any +# source packages use C++ libraries from conda-forge. + +# "--no-build-isolation" means we're also responsible for the dependencies +# not already provided by something in the conda env. In this case, +# structlog and symbolicmode from lsst-rsp. +uv pip install --no-build-isolation \ + 'lsst-rsp>=0.5.1' \ + structlog \ + 'symbolicmode<3' \ + 'jupyter_firefly_extensions' + +# Clean caches +mamba clean -a -y -f --no-banner +uv cache clean diff --git a/scripts/install-system-files b/scripts/install-system-files new file mode 100755 index 0000000..4357ad6 --- /dev/null +++ b/scripts/install-system-files @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -x +set -euo pipefail + +# This is for Fritz, and my nefarious plan to make the "te" in "Jupyter" TECO. +# We switched from TECOC to Paul Koning's Python implementation because it +# simplifies installation a bit. I doubt anyone is going to complain. + +src=/usr/local/share/git +mkdir -p ${src} +cd ${src} +git clone https://github.com/pkoning2/pyteco.git +install -m 0755 pyteco/teco.py /usr/local/bin/teco + +# The default terminal colors look bad in light mode. +git clone https://github.com/seebi/dircolors-solarized.git +cp dircolors-solarized/dircolors* /etc diff --git a/scripts/make-user b/scripts/make-user new file mode 100755 index 0000000..b139bcc --- /dev/null +++ b/scripts/make-user @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -x +set -euo pipefail + +# Create user and home directory. +stackdir=/opt/lsst/software/stack +mkdir -p $stackdir +groupadd -g 1000 lsst_local +grpconv +useradd -u 1000 -g 1000 -d $stackdir -s /bin/bash lsst_local +pwconv + +# We're going to want other directories there too. +chown -R lsst_local:lsst_local $(dirname $stackdir) diff --git a/git-credentials b/skel/git-credentials similarity index 100% rename from git-credentials rename to skel/git-credentials diff --git a/gitconfig b/skel/gitconfig similarity index 100% rename from gitconfig rename to skel/gitconfig diff --git a/pythonrc b/skel/pythonrc similarity index 100% rename from pythonrc rename to skel/pythonrc diff --git a/user_setups b/skel/user_setups similarity index 100% rename from user_setups rename to skel/user_setups diff --git a/stage1-rpm.sh b/stage1-rpm.sh deleted file mode 100755 index d94fa85..0000000 --- a/stage1-rpm.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# Centos mirrors are dead. -sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo -sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo - -# This will be an interactive system, so we do want man pages after all -# And we have non-US users, so we do not want to force only en_US.utf-8 -sed -i -e '/tsflags\=nodocs/d' \ - -e '/override_install_langs\=en_US.utf8/d' /etc/yum.conf -yum clean all -yum install -y epel-release man man-pages -yum repolist -yum -y upgrade -rpm -qa --qf "%{NAME}\n" | xargs yum -y reinstall -# Add some other packages -# libXScrnSaver ... gtk3 are needed for the chromium installation for -# JupyterLab WebPDF conversion -# perl-Digest-MD5 ... file are generally useful utilities -# ...and finally enough editors to cover most people's habits -yum -y install \ - libXScrnSaver alsa-lib cups-libs at-spi2-atk pango gtk3 \ - perl-Digest-MD5 jq unzip ack screen tmux tree file \ - nano vim-enhanced emacs-nox ed -# Clear build cache -yum clean all - -# export RPM list; verdir is an ARG and has already been created. -rpm -qa | sort > ${verdir}/rpmlist.txt diff --git a/stage2-os.sh b/stage2-os.sh deleted file mode 100755 index 954a287..0000000 --- a/stage2-os.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# OS-level packages that are not packaged by either RPM or Conda -mkdir -p ${srcdir}/thirdparty - -# Install Hub -cd /tmp -V="2.14.2" -FN="hub-linux-amd64-${V}" -F="${FN}.tgz" -URL="https://github.com/github/hub/releases/download/v${V}/${F}" -cmd="curl -L ${URL} -o ${F}" -${cmd} -tar xpfz ${F} -install -m 0755 ${FN}/bin/hub /usr/bin -rm -rf ${F} ${FN} - -# This is for Fritz, and my nefarious plan to make the "te" in "Jupyter" -# TECO -# We switched from TECOC to Paul Koning's Python implementation because it -# simplifies installation a bit. I doubt anyone is going to complain. -cd ${srcdir}/thirdparty -source ${LOADRSPSTACK} # To get git -git clone https://github.com/pkoning2/pyteco.git -cd pyteco -install -m 0755 teco.py /usr/local/bin/teco - -# The default terminal colors look bad in light mode. -cd ${srcdir}/thirdparty -git clone https://github.com/seebi/dircolors-solarized.git -cd dircolors-solarized -cp dircolors* /etc - -# Clear our caches -rm -rf /tmp/* /tmp/.[0-z]* diff --git a/stage3-py.sh b/stage3-py.sh deleted file mode 100755 index c821b24..0000000 --- a/stage3-py.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh -set -e - -#This commented-out bit, plus changing the definition of LOADRSPSTACK in -# Dockerfile.template, will clone the environment rather than installing -# into the stack environment itself. This adds 60% or so to the container -# size. -# -# source ${LOADSTACK} -# rspname="rsp-${LSST_CONDA_ENV_NAME}" -# mamba create --name ${rspname} --clone ${LSST_CONDA_ENV_NAME} -# -source ${LOADRSPSTACK} - -# Mamba is compatible with conda, but much faster -if [ -z "$(which mamba)" ]; then - conda install -y mamba -fi - -# Never allow the installation to upgrade rubin_env. Generally enforcing -# the pin is only needed for releases, where the current version may have -# moved ahead. -rubin_env_ver=$(mamba list rubin-env$ --no-banner --json \ - | jq -r '.[0].version') -# Do the rest of the installation. -mamba install --no-banner -y \ - "rubin-env-rsp==${rubin_env_ver}" - -# Force newer firefly. Unnecessary when rubin-env >= 9.0.0 appears. -mamba install --no-banner -y 'firefly-client>=3' - -# As with conda->mamba, uv is compatible with pip but much faster. It -# matters less here, of course, because there are many fewer -# pip-installed packages. -pip install uv - -# These are the things that are not available on conda-forge. -# Note that we are not installing with `--upgrade`. That is so that if -# lower-level layers have already installed the package, pinned to a version -# they need, we won't upgrade it. But if it isn't already installed, we'll -# just take the latest available. `--no-build-isolation` ensures that any -# source packages use C++ libraries from conda-forge. - -# "--no-build-isolation" means we're also responsible for the dependencies -# not already provided by something in the conda env. In this case, -# structlog and symbolicmode from lsst-rsp. - -uv pip install --no-build-isolation \ - rsp-jupyter-extensions \ - jupyter-firefly-extensions \ - 'lsst-rsp>=0.5.1' \ - structlog \ - 'symbolicmode<3' - -# Add stack kernel -python3 -m ipykernel install --name 'LSST' - -# Remove "system" kernel -stacktop="/opt/lsst/software/stack/conda/current" -rm -rf ${stacktop}/envs/${LSST_CONDA_ENV_NAME}/share/jupyter/kernels/python3 - -# Clear mamba, pip, and uv caches -mamba clean -a -y --no-banner -pip cache purge -uv cache clean - -# Create package version docs. -uv pip freeze > ${verdir}/requirements-stack.txt -mamba env export > ${verdir}/conda-stack.yml diff --git a/stage4-jup.sh b/stage4-jup.sh deleted file mode 100755 index 55f8be1..0000000 --- a/stage4-jup.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -set -e -source ${LOADRSPSTACK} -# File sharing doesn't work in the RSP environment; remove the extension. -jupyter labextension disable "@jupyterlab/filebrowser-extension:share-file" -# And Jupyter News is just obnoxious -jupyter labextension disable "@jupyterlab/apputils-extension:announcements" -# Our RSP menu supersedes the Hub menu items -jupyter labextension disable "@jupyterlab/hub-extension:menu" - -# List installed lab extensions -jupyter labextension list 2>&1 diff --git a/stage5-ro.sh b/stage5-ro.sh deleted file mode 100755 index cde0fba..0000000 --- a/stage5-ro.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -set -e - -# Set up default user directory layout -for i in notebooks WORK DATA idleculler ; do \ - mkdir -p /etc/skel/${i} ; \ -done - -# We renamed "lsst" to "lsst_lcl" because "lsst" was a real GitHub group -# that people were in, when we were using GH as our auth source. It still -# seems likely that we may have a legitimate "lsst" group that is not -# the same as the default group for the build user. - -if [ -d /home/lsst ]; then - mv /home/lsst /home/lsst_lcl -fi - -# Passwd and group are injected as secrets. We don't need their shadow -# variants since they will never be used for authentication, and we definitely -# do not need backups of the passwd/group files. Nor do we need the -# subuid/subgid stuff, since we do not want to delegate user or group -# identities any further. - -rm -f /etc/passwd /etc/shadow /etc/group /etc/gshadow \ - /etc/passwd- /etc/shadow- /etc/group- /etc/gshadow- \ - /etc/subuid /etc/subgid \ - /etc/subuid- /etc/subgid- - -# Check out notebooks-at-build-time -# Do a shallow clone (important for the tutorials) -branch="prod" -notebooks="lsst-sqre/system-test rubin-dp0/tutorial-notebooks" -nbdir="/opt/lsst/software/notebooks-at-build-time" -owd=$(pwd) -source ${LOADRSPSTACK} -mkdir -p ${nbdir} -cd ${nbdir} -for n in ${notebooks}; do - git clone --depth 1 -b ${branch} "https://github.com/${n}" -done -cd ${owd}