From 43f2d3207b9c93354a2d4089df069f5535fd8f92 Mon Sep 17 00:00:00 2001 From: Johanna Reiml Date: Sun, 4 Feb 2024 23:54:30 +0100 Subject: [PATCH 01/19] feat: build cuda variants of pytorch --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 + .github/ISSUE_TEMPLATE/feature_request.yml | 2 + .../workflows/docker-build-test-upload.yml | 13 ++++++ .github/workflows/docker-tag-push.yml | 13 ++++++ .github/workflows/docker.yml | 46 +++++++++++++++++++ .github/workflows/registry-move.yml | 5 +- .github/workflows/registry-overviews.yml | 2 + Makefile | 2 + docs/using/selecting.md | 26 ++++++++++- images/pytorch-cuda-notebook/.dockerignore | 2 + images/pytorch-cuda-notebook/Dockerfile | 21 +++++++++ images/pytorch-cuda-notebook/README.md | 8 ++++ images/pytorch-cuda11-notebook/.dockerignore | 2 + images/pytorch-cuda11-notebook/Dockerfile | 21 +++++++++ images/pytorch-cuda11-notebook/README.md | 8 ++++ images/pytorch-notebook/Dockerfile | 2 +- tagging/images_hierarchy.py | 6 +++ tests/images_hierarchy.py | 2 + .../units/unit_pytorch_cuda.py | 5 ++ .../units/unit_pytorch_cuda11.py | 5 ++ 20 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 images/pytorch-cuda-notebook/.dockerignore create mode 100644 images/pytorch-cuda-notebook/Dockerfile create mode 100644 images/pytorch-cuda-notebook/README.md create mode 100644 images/pytorch-cuda11-notebook/.dockerignore create mode 100644 images/pytorch-cuda11-notebook/Dockerfile create mode 100644 images/pytorch-cuda11-notebook/README.md create mode 100644 tests/pytorch-cuda-notebook/units/unit_pytorch_cuda.py create mode 100644 tests/pytorch-cuda11-notebook/units/unit_pytorch_cuda11.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 064028d4ca..e92f69e336 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,6 +27,8 @@ body: - minimal-notebook - pyspark-notebook - pytorch-notebook + - pytorch-cuda-notebook + - pytorch-cuda11-notebook - r-notebook - scipy-notebook - tensorflow-notebook diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index f0da40624e..c525a43420 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -25,6 +25,8 @@ body: - minimal-notebook - pyspark-notebook - pytorch-notebook + - pytorch-cuda-notebook + - pytorch-cuda11-notebook - r-notebook - scipy-notebook - tensorflow-notebook diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index 01520d34ef..64f04ae722 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -29,6 +29,19 @@ jobs: runs-on: ${{ inputs.runs-on }} steps: + # Image with CUDA needs extra disk space + - name: Free disk space ๐Ÿงน + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + if: contains(inputs.image, 'cuda') && inputs.platform == 'x86_64' + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: false + swap-storage: false + - name: Checkout Repo โšก๏ธ uses: actions/checkout@v4 - name: Create dev environment ๐Ÿ“ฆ diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-push.yml index b68a8d1af8..464d357e78 100644 --- a/.github/workflows/docker-tag-push.yml +++ b/.github/workflows/docker-tag-push.yml @@ -27,6 +27,19 @@ jobs: runs-on: ubuntu-latest steps: + # Image with CUDA needs extra disk space + - name: Free disk space ๐Ÿงน + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + if: contains(inputs.image, 'cuda') && inputs.platform == 'x86_64' + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: false + swap-storage: false + - name: Checkout Repo โšก๏ธ uses: actions/checkout@v4 - name: Create dev environment ๐Ÿ“ฆ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a38058c7c9..388bec7cff 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -216,6 +216,46 @@ jobs: needs: [x86_64-scipy] if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + aarch64-pytorch-cuda: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-cuda-notebook + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-pytorch-cuda: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-cuda-notebook + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-pytorch-cuda11: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-cuda11-notebook + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-pytorch-cuda11: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-cuda11-notebook + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + aarch64-datascience: uses: ./.github/workflows/docker-build-test-upload.yml with: @@ -296,6 +336,8 @@ jobs: julia-notebook, tensorflow-notebook, pytorch-notebook, + pytorch-cuda-notebook, + pytorch-cuda11-notebook, datascience-notebook, pyspark-notebook, all-spark-notebook, @@ -350,6 +392,8 @@ jobs: julia-notebook, tensorflow-notebook, pytorch-notebook, + pytorch-cuda-notebook, + pytorch-cuda11-notebook, datascience-notebook, pyspark-notebook, all-spark-notebook, @@ -403,6 +447,8 @@ jobs: julia-notebook, tensorflow-notebook, pytorch-notebook, + pytorch-cuda-notebook, + pytorch-cuda11-notebook, datascience-notebook, pyspark-notebook, all-spark-notebook, diff --git a/.github/workflows/registry-move.yml b/.github/workflows/registry-move.yml index 09ef3d28a2..ab22fd16e3 100644 --- a/.github/workflows/registry-move.yml +++ b/.github/workflows/registry-move.yml @@ -42,8 +42,7 @@ jobs: strategy: fail-fast: false matrix: - image: - [ + image: [ docker-stacks-foundation, base-notebook, minimal-notebook, @@ -52,6 +51,8 @@ jobs: julia-notebook, tensorflow-notebook, pytorch-notebook, + # pytorch-cuda-notebook, + # pytorch-cuda11-notebook, datascience-notebook, pyspark-notebook, all-spark-notebook, diff --git a/.github/workflows/registry-overviews.yml b/.github/workflows/registry-overviews.yml index 91a6b1f4bc..74662f4e2e 100644 --- a/.github/workflows/registry-overviews.yml +++ b/.github/workflows/registry-overviews.yml @@ -43,6 +43,8 @@ jobs: julia-notebook, tensorflow-notebook, pytorch-notebook, + pytorch-cuda-notebook, + pytorch-cuda11-notebook, datascience-notebook, pyspark-notebook, all-spark-notebook, diff --git a/Makefile b/Makefile index 7965b641bf..7d3c57fdd2 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ ALL_IMAGES:= \ scipy-notebook \ tensorflow-notebook \ pytorch-notebook \ + pytorch-cuda-notebook \ + pytorch-cuda11-notebook \ datascience-notebook \ pyspark-notebook \ all-spark-notebook diff --git a/docs/using/selecting.md b/docs/using/selecting.md index d44f456230..e7e8220012 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -195,7 +195,29 @@ It contains: `jupyter/pytorch-notebook` includes popular Python deep learning libraries. - Everything in `jupyter/scipy-notebook` and its ancestor images -- [pytorch](https://pytorch.org/) machine learning library +- [pytorch](https://pytorch.org/) machine learning library (only CPU) + +### jupyter/pytorch-cuda-notebook + +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/pytorch-cuda-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/pytorch-cuda-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/pytorch-cuda-notebook?tab=tags) + +`jupyter/pytorch-cuda-notebook` includes popular Python deep learning libraries. + +- Everything in `jupyter/scipy-notebook` and its ancestor images +- [pytorch](https://pytorch.org/) machine learning library (with CUDA 12.X) + +### jupyter/pytorch-cuda11-notebook + +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/pytorch-cuda11-notebook) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/pytorch-cuda11-notebook/Dockerfile) | +[Quay.io image tags](https://quay.io/repository/jupyter/pytorch-cuda11-notebook?tab=tags) + +`jupyter/pytorch-cuda11-notebook` includes popular Python deep learning libraries. + +- Everything in `jupyter/scipy-notebook` and its ancestor images +- [pytorch](https://pytorch.org/) machine learning library (with CUDA 11.8) ### jupyter/datascience-notebook @@ -317,7 +339,7 @@ See the [contributing guide](../contributing/stacks.md) for information about ho [almond]: https://almond.sh [almond_b]: https://mybinder.org/v2/gh/almond-sh/examples/master?urlpath=lab%2Ftree%2Fnotebooks%2Findex.ipynb -### GPU-accelerated notebooks +### Other GPU-accelerated notebooks | Flavor | Description | | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | diff --git a/images/pytorch-cuda-notebook/.dockerignore b/images/pytorch-cuda-notebook/.dockerignore new file mode 100644 index 0000000000..9dea340f35 --- /dev/null +++ b/images/pytorch-cuda-notebook/.dockerignore @@ -0,0 +1,2 @@ +# Documentation +README.md diff --git a/images/pytorch-cuda-notebook/Dockerfile b/images/pytorch-cuda-notebook/Dockerfile new file mode 100644 index 0000000000..5b7f0be3c0 --- /dev/null +++ b/images/pytorch-cuda-notebook/Dockerfile @@ -0,0 +1,21 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_CONTAINER + +LABEL maintainer="Jupyter Project " + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install PyTorch with pip (https://pytorch.org/get-started/locally/) +# hadolint ignore=DL3013 +RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com \ + 'torch' \ + 'torchvision' \ + 'torchaudio' && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/images/pytorch-cuda-notebook/README.md b/images/pytorch-cuda-notebook/README.md new file mode 100644 index 0000000000..1900d7cb5f --- /dev/null +++ b/images/pytorch-cuda-notebook/README.md @@ -0,0 +1,8 @@ +# Jupyter Notebook Deep Learning Stack + +GitHub Actions in the project builds and pushes this image to the Registry. + +Please visit the project documentation site for help to use and contribute to this image and others. + +- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html) +- [Selecting an Image :: Core Stacks :: jupyter/pytorch-cuda-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-pytorch-cuda-notebook) diff --git a/images/pytorch-cuda11-notebook/.dockerignore b/images/pytorch-cuda11-notebook/.dockerignore new file mode 100644 index 0000000000..9dea340f35 --- /dev/null +++ b/images/pytorch-cuda11-notebook/.dockerignore @@ -0,0 +1,2 @@ +# Documentation +README.md diff --git a/images/pytorch-cuda11-notebook/Dockerfile b/images/pytorch-cuda11-notebook/Dockerfile new file mode 100644 index 0000000000..01a52e7ff6 --- /dev/null +++ b/images/pytorch-cuda11-notebook/Dockerfile @@ -0,0 +1,21 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook +FROM $BASE_CONTAINER + +LABEL maintainer="Jupyter Project " + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install PyTorch with pip (https://pytorch.org/get-started/locally/) +# hadolint ignore=DL3013 +RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu118' \ + 'torch' \ + 'torchvision' \ + 'torchaudio' && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/images/pytorch-cuda11-notebook/README.md b/images/pytorch-cuda11-notebook/README.md new file mode 100644 index 0000000000..67e44af0f1 --- /dev/null +++ b/images/pytorch-cuda11-notebook/README.md @@ -0,0 +1,8 @@ +# Jupyter Notebook Deep Learning Stack + +GitHub Actions in the project builds and pushes this image to the Registry. + +Please visit the project documentation site for help to use and contribute to this image and others. + +- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html) +- [Selecting an Image :: Core Stacks :: jupyter/pytorch-cuda11-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-pytorch-cuda11-notebook) diff --git a/images/pytorch-notebook/Dockerfile b/images/pytorch-notebook/Dockerfile index f1a5c5400e..922e638979 100644 --- a/images/pytorch-notebook/Dockerfile +++ b/images/pytorch-notebook/Dockerfile @@ -11,7 +11,7 @@ LABEL maintainer="Jupyter Project " # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# Install PyTorch with pip +# Install PyTorch with pip (https://pytorch.org/get-started/locally/) # hadolint ignore=DL3013 RUN pip install --no-cache-dir --index-url 'https://download.pytorch.org/whl/cpu' \ 'torch' \ diff --git a/tagging/images_hierarchy.py b/tagging/images_hierarchy.py index 8c3e3fd384..ac7b70b2cf 100644 --- a/tagging/images_hierarchy.py +++ b/tagging/images_hierarchy.py @@ -75,6 +75,12 @@ class ImageDescription: "pytorch-notebook": ImageDescription( parent_image="scipy-notebook", taggers=[PytorchVersionTagger()] ), + "pytorch-cuda-notebook": ImageDescription( + parent_image="scipy-notebook", taggers=[PytorchVersionTagger()] + ), + "pytorch-cuda11-notebook": ImageDescription( + parent_image="scipy-notebook", taggers=[PytorchVersionTagger()] + ), "datascience-notebook": ImageDescription( parent_image="scipy-notebook", taggers=[RVersionTagger(), JuliaVersionTagger()], diff --git a/tests/images_hierarchy.py b/tests/images_hierarchy.py index 193ec2708a..e83ff13293 100644 --- a/tests/images_hierarchy.py +++ b/tests/images_hierarchy.py @@ -16,6 +16,8 @@ "julia-notebook": "minimal-notebook", "tensorflow-notebook": "scipy-notebook", "pytorch-notebook": "scipy-notebook", + "pytorch-cuda-notebook": "scipy-notebook", + "pytorch-cuda11-notebook": "scipy-notebook", "datascience-notebook": "scipy-notebook", "pyspark-notebook": "scipy-notebook", "all-spark-notebook": "pyspark-notebook", diff --git a/tests/pytorch-cuda-notebook/units/unit_pytorch_cuda.py b/tests/pytorch-cuda-notebook/units/unit_pytorch_cuda.py new file mode 100644 index 0000000000..1b739a5924 --- /dev/null +++ b/tests/pytorch-cuda-notebook/units/unit_pytorch_cuda.py @@ -0,0 +1,5 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import torch + +print(torch.tensor([[1.0, 4.0, 7.0], [4.0, 9.0, 11.0]])) diff --git a/tests/pytorch-cuda11-notebook/units/unit_pytorch_cuda11.py b/tests/pytorch-cuda11-notebook/units/unit_pytorch_cuda11.py new file mode 100644 index 0000000000..1b739a5924 --- /dev/null +++ b/tests/pytorch-cuda11-notebook/units/unit_pytorch_cuda11.py @@ -0,0 +1,5 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import torch + +print(torch.tensor([[1.0, 4.0, 7.0], [4.0, 9.0, 11.0]])) From 6198575d0fe318af727c7e89bd1424fb65fdfdb1 Mon Sep 17 00:00:00 2001 From: Johanna Reiml Date: Mon, 5 Feb 2024 12:46:27 +0100 Subject: [PATCH 02/19] feat: build with variant tag --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 - .github/ISSUE_TEMPLATE/feature_request.yml | 2 - .github/actions/load-image/action.yml | 7 +- .../docker-build-test-upload-all.yml | 244 ++++++++++ .../workflows/docker-build-test-upload.yml | 45 +- .github/workflows/docker-merge-tags.yml | 11 +- .github/workflows/docker-tag-push.yml | 11 +- .github/workflows/docker.yml | 436 ++---------------- .github/workflows/registry-move.yml | 5 +- .github/workflows/registry-overviews.yml | 2 - Makefile | 13 +- docs/using/selecting.md | 24 +- images/pytorch-cuda-notebook/.dockerignore | 2 - images/pytorch-cuda-notebook/README.md | 8 - images/pytorch-cuda11-notebook/.dockerignore | 2 - images/pytorch-cuda11-notebook/README.md | 8 - .../cuda}/Dockerfile | 0 .../cuda11}/Dockerfile | 0 tagging/apply_tags.py | 18 +- tagging/get_prefix.py | 25 + tagging/images_hierarchy.py | 6 - tagging/merge_tags.py | 18 +- tagging/write_manifest.py | 13 +- tagging/write_tags_file.py | 21 +- tests/images_hierarchy.py | 2 - .../units/unit_pytorch_cuda.py | 5 - .../units/unit_pytorch_cuda11.py | 5 - 27 files changed, 437 insertions(+), 498 deletions(-) create mode 100644 .github/workflows/docker-build-test-upload-all.yml delete mode 100644 images/pytorch-cuda-notebook/.dockerignore delete mode 100644 images/pytorch-cuda-notebook/README.md delete mode 100644 images/pytorch-cuda11-notebook/.dockerignore delete mode 100644 images/pytorch-cuda11-notebook/README.md rename images/{pytorch-cuda-notebook => pytorch-notebook/cuda}/Dockerfile (100%) rename images/{pytorch-cuda11-notebook => pytorch-notebook/cuda11}/Dockerfile (100%) create mode 100644 tagging/get_prefix.py delete mode 100644 tests/pytorch-cuda-notebook/units/unit_pytorch_cuda.py delete mode 100644 tests/pytorch-cuda11-notebook/units/unit_pytorch_cuda11.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index e92f69e336..064028d4ca 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,8 +27,6 @@ body: - minimal-notebook - pyspark-notebook - pytorch-notebook - - pytorch-cuda-notebook - - pytorch-cuda11-notebook - r-notebook - scipy-notebook - tensorflow-notebook diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index c525a43420..f0da40624e 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -25,8 +25,6 @@ body: - minimal-notebook - pyspark-notebook - pytorch-notebook - - pytorch-cuda-notebook - - pytorch-cuda11-notebook - r-notebook - scipy-notebook - tensorflow-notebook diff --git a/.github/actions/load-image/action.yml b/.github/actions/load-image/action.yml index cbf5a8a5eb..cf53e4a6f6 100644 --- a/.github/actions/load-image/action.yml +++ b/.github/actions/load-image/action.yml @@ -8,6 +8,9 @@ inputs: platform: description: Image platform required: true + variant: + description: Variant tag prefix + required: true runs: using: composite @@ -15,10 +18,10 @@ runs: - name: Download built image ๐Ÿ“ฅ uses: actions/download-artifact@v4 with: - name: ${{ inputs.image }}-${{ inputs.platform }} + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }} path: /tmp/jupyter/images/ - name: Load downloaded image to docker ๐Ÿ“ฅ run: | - zstd --uncompress --stdout --rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst | docker load + zstd --uncompress --stdout --rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst | docker load docker image ls --all shell: bash diff --git a/.github/workflows/docker-build-test-upload-all.yml b/.github/workflows/docker-build-test-upload-all.yml new file mode 100644 index 0000000000..6d0a6cacfc --- /dev/null +++ b/.github/workflows/docker-build-test-upload-all.yml @@ -0,0 +1,244 @@ +name: Build, test, and push Docker Images + +env: + REGISTRY: quay.io + OWNER: ${{ github.repository_owner }} + +on: + workflow_call: + inputs: + platform: + description: Image platform + required: true + type: string + runs-on-fast: + description: GitHub Actions Runner image (fast) + required: true + type: string + runs-on-slow: + description: GitHub Actions Runner image (slow) + required: true + type: string + secrets: + REGISTRY_USERNAME: + required: true + REGISTRY_TOKEN: + required: true + +jobs: + foundation: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: "" + parent-variant: default + image: docker-stacks-foundation + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-fast }} + + base: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: docker-stacks-foundation + parent-variant: default + image: base-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-fast }} + needs: [foundation] + + minimal: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: base-notebook + parent-variant: default + image: minimal-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-fast }} + needs: [base] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + scipy: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + parent-variant: default + image: scipy-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-fast }} + needs: [minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + r: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + parent-variant: default + image: r-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-slow }} + needs: [minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + julia: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + parent-variant: default + image: julia-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-slow }} + needs: [minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + tensorflow: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + parent-variant: default + image: tensorflow-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-slow }} + needs: [scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + pytorch: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + parent-variant: default + image: pytorch-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-slow }} + needs: [scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + pytorch-cuda: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + parent-variant: default + image: pytorch-notebook + variant: cuda + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-slow }} + needs: [scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + pytorch-cuda11: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + parent-variant: default + image: pytorch-notebook + variant: cuda11 + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-slow }} + needs: [scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + datascience: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + parent-variant: default + image: datascience-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-slow }} + needs: [scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + pyspark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + parent-variant: default + image: pyspark-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-fast }} + needs: [scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + all-spark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: pyspark-notebook + parent-variant: default + image: all-spark-notebook + variant: default + platform: ${{ inputs.platform }} + runs-on: ${{ inputs.runs-on-fast }} + needs: [pyspark] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + tag-push: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: ${{ inputs.platform }} + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + { image: minimal-notebook, variant: default }, + { image: scipy-notebook, variant: default }, + { image: r-notebook, variant: default }, + { image: julia-notebook, variant: default }, + { image: tensorflow-notebook, variant: default }, + { image: pytorch-notebook, variant: default }, + { image: pytorch-notebook, variant: cuda }, + { image: pytorch-notebook, variant: cuda11 }, + { image: datascience-notebook, variant: default }, + { image: pyspark-notebook, variant: default }, + { image: all-spark-notebook, variant: default }, + ] + needs: + [ + foundation, + base, + minimal, + scipy, + r, + julia, + tensorflow, + pytorch, + pytorch-cuda, + pytorch-cuda11, + datascience, + pyspark, + all-spark, + ] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + tag-push-fast: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: ${{ inputs.platform }} + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + ] + needs: [foundation, base] + if: ${{ contains(github.event.pull_request.title, '[FAST_BUILD]') }} diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index 64f04ae722..40f170acb1 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -11,10 +11,18 @@ on: description: Parent image name required: true type: string + parent-variant: + description: Parent variant tag prefix + required: true + type: string image: description: Image name required: true type: string + variant: + description: Variant tag prefix + required: true + type: string platform: description: Image platform required: true @@ -31,8 +39,8 @@ jobs: steps: # Image with CUDA needs extra disk space - name: Free disk space ๐Ÿงน + if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64' uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be - if: contains(inputs.image, 'cuda') && inputs.platform == 'x86_64' with: tool-cache: false android: true @@ -65,13 +73,15 @@ jobs: with: image: ${{ inputs.parent-image }} platform: ${{ inputs.platform }} + variant: ${{ inputs.parent-variant }} - name: Pull ubuntu:22.04 image ๐Ÿ“ฅ if: inputs.parent-image == '' run: docker pull ubuntu:22.04 shell: bash - - name: Build image ๐Ÿ›  + - name: Build image (default) ๐Ÿ›  + if: inputs.variant == 'default' run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }} env: DOCKER_BUILDKIT: 1 @@ -79,45 +89,54 @@ jobs: BUILDKIT_PROGRESS: plain shell: bash + - name: Build image (variant) ๐Ÿ›  + if: inputs.variant != 'default' + run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/${{ inputs.variant }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }} + env: + DOCKER_BUILDKIT: 1 + # Full logs for CI build + BUILDKIT_PROGRESS: plain + shell: bash + - name: Run tests โœ… run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} shell: bash - name: Write tags file ๐Ÿท run: | - python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} + python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }} shell: bash - name: Upload tags file ๐Ÿ’พ uses: actions/upload-artifact@v4 with: - name: ${{ inputs.image }}-${{ inputs.platform }}-tags - path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.image }}.txt + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-tags + path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}.txt retention-days: 3 - name: Write manifest and build history file ๐Ÿท - run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} + run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }} shell: bash - name: Upload manifest file ๐Ÿ’พ uses: actions/upload-artifact@v4 with: - name: ${{ inputs.image }}-${{ inputs.platform }}-manifest - path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-manifest + path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-*.md retention-days: 3 - name: Upload build history line ๐Ÿ’พ uses: actions/upload-artifact@v4 with: - name: ${{ inputs.image }}-${{ inputs.platform }}-history_line - path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-history_line + path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-*.txt retention-days: 3 - name: Save image as a tar for later use ๐Ÿ’พ run: | mkdir -p /tmp/jupyter/images/ - docker save ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} | zstd > /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst + docker save ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} | zstd > /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst shell: bash - name: Upload image as artifact ๐Ÿ’พ uses: actions/upload-artifact@v4 with: - name: ${{ inputs.image }}-${{ inputs.platform }} - path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }} + path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst retention-days: 3 diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml index 038dbff145..1245a4151f 100644 --- a/.github/workflows/docker-merge-tags.yml +++ b/.github/workflows/docker-merge-tags.yml @@ -7,6 +7,10 @@ env: on: workflow_call: inputs: + variant: + description: Variant tag prefix + required: true + type: string image: description: Image name required: true @@ -30,13 +34,14 @@ jobs: - name: Download x86_64 tags file ๐Ÿ“ฅ uses: actions/download-artifact@v4 with: - name: ${{ inputs.image }}-x86_64-tags + name: ${{ inputs.image }}-x86_64-${{ inputs.variant }}-tags path: /tmp/jupyter/tags/ - name: Download aarch64 tags file ๐Ÿ“ฅ uses: actions/download-artifact@v4 with: - name: ${{ inputs.image }}-aarch64-tags + name: ${{ inputs.image }}-aarch64-${{ inputs.variant }}-tags path: /tmp/jupyter/tags/ + if: github.repository_owner == 'jupyter' # Docker might be stuck when pulling images # https://github.com/docker/for-mac/issues/2083 @@ -57,5 +62,5 @@ jobs: - name: Merge tags for the images ๐Ÿ”€ if: env.PUSH_TO_REGISTRY == 'true' - run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ + run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --variant ${{ inputs.variant }} shell: bash diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-push.yml index 464d357e78..c0093e69e4 100644 --- a/.github/workflows/docker-tag-push.yml +++ b/.github/workflows/docker-tag-push.yml @@ -16,6 +16,10 @@ on: description: Image platform required: true type: string + variant: + description: Variant tag prefix + required: true + type: string secrets: REGISTRY_USERNAME: required: true @@ -29,8 +33,8 @@ jobs: steps: # Image with CUDA needs extra disk space - name: Free disk space ๐Ÿงน + if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64' uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be - if: contains(inputs.image, 'cuda') && inputs.platform == 'x86_64' with: tool-cache: false android: true @@ -49,6 +53,7 @@ jobs: with: image: ${{ inputs.image }} platform: ${{ inputs.platform }} + variant: ${{ inputs.variant }} - name: Login to Registry ๐Ÿ” if: env.PUSH_TO_REGISTRY == 'true' @@ -61,10 +66,10 @@ jobs: - name: Download tags file ๐Ÿ“ฅ uses: actions/download-artifact@v4 with: - name: ${{ inputs.image }}-${{ inputs.platform }}-tags + name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-tags path: /tmp/jupyter/tags/ - name: Apply tags to the loaded image ๐Ÿท - run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} + run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --variant ${{ inputs.variant }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} # This step is needed to prevent pushing non-multi-arch "latest" tag - name: Remove the "latest" tag from the image ๐Ÿ—‘๏ธ run: docker image rmi ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }}:latest diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 388bec7cff..122542745b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: Build, test, and push Docker Images +name: Build, merge tags, update wiki, test recipes # [FAST_BUILD] in the PR title makes this workflow only build # the `jupyter/docker-stacks-foundation` and `jupyter/base-notebook` images @@ -12,6 +12,7 @@ on: pull_request: paths: - ".github/workflows/docker.yml" + - ".github/workflows/docker-build-test-upload-all.yml" # We use local reusable workflows to make architecture clean and simple # https://docs.github.com/en/actions/using-workflows/reusing-workflows - ".github/workflows/docker-build-test-upload.yml" @@ -36,6 +37,7 @@ on: - main paths: - ".github/workflows/docker.yml" + - ".github/workflows/docker-build-test-upload-all.yml" - ".github/workflows/docker-build-test-upload.yml" - ".github/workflows/docker-merge-tags.yml" - ".github/workflows/docker-tag-push.yml" @@ -60,426 +62,86 @@ concurrency: cancel-in-progress: true jobs: - aarch64-foundation: - uses: ./.github/workflows/docker-build-test-upload.yml + build-aarch64: + uses: ./.github/workflows/docker-build-test-upload-all.yml with: - parent-image: "" - image: docker-stacks-foundation platform: aarch64 - runs-on: ARM64_FAST - if: github.repository_owner == 'jupyter' - - x86_64-foundation: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: "" - image: docker-stacks-foundation - platform: x86_64 - runs-on: ubuntu-latest - - aarch64-base: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: docker-stacks-foundation - image: base-notebook - platform: aarch64 - runs-on: ARM64_FAST - needs: [aarch64-foundation] - if: github.repository_owner == 'jupyter' - - x86_64-base: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: docker-stacks-foundation - image: base-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-foundation] - - aarch64-minimal: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: base-notebook - image: minimal-notebook - platform: aarch64 - runs-on: ARM64_FAST - needs: [aarch64-base] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-minimal: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: base-notebook - image: minimal-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-base] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-scipy: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - image: scipy-notebook - platform: aarch64 - runs-on: ARM64_FAST - needs: [aarch64-minimal] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-scipy: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - image: scipy-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-minimal] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-r: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - image: r-notebook - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-minimal] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-r: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - image: r-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-minimal] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-julia: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - image: julia-notebook - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-minimal] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-julia: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - image: julia-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-minimal] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-tensorflow: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: tensorflow-notebook - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-tensorflow: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: tensorflow-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-pytorch: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pytorch-notebook - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-pytorch: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pytorch-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-pytorch-cuda: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pytorch-cuda-notebook - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-pytorch-cuda: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pytorch-cuda-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-pytorch-cuda11: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pytorch-cuda11-notebook - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-pytorch-cuda11: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pytorch-cuda11-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-datascience: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: datascience-notebook - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-datascience: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: datascience-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-pyspark: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pyspark-notebook - platform: aarch64 - runs-on: ARM64_FAST - needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-pyspark: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pyspark-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-all-spark: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: pyspark-notebook - image: all-spark-notebook - platform: aarch64 - runs-on: ARM64_FAST - needs: [aarch64-pyspark] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-all-spark: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: pyspark-notebook - image: all-spark-notebook - platform: x86_64 - runs-on: ubuntu-latest - needs: [x86_64-pyspark] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - aarch64-images-tag-push: - uses: ./.github/workflows/docker-tag-push.yml - with: - platform: aarch64 - image: ${{ matrix.image }} - secrets: - REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} - strategy: - matrix: - image: - [ - docker-stacks-foundation, - base-notebook, - minimal-notebook, - scipy-notebook, - r-notebook, - julia-notebook, - tensorflow-notebook, - pytorch-notebook, - pytorch-cuda-notebook, - pytorch-cuda11-notebook, - datascience-notebook, - pyspark-notebook, - all-spark-notebook, - ] - needs: - [ - aarch64-foundation, - aarch64-base, - aarch64-minimal, - aarch64-scipy, - aarch64-r, - aarch64-julia, - aarch64-tensorflow, - aarch64-pytorch, - aarch64-datascience, - aarch64-pyspark, - aarch64-all-spark, - ] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - - aarch64-images-tag-push-fast: - uses: ./.github/workflows/docker-tag-push.yml - with: - platform: aarch64 - image: ${{ matrix.image }} - secrets: - REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} - strategy: - matrix: - image: [docker-stacks-foundation, base-notebook] - needs: [aarch64-foundation, aarch64-base] - if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') - - x86_64-images-tag-push: - uses: ./.github/workflows/docker-tag-push.yml - with: - platform: x86_64 - image: ${{ matrix.image }} + runs-on-fast: ARM64_FAST + runs-on-slow: ARM64_SLOW secrets: REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} - strategy: - matrix: - image: - [ - docker-stacks-foundation, - base-notebook, - minimal-notebook, - scipy-notebook, - r-notebook, - julia-notebook, - tensorflow-notebook, - pytorch-notebook, - pytorch-cuda-notebook, - pytorch-cuda11-notebook, - datascience-notebook, - pyspark-notebook, - all-spark-notebook, - ] - needs: - [ - x86_64-foundation, - x86_64-base, - x86_64-minimal, - x86_64-scipy, - x86_64-r, - x86_64-julia, - x86_64-tensorflow, - x86_64-pytorch, - x86_64-datascience, - x86_64-pyspark, - x86_64-all-spark, - ] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + if: github.repository_owner == 'jupyter' - x86_64-images-tag-push-fast: - uses: ./.github/workflows/docker-tag-push.yml + build-x86_64: + uses: ./.github/workflows/docker-build-test-upload-all.yml with: platform: x86_64 - image: ${{ matrix.image }} + runs-on-fast: ubuntu-latest + runs-on-slow: ubuntu-latest secrets: REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} - strategy: - matrix: - image: [docker-stacks-foundation, base-notebook] - needs: [x86_64-foundation, x86_64-base] - if: contains(github.event.pull_request.title, '[FAST_BUILD]') merge-tags: uses: ./.github/workflows/docker-merge-tags.yml with: - image: ${{ matrix.image }} + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} secrets: REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} strategy: matrix: - image: + image-variant: [ - docker-stacks-foundation, - base-notebook, - minimal-notebook, - scipy-notebook, - r-notebook, - julia-notebook, - tensorflow-notebook, - pytorch-notebook, - pytorch-cuda-notebook, - pytorch-cuda11-notebook, - datascience-notebook, - pyspark-notebook, - all-spark-notebook, + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + { image: minimal-notebook, variant: default }, + { image: scipy-notebook, variant: default }, + { image: r-notebook, variant: default }, + { image: julia-notebook, variant: default }, + { image: tensorflow-notebook, variant: default }, + { image: pytorch-notebook, variant: default }, + { image: pytorch-notebook, variant: cuda }, + { image: pytorch-notebook, variant: cuda11 }, + { image: datascience-notebook, variant: default }, + { image: pyspark-notebook, variant: default }, + { image: all-spark-notebook, variant: default }, ] - needs: [aarch64-images-tag-push, x86_64-images-tag-push] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + needs: [build-aarch64, build-x86_64] + if: | + always() && + needs.build-x86_64.result == 'success' && + (needs.build-aarch64.result == 'success' || needs.build-aarch64.result == 'skipped') && + !contains(github.event.pull_request.title, '[FAST_BUILD]') merge-tags-fast: uses: ./.github/workflows/docker-merge-tags.yml with: - image: ${{ matrix.image }} + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} secrets: REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} strategy: matrix: - image: [docker-stacks-foundation, base-notebook] - needs: [aarch64-images-tag-push-fast, x86_64-images-tag-push-fast] - if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + ] + needs: [build-aarch64, build-x86_64] + if: | + always() && + needs.build-x86_64.result == 'success' && + (needs.build-aarch64.result == 'success' || needs.build-aarch64.result == 'skipped') && + contains(github.event.pull_request.title, '[FAST_BUILD]') wiki-update: uses: ./.github/workflows/docker-wiki-update.yml - needs: [aarch64-images-tag-push, x86_64-images-tag-push] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - permissions: - contents: write - - wiki-update-fast: - uses: ./.github/workflows/docker-wiki-update.yml - needs: [aarch64-images-tag-push-fast, x86_64-images-tag-push-fast] - if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') + needs: [build-aarch64, build-x86_64] + if: github.repository_owner == 'jupyter' permissions: contents: write diff --git a/.github/workflows/registry-move.yml b/.github/workflows/registry-move.yml index ab22fd16e3..09ef3d28a2 100644 --- a/.github/workflows/registry-move.yml +++ b/.github/workflows/registry-move.yml @@ -42,7 +42,8 @@ jobs: strategy: fail-fast: false matrix: - image: [ + image: + [ docker-stacks-foundation, base-notebook, minimal-notebook, @@ -51,8 +52,6 @@ jobs: julia-notebook, tensorflow-notebook, pytorch-notebook, - # pytorch-cuda-notebook, - # pytorch-cuda11-notebook, datascience-notebook, pyspark-notebook, all-spark-notebook, diff --git a/.github/workflows/registry-overviews.yml b/.github/workflows/registry-overviews.yml index 74662f4e2e..91a6b1f4bc 100644 --- a/.github/workflows/registry-overviews.yml +++ b/.github/workflows/registry-overviews.yml @@ -43,8 +43,6 @@ jobs: julia-notebook, tensorflow-notebook, pytorch-notebook, - pytorch-cuda-notebook, - pytorch-cuda11-notebook, datascience-notebook, pyspark-notebook, all-spark-notebook, diff --git a/Makefile b/Makefile index 7d3c57fdd2..9070209b2a 100644 --- a/Makefile +++ b/Makefile @@ -19,8 +19,6 @@ ALL_IMAGES:= \ scipy-notebook \ tensorflow-notebook \ pytorch-notebook \ - pytorch-cuda-notebook \ - pytorch-cuda11-notebook \ datascience-notebook \ pyspark-notebook \ all-spark-notebook @@ -38,8 +36,9 @@ help: build/%: DOCKER_BUILD_ARGS?= +build/%: ROOT_CONTAINER?=ubuntu:22.04 build/%: ## build the latest image for a stack using the system's architecture - docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg REGISTRY="$(REGISTRY)" --build-arg OWNER="$(OWNER)" + docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg REGISTRY="$(REGISTRY)" --build-arg OWNER="$(OWNER)" --build-arg ROOT_CONTAINER="$(ROOT_CONTAINER)" @echo -n "Built image size: " @docker images "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" --format "{{.Size}}" build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks @@ -68,11 +67,11 @@ linkcheck-docs: ## check broken links sphinx-build -W --keep-going --color -b linkcheck docs/ docs/_build/ - +hook/%: VARIANT?=default hook/%: ## run post-build hooks for an image - python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --registry "$(REGISTRY)" --owner "$(OWNER)" && \ - python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry "$(REGISTRY)" --owner "$(OWNER)" && \ - python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --registry "$(REGISTRY)" --owner "$(OWNER)" + python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --registry "$(REGISTRY)" --owner "$(OWNER)" --variant "$(VARIANT)" && \ + python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry "$(REGISTRY)" --owner "$(OWNER)" --variant "$(VARIANT)" && \ + python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --variant "$(VARIANT)" --registry "$(REGISTRY)" --owner "$(OWNER)" hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images diff --git a/docs/using/selecting.md b/docs/using/selecting.md index e7e8220012..8d951918a3 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -195,29 +195,7 @@ It contains: `jupyter/pytorch-notebook` includes popular Python deep learning libraries. - Everything in `jupyter/scipy-notebook` and its ancestor images -- [pytorch](https://pytorch.org/) machine learning library (only CPU) - -### jupyter/pytorch-cuda-notebook - -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/pytorch-cuda-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/pytorch-cuda-notebook/Dockerfile) | -[Quay.io image tags](https://quay.io/repository/jupyter/pytorch-cuda-notebook?tab=tags) - -`jupyter/pytorch-cuda-notebook` includes popular Python deep learning libraries. - -- Everything in `jupyter/scipy-notebook` and its ancestor images -- [pytorch](https://pytorch.org/) machine learning library (with CUDA 12.X) - -### jupyter/pytorch-cuda11-notebook - -[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/pytorch-cuda11-notebook) | -[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/pytorch-cuda11-notebook/Dockerfile) | -[Quay.io image tags](https://quay.io/repository/jupyter/pytorch-cuda11-notebook?tab=tags) - -`jupyter/pytorch-cuda11-notebook` includes popular Python deep learning libraries. - -- Everything in `jupyter/scipy-notebook` and its ancestor images -- [pytorch](https://pytorch.org/) machine learning library (with CUDA 11.8) +- [pytorch](https://pytorch.org/) machine learning library (optionally with CUDA) ### jupyter/datascience-notebook diff --git a/images/pytorch-cuda-notebook/.dockerignore b/images/pytorch-cuda-notebook/.dockerignore deleted file mode 100644 index 9dea340f35..0000000000 --- a/images/pytorch-cuda-notebook/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# Documentation -README.md diff --git a/images/pytorch-cuda-notebook/README.md b/images/pytorch-cuda-notebook/README.md deleted file mode 100644 index 1900d7cb5f..0000000000 --- a/images/pytorch-cuda-notebook/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Jupyter Notebook Deep Learning Stack - -GitHub Actions in the project builds and pushes this image to the Registry. - -Please visit the project documentation site for help to use and contribute to this image and others. - -- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html) -- [Selecting an Image :: Core Stacks :: jupyter/pytorch-cuda-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-pytorch-cuda-notebook) diff --git a/images/pytorch-cuda11-notebook/.dockerignore b/images/pytorch-cuda11-notebook/.dockerignore deleted file mode 100644 index 9dea340f35..0000000000 --- a/images/pytorch-cuda11-notebook/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# Documentation -README.md diff --git a/images/pytorch-cuda11-notebook/README.md b/images/pytorch-cuda11-notebook/README.md deleted file mode 100644 index 67e44af0f1..0000000000 --- a/images/pytorch-cuda11-notebook/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Jupyter Notebook Deep Learning Stack - -GitHub Actions in the project builds and pushes this image to the Registry. - -Please visit the project documentation site for help to use and contribute to this image and others. - -- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html) -- [Selecting an Image :: Core Stacks :: jupyter/pytorch-cuda11-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-pytorch-cuda11-notebook) diff --git a/images/pytorch-cuda-notebook/Dockerfile b/images/pytorch-notebook/cuda/Dockerfile similarity index 100% rename from images/pytorch-cuda-notebook/Dockerfile rename to images/pytorch-notebook/cuda/Dockerfile diff --git a/images/pytorch-cuda11-notebook/Dockerfile b/images/pytorch-notebook/cuda11/Dockerfile similarity index 100% rename from images/pytorch-cuda11-notebook/Dockerfile rename to images/pytorch-notebook/cuda11/Dockerfile diff --git a/tagging/apply_tags.py b/tagging/apply_tags.py index 1c33a8fe1c..169a3db869 100755 --- a/tagging/apply_tags.py +++ b/tagging/apply_tags.py @@ -8,6 +8,7 @@ import plumbum from tagging.get_platform import unify_aarch64 +from tagging.get_prefix import get_file_prefix_for_platform docker = plumbum.local["docker"] @@ -20,14 +21,16 @@ def apply_tags( owner: str, tags_dir: Path, platform: str, + variant: str, ) -> None: """ Tags //:latest with the tags reported by all taggers for this image """ LOGGER.info(f"Tagging image: {short_image_name}") + file_prefix = get_file_prefix_for_platform(platform, variant) image = f"{registry}/{owner}/{short_image_name}:latest" - filename = f"{platform}-{short_image_name}.txt" + filename = f"{file_prefix}-{short_image_name}.txt" tags = (tags_dir / filename).read_text().splitlines() for tag in tags: @@ -69,9 +72,20 @@ def apply_tags( required=True, help="Owner of the image", ) + arg_parser.add_argument( + "--variant", + required=True, + type=str, + help="Variant tag prefix", + ) args = arg_parser.parse_args() args.platform = unify_aarch64(args.platform) apply_tags( - args.short_image_name, args.registry, args.owner, args.tags_dir, args.platform + args.short_image_name, + args.registry, + args.owner, + args.tags_dir, + args.platform, + args.variant, ) diff --git a/tagging/get_prefix.py b/tagging/get_prefix.py new file mode 100644 index 0000000000..f60c399beb --- /dev/null +++ b/tagging/get_prefix.py @@ -0,0 +1,25 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from tagging.get_platform import get_platform + +DEFAULT_VARIANT = "default" + + +def get_file_prefix_for_platform(platform: str, name: str) -> str: + return f"{platform}-{name}" + + +def get_tag_prefix_for_platform(platform: str, name: str) -> str: + if name == DEFAULT_VARIANT: + return platform + return f"{platform}-{name}" + + +def get_file_prefix(variant: str) -> str: + platform = get_platform() + return get_file_prefix_for_platform(platform, variant) + + +def get_tag_prefix(variant: str) -> str: + platform = get_platform() + return get_tag_prefix_for_platform(platform, variant) diff --git a/tagging/images_hierarchy.py b/tagging/images_hierarchy.py index ac7b70b2cf..8c3e3fd384 100644 --- a/tagging/images_hierarchy.py +++ b/tagging/images_hierarchy.py @@ -75,12 +75,6 @@ class ImageDescription: "pytorch-notebook": ImageDescription( parent_image="scipy-notebook", taggers=[PytorchVersionTagger()] ), - "pytorch-cuda-notebook": ImageDescription( - parent_image="scipy-notebook", taggers=[PytorchVersionTagger()] - ), - "pytorch-cuda11-notebook": ImageDescription( - parent_image="scipy-notebook", taggers=[PytorchVersionTagger()] - ), "datascience-notebook": ImageDescription( parent_image="scipy-notebook", taggers=[RVersionTagger(), JuliaVersionTagger()], diff --git a/tagging/merge_tags.py b/tagging/merge_tags.py index 885a482185..55b10e6736 100755 --- a/tagging/merge_tags.py +++ b/tagging/merge_tags.py @@ -8,6 +8,7 @@ import plumbum from tagging.get_platform import ALL_PLATFORMS +from tagging.get_prefix import get_file_prefix_for_platform, get_tag_prefix_for_platform docker = plumbum.local["docker"] @@ -16,6 +17,7 @@ def merge_tags( short_image_name: str, + variant: str, tags_dir: Path, ) -> None: """ @@ -26,9 +28,12 @@ def merge_tags( all_tags: set[str] = set() for platform in ALL_PLATFORMS: - filename = f"{platform}-{short_image_name}.txt" - tags = (tags_dir / filename).read_text().splitlines() - all_tags.update(tag.replace(platform + "-", "") for tag in tags) + file_prefix = get_file_prefix_for_platform(platform, variant) + filename = f"{file_prefix}-{short_image_name}.txt" + file_path = tags_dir / filename + if file_path.exists(): + tags = file_path.read_text().splitlines() + all_tags.update(tag.replace(platform + "-", "") for tag in tags) LOGGER.info(f"Got tags: {all_tags}") @@ -61,6 +66,11 @@ def merge_tags( required=True, help="Short image name", ) + arg_parser.add_argument( + "--variant", + required=True, + help="Variant tag prefix", + ) arg_parser.add_argument( "--tags-dir", required=True, @@ -69,4 +79,4 @@ def merge_tags( ) args = arg_parser.parse_args() - merge_tags(args.short_image_name, args.tags_dir) + merge_tags(args.short_image_name, args.variant, args.tags_dir) diff --git a/tagging/write_manifest.py b/tagging/write_manifest.py index c4605fd7a8..cdf51a77d4 100755 --- a/tagging/write_manifest.py +++ b/tagging/write_manifest.py @@ -9,7 +9,7 @@ from docker.models.containers import Container from tagging.docker_runner import DockerRunner -from tagging.get_platform import get_platform +from tagging.get_prefix import get_file_prefix, get_tag_prefix from tagging.get_taggers_and_manifests import get_taggers_and_manifests from tagging.git_helper import GitHelper from tagging.manifests import ManifestHeader, ManifestInterface @@ -73,6 +73,7 @@ def write_manifest( short_image_name: str, registry: str, owner: str, + variant: str, hist_lines_dir: Path, manifests_dir: Path, ) -> None: @@ -81,12 +82,12 @@ def write_manifest( image = f"{registry}/{owner}/{short_image_name}:latest" - file_prefix = get_platform() + file_prefix = get_file_prefix(variant) commit_hash_tag = GitHelper.commit_hash_tag() filename = f"{file_prefix}-{short_image_name}-{commit_hash_tag}" with DockerRunner(image) as container: - tags_prefix = get_platform() + tags_prefix = get_tag_prefix(variant) all_tags = [ tags_prefix + "-" + tagger.tag_value(container) for tagger in taggers ] @@ -137,6 +138,11 @@ def write_manifest( required=True, help="Owner of the image", ) + arg_parser.add_argument( + "--variant", + required=True, + help="Variant tag prefix", + ) args = arg_parser.parse_args() LOGGER.info(f"Current build timestamp: {BUILD_TIMESTAMP}") @@ -145,6 +151,7 @@ def write_manifest( args.short_image_name, args.registry, args.owner, + args.variant, args.hist_lines_dir, args.manifests_dir, ) diff --git a/tagging/write_tags_file.py b/tagging/write_tags_file.py index 880fec40da..21c127cd21 100755 --- a/tagging/write_tags_file.py +++ b/tagging/write_tags_file.py @@ -6,7 +6,7 @@ from pathlib import Path from tagging.docker_runner import DockerRunner -from tagging.get_platform import get_platform +from tagging.get_prefix import get_file_prefix, get_tag_prefix from tagging.get_taggers_and_manifests import get_taggers_and_manifests LOGGER = logging.getLogger(__name__) @@ -16,6 +16,7 @@ def write_tags_file( short_image_name: str, registry: str, owner: str, + variant: str, tags_dir: Path, ) -> None: """ @@ -25,9 +26,10 @@ def write_tags_file( taggers, _ = get_taggers_and_manifests(short_image_name) image = f"{registry}/{owner}/{short_image_name}:latest" - tags_prefix = get_platform() - filename = f"{tags_prefix}-{short_image_name}.txt" + file_prefix = get_file_prefix(variant) + filename = f"{file_prefix}-{short_image_name}.txt" + tags_prefix = get_tag_prefix(variant) tags = [f"{registry}/{owner}/{short_image_name}:{tags_prefix}-latest"] with DockerRunner(image) as container: for tagger in taggers: @@ -70,6 +72,17 @@ def write_tags_file( required=True, help="Owner of the image", ) + arg_parser.add_argument( + "--variant", + required=True, + help="Variant tag prefix", + ) args = arg_parser.parse_args() - write_tags_file(args.short_image_name, args.registry, args.owner, args.tags_dir) + write_tags_file( + args.short_image_name, + args.registry, + args.owner, + args.variant, + args.tags_dir, + ) diff --git a/tests/images_hierarchy.py b/tests/images_hierarchy.py index e83ff13293..193ec2708a 100644 --- a/tests/images_hierarchy.py +++ b/tests/images_hierarchy.py @@ -16,8 +16,6 @@ "julia-notebook": "minimal-notebook", "tensorflow-notebook": "scipy-notebook", "pytorch-notebook": "scipy-notebook", - "pytorch-cuda-notebook": "scipy-notebook", - "pytorch-cuda11-notebook": "scipy-notebook", "datascience-notebook": "scipy-notebook", "pyspark-notebook": "scipy-notebook", "all-spark-notebook": "pyspark-notebook", diff --git a/tests/pytorch-cuda-notebook/units/unit_pytorch_cuda.py b/tests/pytorch-cuda-notebook/units/unit_pytorch_cuda.py deleted file mode 100644 index 1b739a5924..0000000000 --- a/tests/pytorch-cuda-notebook/units/unit_pytorch_cuda.py +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import torch - -print(torch.tensor([[1.0, 4.0, 7.0], [4.0, 9.0, 11.0]])) diff --git a/tests/pytorch-cuda11-notebook/units/unit_pytorch_cuda11.py b/tests/pytorch-cuda11-notebook/units/unit_pytorch_cuda11.py deleted file mode 100644 index 1b739a5924..0000000000 --- a/tests/pytorch-cuda11-notebook/units/unit_pytorch_cuda11.py +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import torch - -print(torch.tensor([[1.0, 4.0, 7.0], [4.0, 9.0, 11.0]])) From 95e4bb10ef40d03cf06d4cc5153f6fd4cff01e41 Mon Sep 17 00:00:00 2001 From: Johanna Reiml Date: Tue, 6 Feb 2024 12:05:54 +0100 Subject: [PATCH 03/19] style: remove unused import --- tagging/merge_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tagging/merge_tags.py b/tagging/merge_tags.py index 55b10e6736..06b18e7019 100755 --- a/tagging/merge_tags.py +++ b/tagging/merge_tags.py @@ -8,7 +8,7 @@ import plumbum from tagging.get_platform import ALL_PLATFORMS -from tagging.get_prefix import get_file_prefix_for_platform, get_tag_prefix_for_platform +from tagging.get_prefix import get_file_prefix_for_platform docker = plumbum.local["docker"] From ba57a88c7b2c045eb4260ec95464cb3ce796d1f9 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 20:45:54 +0100 Subject: [PATCH 04/19] refactor: rename get_prefix params (cherry picked from commit 12b50af258c2f331d4100fb63fd41ad1a30acb1d) --- tagging/get_prefix.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tagging/get_prefix.py b/tagging/get_prefix.py index f60c399beb..6e7c718576 100644 --- a/tagging/get_prefix.py +++ b/tagging/get_prefix.py @@ -5,14 +5,14 @@ DEFAULT_VARIANT = "default" -def get_file_prefix_for_platform(platform: str, name: str) -> str: - return f"{platform}-{name}" +def get_file_prefix_for_platform(platform: str, variant: str) -> str: + return f"{platform}-{variant}" -def get_tag_prefix_for_platform(platform: str, name: str) -> str: - if name == DEFAULT_VARIANT: +def get_tag_prefix_for_platform(platform: str, variant: str) -> str: + if variant == DEFAULT_VARIANT: return platform - return f"{platform}-{name}" + return f"{platform}-{variant}" def get_file_prefix(variant: str) -> str: From 2ead783f39024772e191fe99bc10acbf4605aadb Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 20:46:30 +0100 Subject: [PATCH 05/19] revert: drop ROOT_CONTAINER addition from Makefile (cherry picked from commit f42314513df2855957a05c6ba0c748d2df26d7b0) --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9070209b2a..abd5ec4e3f 100644 --- a/Makefile +++ b/Makefile @@ -36,9 +36,8 @@ help: build/%: DOCKER_BUILD_ARGS?= -build/%: ROOT_CONTAINER?=ubuntu:22.04 build/%: ## build the latest image for a stack using the system's architecture - docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg REGISTRY="$(REGISTRY)" --build-arg OWNER="$(OWNER)" --build-arg ROOT_CONTAINER="$(ROOT_CONTAINER)" + docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg REGISTRY="$(REGISTRY)" --build-arg OWNER="$(OWNER)" @echo -n "Built image size: " @docker images "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" --format "{{.Size}}" build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks From f320f0734437843bc5595920e4533af2b2b254cc Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 20:48:13 +0100 Subject: [PATCH 06/19] style: use consistent three empty lines in Makefile (cherry picked from commit 446b45aab37a37720462b5df305ce96b139cf67a) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index abd5ec4e3f..52400cc87a 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ linkcheck-docs: ## check broken links sphinx-build -W --keep-going --color -b linkcheck docs/ docs/_build/ + hook/%: VARIANT?=default hook/%: ## run post-build hooks for an image python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --registry "$(REGISTRY)" --owner "$(OWNER)" --variant "$(VARIANT)" && \ From 02b36090f0d9051a33c02c2e811fde174a546a78 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 21:15:22 +0100 Subject: [PATCH 07/19] refactor: add default value for parent-image (cherry picked from commit 32955cec99c7202f0ce50647dfc61ec98f57f741) --- .github/workflows/docker-build-test-upload.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index 40f170acb1..b65d5ee45c 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -13,8 +13,9 @@ on: type: string parent-variant: description: Parent variant tag prefix - required: true + required: false type: string + default: default image: description: Image name required: true From b7b5cfa43c7944c553cac560aafba7b7fd56d41a Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 21:14:54 +0100 Subject: [PATCH 08/19] revert: use original workflow structure (cherry picked from commit 68c6744513636ec93d14f9bd0bbd123907efd13b) --- .../docker-build-test-upload-all.yml | 244 ---------- .github/workflows/docker.yml | 434 ++++++++++++++++-- 2 files changed, 408 insertions(+), 270 deletions(-) delete mode 100644 .github/workflows/docker-build-test-upload-all.yml diff --git a/.github/workflows/docker-build-test-upload-all.yml b/.github/workflows/docker-build-test-upload-all.yml deleted file mode 100644 index 6d0a6cacfc..0000000000 --- a/.github/workflows/docker-build-test-upload-all.yml +++ /dev/null @@ -1,244 +0,0 @@ -name: Build, test, and push Docker Images - -env: - REGISTRY: quay.io - OWNER: ${{ github.repository_owner }} - -on: - workflow_call: - inputs: - platform: - description: Image platform - required: true - type: string - runs-on-fast: - description: GitHub Actions Runner image (fast) - required: true - type: string - runs-on-slow: - description: GitHub Actions Runner image (slow) - required: true - type: string - secrets: - REGISTRY_USERNAME: - required: true - REGISTRY_TOKEN: - required: true - -jobs: - foundation: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: "" - parent-variant: default - image: docker-stacks-foundation - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-fast }} - - base: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: docker-stacks-foundation - parent-variant: default - image: base-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-fast }} - needs: [foundation] - - minimal: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: base-notebook - parent-variant: default - image: minimal-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-fast }} - needs: [base] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - scipy: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - parent-variant: default - image: scipy-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-fast }} - needs: [minimal] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - r: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - parent-variant: default - image: r-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-slow }} - needs: [minimal] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - julia: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: minimal-notebook - parent-variant: default - image: julia-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-slow }} - needs: [minimal] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - tensorflow: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - parent-variant: default - image: tensorflow-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-slow }} - needs: [scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - pytorch: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - parent-variant: default - image: pytorch-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-slow }} - needs: [scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - pytorch-cuda: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - parent-variant: default - image: pytorch-notebook - variant: cuda - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-slow }} - needs: [scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - pytorch-cuda11: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - parent-variant: default - image: pytorch-notebook - variant: cuda11 - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-slow }} - needs: [scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - datascience: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - parent-variant: default - image: datascience-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-slow }} - needs: [scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - pyspark: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - parent-variant: default - image: pyspark-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-fast }} - needs: [scipy] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - all-spark: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: pyspark-notebook - parent-variant: default - image: all-spark-notebook - variant: default - platform: ${{ inputs.platform }} - runs-on: ${{ inputs.runs-on-fast }} - needs: [pyspark] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - tag-push: - uses: ./.github/workflows/docker-tag-push.yml - with: - platform: ${{ inputs.platform }} - image: ${{ matrix.image-variant.image }} - variant: ${{ matrix.image-variant.variant }} - secrets: - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - strategy: - matrix: - image-variant: - [ - { image: docker-stacks-foundation, variant: default }, - { image: base-notebook, variant: default }, - { image: minimal-notebook, variant: default }, - { image: scipy-notebook, variant: default }, - { image: r-notebook, variant: default }, - { image: julia-notebook, variant: default }, - { image: tensorflow-notebook, variant: default }, - { image: pytorch-notebook, variant: default }, - { image: pytorch-notebook, variant: cuda }, - { image: pytorch-notebook, variant: cuda11 }, - { image: datascience-notebook, variant: default }, - { image: pyspark-notebook, variant: default }, - { image: all-spark-notebook, variant: default }, - ] - needs: - [ - foundation, - base, - minimal, - scipy, - r, - julia, - tensorflow, - pytorch, - pytorch-cuda, - pytorch-cuda11, - datascience, - pyspark, - all-spark, - ] - if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - - tag-push-fast: - uses: ./.github/workflows/docker-tag-push.yml - with: - platform: ${{ inputs.platform }} - image: ${{ matrix.image-variant.image }} - variant: ${{ matrix.image-variant.variant }} - secrets: - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - strategy: - matrix: - image-variant: - [ - { image: docker-stacks-foundation, variant: default }, - { image: base-notebook, variant: default }, - ] - needs: [foundation, base] - if: ${{ contains(github.event.pull_request.title, '[FAST_BUILD]') }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 122542745b..d5b4922f09 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: Build, merge tags, update wiki, test recipes +name: Build, test, and push Docker Images # [FAST_BUILD] in the PR title makes this workflow only build # the `jupyter/docker-stacks-foundation` and `jupyter/base-notebook` images @@ -12,7 +12,6 @@ on: pull_request: paths: - ".github/workflows/docker.yml" - - ".github/workflows/docker-build-test-upload-all.yml" # We use local reusable workflows to make architecture clean and simple # https://docs.github.com/en/actions/using-workflows/reusing-workflows - ".github/workflows/docker-build-test-upload.yml" @@ -37,7 +36,6 @@ on: - main paths: - ".github/workflows/docker.yml" - - ".github/workflows/docker-build-test-upload-all.yml" - ".github/workflows/docker-build-test-upload.yml" - ".github/workflows/docker-merge-tags.yml" - ".github/workflows/docker-tag-push.yml" @@ -62,26 +60,411 @@ concurrency: cancel-in-progress: true jobs: - build-aarch64: - uses: ./.github/workflows/docker-build-test-upload-all.yml + aarch64-foundation: + uses: ./.github/workflows/docker-build-test-upload.yml with: + parent-image: "" + image: docker-stacks-foundation + variant: default platform: aarch64 - runs-on-fast: ARM64_FAST - runs-on-slow: ARM64_SLOW + runs-on: ARM64_FAST + if: github.repository_owner == 'jupyter' + + x86_64-foundation: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: "" + image: docker-stacks-foundation + variant: default + platform: x86_64 + runs-on: ubuntu-latest + + aarch64-base: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: docker-stacks-foundation + image: base-notebook + variant: default + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-foundation] + if: github.repository_owner == 'jupyter' + + x86_64-base: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: docker-stacks-foundation + image: base-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-foundation] + + aarch64-minimal: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: base-notebook + image: minimal-notebook + variant: default + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-base] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-minimal: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: base-notebook + image: minimal-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-base] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-scipy: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: scipy-notebook + variant: default + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-minimal] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-scipy: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: scipy-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-r: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: r-notebook + variant: default + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-minimal] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-r: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: r-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-julia: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: julia-notebook + variant: default + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-minimal] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-julia: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: minimal-notebook + image: julia-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-minimal] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-tensorflow: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: tensorflow-notebook + variant: default + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-tensorflow: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: tensorflow-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-pytorch: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + variant: default + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-pytorch: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-pytorch-cuda: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + variant: cuda + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-pytorch-cuda: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + variant: cuda + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-pytorch-cuda11: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + variant: cuda11 + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-pytorch-cuda11: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pytorch-notebook + variant: cuda11 + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-datascience: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: datascience-notebook + variant: default + platform: aarch64 + runs-on: ARM64_SLOW + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-datascience: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: datascience-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-pyspark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pyspark-notebook + variant: default + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-scipy] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-pyspark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: scipy-notebook + image: pyspark-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-scipy] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-all-spark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: pyspark-notebook + image: all-spark-notebook + variant: default + platform: aarch64 + runs-on: ARM64_FAST + needs: [aarch64-pyspark] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-all-spark: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parent-image: pyspark-notebook + image: all-spark-notebook + variant: default + platform: x86_64 + runs-on: ubuntu-latest + needs: [x86_64-pyspark] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} + + aarch64-images-tag-push: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: aarch64 + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} secrets: REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} - if: github.repository_owner == 'jupyter' + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + { image: minimal-notebook, variant: default }, + { image: scipy-notebook, variant: default }, + { image: r-notebook, variant: default }, + { image: julia-notebook, variant: default }, + { image: tensorflow-notebook, variant: default }, + { image: pytorch-notebook, variant: default }, + { image: pytorch-notebook, variant: cuda }, + { image: pytorch-notebook, variant: cuda11 }, + { image: datascience-notebook, variant: default }, + { image: pyspark-notebook, variant: default }, + { image: all-spark-notebook, variant: default }, + ] + needs: + [ + aarch64-foundation, + aarch64-base, + aarch64-minimal, + aarch64-scipy, + aarch64-r, + aarch64-julia, + aarch64-tensorflow, + aarch64-pytorch, + aarch64-datascience, + aarch64-pyspark, + aarch64-all-spark, + ] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + + aarch64-images-tag-push-fast: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: aarch64 + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + ] + needs: [aarch64-foundation, aarch64-base] + if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') + + x86_64-images-tag-push: + uses: ./.github/workflows/docker-tag-push.yml + with: + platform: x86_64 + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} + secrets: + REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + { image: minimal-notebook, variant: default }, + { image: scipy-notebook, variant: default }, + { image: r-notebook, variant: default }, + { image: julia-notebook, variant: default }, + { image: tensorflow-notebook, variant: default }, + { image: pytorch-notebook, variant: default }, + { image: pytorch-notebook, variant: cuda }, + { image: pytorch-notebook, variant: cuda11 }, + { image: datascience-notebook, variant: default }, + { image: pyspark-notebook, variant: default }, + { image: all-spark-notebook, variant: default }, + ] + needs: + [ + x86_64-foundation, + x86_64-base, + x86_64-minimal, + x86_64-scipy, + x86_64-r, + x86_64-julia, + x86_64-tensorflow, + x86_64-pytorch, + x86_64-datascience, + x86_64-pyspark, + x86_64-all-spark, + ] + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - build-x86_64: - uses: ./.github/workflows/docker-build-test-upload-all.yml + x86_64-images-tag-push-fast: + uses: ./.github/workflows/docker-tag-push.yml with: platform: x86_64 - runs-on-fast: ubuntu-latest - runs-on-slow: ubuntu-latest + image: ${{ matrix.image-variant.image }} + variant: ${{ matrix.image-variant.variant }} secrets: REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }} REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + strategy: + matrix: + image-variant: + [ + { image: docker-stacks-foundation, variant: default }, + { image: base-notebook, variant: default }, + ] + needs: [x86_64-foundation, x86_64-base] + if: contains(github.event.pull_request.title, '[FAST_BUILD]') merge-tags: uses: ./.github/workflows/docker-merge-tags.yml @@ -109,12 +492,8 @@ jobs: { image: pyspark-notebook, variant: default }, { image: all-spark-notebook, variant: default }, ] - needs: [build-aarch64, build-x86_64] - if: | - always() && - needs.build-x86_64.result == 'success' && - (needs.build-aarch64.result == 'success' || needs.build-aarch64.result == 'skipped') && - !contains(github.event.pull_request.title, '[FAST_BUILD]') + needs: [aarch64-images-tag-push, x86_64-images-tag-push] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') merge-tags-fast: uses: ./.github/workflows/docker-merge-tags.yml @@ -131,17 +510,20 @@ jobs: { image: docker-stacks-foundation, variant: default }, { image: base-notebook, variant: default }, ] - needs: [build-aarch64, build-x86_64] - if: | - always() && - needs.build-x86_64.result == 'success' && - (needs.build-aarch64.result == 'success' || needs.build-aarch64.result == 'skipped') && - contains(github.event.pull_request.title, '[FAST_BUILD]') + needs: [aarch64-images-tag-push-fast, x86_64-images-tag-push-fast] + if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') wiki-update: uses: ./.github/workflows/docker-wiki-update.yml - needs: [build-aarch64, build-x86_64] - if: github.repository_owner == 'jupyter' + needs: [aarch64-images-tag-push, x86_64-images-tag-push] + if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + permissions: + contents: write + + wiki-update-fast: + uses: ./.github/workflows/docker-wiki-update.yml + needs: [aarch64-images-tag-push-fast, x86_64-images-tag-push-fast] + if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') permissions: contents: write From 23044cedbef0a29b88ee5f198ab9c0c971e142dd Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 21:16:30 +0100 Subject: [PATCH 09/19] refactor: use single build image step (cherry picked from commit 5f1ac0aeedcb5969a6d4b2a5bc939817378ab55d) --- .github/workflows/docker-build-test-upload.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index b65d5ee45c..453f06cba5 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -81,18 +81,8 @@ jobs: run: docker pull ubuntu:22.04 shell: bash - - name: Build image (default) ๐Ÿ›  - if: inputs.variant == 'default' - run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }} - env: - DOCKER_BUILDKIT: 1 - # Full logs for CI build - BUILDKIT_PROGRESS: plain - shell: bash - - - name: Build image (variant) ๐Ÿ›  - if: inputs.variant != 'default' - run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/${{ inputs.variant }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }} + - name: Build image ๐Ÿ›  + run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/${{ inputs.variant != 'default' && inputs.variant || '.' }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }} env: DOCKER_BUILDKIT: 1 # Full logs for CI build From e492b58d7fd41fcc78c500e9a2f959ad0c314378 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 21:22:52 +0100 Subject: [PATCH 10/19] fix: run merge tags regardless of repository owner (cherry picked from commit 3fce366a98adc5db0d127f28ddf3157d13297a0f) --- .github/workflows/docker.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d5b4922f09..aedf58936d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -493,7 +493,11 @@ jobs: { image: all-spark-notebook, variant: default }, ] needs: [aarch64-images-tag-push, x86_64-images-tag-push] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: | + always() && + needs.x86_64-images-tag-push.result == 'success' && + (needs.aarch64-images-tag-push.result == 'success' || needs.aarch64-images-tag-push.result == 'skipped') && + !contains(github.event.pull_request.title, '[FAST_BUILD]') merge-tags-fast: uses: ./.github/workflows/docker-merge-tags.yml @@ -511,7 +515,11 @@ jobs: { image: base-notebook, variant: default }, ] needs: [aarch64-images-tag-push-fast, x86_64-images-tag-push-fast] - if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') + if: | + always() && + needs.x86_64-images-tag-push-fast.result == 'success' && + (needs.aarch64-images-tag-push-fast.result == 'success' || needs.aarch64-images-tag-push-fast.result == 'skipped') && + contains(github.event.pull_request.title, '[FAST_BUILD]') wiki-update: uses: ./.github/workflows/docker-wiki-update.yml From 74869cc69132733dbda6b6dd6debd2715b474897 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 21:25:53 +0100 Subject: [PATCH 11/19] refactor: build cuda12 instead of cuda tag (cherry picked from commit 217144ecd322356376f04efb92792a20b4380177) --- .github/workflows/docker.yml | 14 +++++++------- .../pytorch-notebook/{cuda => cuda12}/Dockerfile | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) rename images/pytorch-notebook/{cuda => cuda12}/Dockerfile (91%) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index aedf58936d..53378e1e20 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -232,23 +232,23 @@ jobs: needs: [x86_64-scipy] if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - aarch64-pytorch-cuda: + aarch64-pytorch-cuda12: uses: ./.github/workflows/docker-build-test-upload.yml with: parent-image: scipy-notebook image: pytorch-notebook - variant: cuda + variant: cuda12 platform: aarch64 runs-on: ARM64_SLOW needs: [aarch64-scipy] if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - x86_64-pytorch-cuda: + x86_64-pytorch-cuda12: uses: ./.github/workflows/docker-build-test-upload.yml with: parent-image: scipy-notebook image: pytorch-notebook - variant: cuda + variant: cuda12 platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-scipy] @@ -363,7 +363,7 @@ jobs: { image: julia-notebook, variant: default }, { image: tensorflow-notebook, variant: default }, { image: pytorch-notebook, variant: default }, - { image: pytorch-notebook, variant: cuda }, + { image: pytorch-notebook, variant: cuda12 }, { image: pytorch-notebook, variant: cuda11 }, { image: datascience-notebook, variant: default }, { image: pyspark-notebook, variant: default }, @@ -425,7 +425,7 @@ jobs: { image: julia-notebook, variant: default }, { image: tensorflow-notebook, variant: default }, { image: pytorch-notebook, variant: default }, - { image: pytorch-notebook, variant: cuda }, + { image: pytorch-notebook, variant: cuda12 }, { image: pytorch-notebook, variant: cuda11 }, { image: datascience-notebook, variant: default }, { image: pyspark-notebook, variant: default }, @@ -486,7 +486,7 @@ jobs: { image: julia-notebook, variant: default }, { image: tensorflow-notebook, variant: default }, { image: pytorch-notebook, variant: default }, - { image: pytorch-notebook, variant: cuda }, + { image: pytorch-notebook, variant: cuda12 }, { image: pytorch-notebook, variant: cuda11 }, { image: datascience-notebook, variant: default }, { image: pyspark-notebook, variant: default }, diff --git a/images/pytorch-notebook/cuda/Dockerfile b/images/pytorch-notebook/cuda12/Dockerfile similarity index 91% rename from images/pytorch-notebook/cuda/Dockerfile rename to images/pytorch-notebook/cuda12/Dockerfile index 5b7f0be3c0..64e569dff0 100644 --- a/images/pytorch-notebook/cuda/Dockerfile +++ b/images/pytorch-notebook/cuda12/Dockerfile @@ -13,7 +13,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install PyTorch with pip (https://pytorch.org/get-started/locally/) # hadolint ignore=DL3013 -RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com \ +RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu121' \ 'torch' \ 'torchvision' \ 'torchaudio' && \ From c48e8d2998635aba9142dba20ce12790cd168d07 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Sun, 11 Feb 2024 22:37:55 +0100 Subject: [PATCH 12/19] docs: add note about CUDA tags to documentation --- docs/using/selecting.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/using/selecting.md b/docs/using/selecting.md index 8d951918a3..a354130c96 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -195,7 +195,9 @@ It contains: `jupyter/pytorch-notebook` includes popular Python deep learning libraries. - Everything in `jupyter/scipy-notebook` and its ancestor images -- [pytorch](https://pytorch.org/) machine learning library (optionally with CUDA) +- [pytorch](https://pytorch.org/) machine learning library + +> **GPU Acceleration:** Append a CUDA version prefix (`cuda12-` for CUDA 12 or `cuda11-` for CUDA 11) to the image tag to allow PyTorch operations to use compatible NVIDIA GPUs for accelerated computation. ### jupyter/datascience-notebook From d25dbbdf63ec9703121dcdf1deca7b0d53998308 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Mon, 12 Feb 2024 21:52:42 +0100 Subject: [PATCH 13/19] refactor: add default value for variant in build-test-upload --- .../workflows/docker-build-test-upload.yml | 3 ++- .github/workflows/docker.yml | 22 ------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index 453f06cba5..c1fcae987a 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -22,8 +22,9 @@ on: type: string variant: description: Variant tag prefix - required: true + required: false type: string + default: default platform: description: Image platform required: true diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 53378e1e20..52b8c446a9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -65,7 +65,6 @@ jobs: with: parent-image: "" image: docker-stacks-foundation - variant: default platform: aarch64 runs-on: ARM64_FAST if: github.repository_owner == 'jupyter' @@ -75,7 +74,6 @@ jobs: with: parent-image: "" image: docker-stacks-foundation - variant: default platform: x86_64 runs-on: ubuntu-latest @@ -84,7 +82,6 @@ jobs: with: parent-image: docker-stacks-foundation image: base-notebook - variant: default platform: aarch64 runs-on: ARM64_FAST needs: [aarch64-foundation] @@ -95,7 +92,6 @@ jobs: with: parent-image: docker-stacks-foundation image: base-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-foundation] @@ -105,7 +101,6 @@ jobs: with: parent-image: base-notebook image: minimal-notebook - variant: default platform: aarch64 runs-on: ARM64_FAST needs: [aarch64-base] @@ -116,7 +111,6 @@ jobs: with: parent-image: base-notebook image: minimal-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-base] @@ -127,7 +121,6 @@ jobs: with: parent-image: minimal-notebook image: scipy-notebook - variant: default platform: aarch64 runs-on: ARM64_FAST needs: [aarch64-minimal] @@ -138,7 +131,6 @@ jobs: with: parent-image: minimal-notebook image: scipy-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-minimal] @@ -149,7 +141,6 @@ jobs: with: parent-image: minimal-notebook image: r-notebook - variant: default platform: aarch64 runs-on: ARM64_SLOW needs: [aarch64-minimal] @@ -160,7 +151,6 @@ jobs: with: parent-image: minimal-notebook image: r-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-minimal] @@ -171,7 +161,6 @@ jobs: with: parent-image: minimal-notebook image: julia-notebook - variant: default platform: aarch64 runs-on: ARM64_SLOW needs: [aarch64-minimal] @@ -182,7 +171,6 @@ jobs: with: parent-image: minimal-notebook image: julia-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-minimal] @@ -193,7 +181,6 @@ jobs: with: parent-image: scipy-notebook image: tensorflow-notebook - variant: default platform: aarch64 runs-on: ARM64_SLOW needs: [aarch64-scipy] @@ -204,7 +191,6 @@ jobs: with: parent-image: scipy-notebook image: tensorflow-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-scipy] @@ -215,7 +201,6 @@ jobs: with: parent-image: scipy-notebook image: pytorch-notebook - variant: default platform: aarch64 runs-on: ARM64_SLOW needs: [aarch64-scipy] @@ -226,7 +211,6 @@ jobs: with: parent-image: scipy-notebook image: pytorch-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-scipy] @@ -281,7 +265,6 @@ jobs: with: parent-image: scipy-notebook image: datascience-notebook - variant: default platform: aarch64 runs-on: ARM64_SLOW needs: [aarch64-scipy] @@ -292,7 +275,6 @@ jobs: with: parent-image: scipy-notebook image: datascience-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-scipy] @@ -303,7 +285,6 @@ jobs: with: parent-image: scipy-notebook image: pyspark-notebook - variant: default platform: aarch64 runs-on: ARM64_FAST needs: [aarch64-scipy] @@ -314,7 +295,6 @@ jobs: with: parent-image: scipy-notebook image: pyspark-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-scipy] @@ -325,7 +305,6 @@ jobs: with: parent-image: pyspark-notebook image: all-spark-notebook - variant: default platform: aarch64 runs-on: ARM64_FAST needs: [aarch64-pyspark] @@ -336,7 +315,6 @@ jobs: with: parent-image: pyspark-notebook image: all-spark-notebook - variant: default platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-pyspark] From 173a00641e9348442e19b37bfb98c00c98026185 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Mon, 12 Feb 2024 21:54:18 +0100 Subject: [PATCH 14/19] refactor: swap ordering of cuda11/cuda12 variants --- .github/workflows/docker.yml | 22 +++++++++++----------- docs/using/selecting.md | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 52b8c446a9..88b26565c7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -216,45 +216,45 @@ jobs: needs: [x86_64-scipy] if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - aarch64-pytorch-cuda12: + aarch64-pytorch-cuda11: uses: ./.github/workflows/docker-build-test-upload.yml with: parent-image: scipy-notebook image: pytorch-notebook - variant: cuda12 + variant: cuda11 platform: aarch64 runs-on: ARM64_SLOW needs: [aarch64-scipy] if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - x86_64-pytorch-cuda12: + x86_64-pytorch-cuda11: uses: ./.github/workflows/docker-build-test-upload.yml with: parent-image: scipy-notebook image: pytorch-notebook - variant: cuda12 + variant: cuda11 platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-scipy] if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - aarch64-pytorch-cuda11: + aarch64-pytorch-cuda12: uses: ./.github/workflows/docker-build-test-upload.yml with: parent-image: scipy-notebook image: pytorch-notebook - variant: cuda11 + variant: cuda12 platform: aarch64 runs-on: ARM64_SLOW needs: [aarch64-scipy] if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - x86_64-pytorch-cuda11: + x86_64-pytorch-cuda12: uses: ./.github/workflows/docker-build-test-upload.yml with: parent-image: scipy-notebook image: pytorch-notebook - variant: cuda11 + variant: cuda12 platform: x86_64 runs-on: ubuntu-latest needs: [x86_64-scipy] @@ -341,8 +341,8 @@ jobs: { image: julia-notebook, variant: default }, { image: tensorflow-notebook, variant: default }, { image: pytorch-notebook, variant: default }, - { image: pytorch-notebook, variant: cuda12 }, { image: pytorch-notebook, variant: cuda11 }, + { image: pytorch-notebook, variant: cuda12 }, { image: datascience-notebook, variant: default }, { image: pyspark-notebook, variant: default }, { image: all-spark-notebook, variant: default }, @@ -403,8 +403,8 @@ jobs: { image: julia-notebook, variant: default }, { image: tensorflow-notebook, variant: default }, { image: pytorch-notebook, variant: default }, - { image: pytorch-notebook, variant: cuda12 }, { image: pytorch-notebook, variant: cuda11 }, + { image: pytorch-notebook, variant: cuda12 }, { image: datascience-notebook, variant: default }, { image: pyspark-notebook, variant: default }, { image: all-spark-notebook, variant: default }, @@ -464,8 +464,8 @@ jobs: { image: julia-notebook, variant: default }, { image: tensorflow-notebook, variant: default }, { image: pytorch-notebook, variant: default }, - { image: pytorch-notebook, variant: cuda12 }, { image: pytorch-notebook, variant: cuda11 }, + { image: pytorch-notebook, variant: cuda12 }, { image: datascience-notebook, variant: default }, { image: pyspark-notebook, variant: default }, { image: all-spark-notebook, variant: default }, diff --git a/docs/using/selecting.md b/docs/using/selecting.md index a354130c96..8d03fa5e3f 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -197,7 +197,7 @@ It contains: - Everything in `jupyter/scipy-notebook` and its ancestor images - [pytorch](https://pytorch.org/) machine learning library -> **GPU Acceleration:** Append a CUDA version prefix (`cuda12-` for CUDA 12 or `cuda11-` for CUDA 11) to the image tag to allow PyTorch operations to use compatible NVIDIA GPUs for accelerated computation. +> **GPU Acceleration:** Append a CUDA version prefix (`cuda11-` for CUDA 11 or `cuda12-` for CUDA 12) to the image tag to allow PyTorch operations to use compatible NVIDIA GPUs for accelerated computation. ### jupyter/datascience-notebook From be3fdb42c7128502304ac9552bb4966ef1aae49c Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Mon, 12 Feb 2024 21:55:11 +0100 Subject: [PATCH 15/19] refactor: remove optional str type in arg parser --- tagging/apply_tags.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tagging/apply_tags.py b/tagging/apply_tags.py index 169a3db869..9c50cd0e01 100755 --- a/tagging/apply_tags.py +++ b/tagging/apply_tags.py @@ -75,7 +75,6 @@ def apply_tags( arg_parser.add_argument( "--variant", required=True, - type=str, help="Variant tag prefix", ) args = arg_parser.parse_args() From f2927ea46865dd563c23f9df7b24ce6beb443d11 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Mon, 12 Feb 2024 23:21:57 +0100 Subject: [PATCH 16/19] fix: add proper env variables to CUDA Dockerfiles --- images/pytorch-notebook/cuda11/Dockerfile | 4 ++++ images/pytorch-notebook/cuda12/Dockerfile | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/images/pytorch-notebook/cuda11/Dockerfile b/images/pytorch-notebook/cuda11/Dockerfile index 01a52e7ff6..761659613c 100644 --- a/images/pytorch-notebook/cuda11/Dockerfile +++ b/images/pytorch-notebook/cuda11/Dockerfile @@ -19,3 +19,7 @@ RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index 'torchaudio' && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" + +# # https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/1.14.5/docker-specialized.html#dockerfiles +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility diff --git a/images/pytorch-notebook/cuda12/Dockerfile b/images/pytorch-notebook/cuda12/Dockerfile index 64e569dff0..e44b1b7311 100644 --- a/images/pytorch-notebook/cuda12/Dockerfile +++ b/images/pytorch-notebook/cuda12/Dockerfile @@ -19,3 +19,7 @@ RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index 'torchaudio' && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" + +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/1.14.5/docker-specialized.html#dockerfiles +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility From efb13665608bbe012ecd67e6a305cfd2985cfc9d Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Mon, 12 Feb 2024 23:23:46 +0100 Subject: [PATCH 17/19] fix: remove CUDA build for aarch64 --- .github/workflows/docker.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 88b26565c7..22a34b708c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -216,17 +216,6 @@ jobs: needs: [x86_64-scipy] if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - aarch64-pytorch-cuda11: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pytorch-notebook - variant: cuda11 - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - x86_64-pytorch-cuda11: uses: ./.github/workflows/docker-build-test-upload.yml with: @@ -238,17 +227,6 @@ jobs: needs: [x86_64-scipy] if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} - aarch64-pytorch-cuda12: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - parent-image: scipy-notebook - image: pytorch-notebook - variant: cuda12 - platform: aarch64 - runs-on: ARM64_SLOW - needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') - x86_64-pytorch-cuda12: uses: ./.github/workflows/docker-build-test-upload.yml with: @@ -341,8 +319,6 @@ jobs: { image: julia-notebook, variant: default }, { image: tensorflow-notebook, variant: default }, { image: pytorch-notebook, variant: default }, - { image: pytorch-notebook, variant: cuda11 }, - { image: pytorch-notebook, variant: cuda12 }, { image: datascience-notebook, variant: default }, { image: pyspark-notebook, variant: default }, { image: all-spark-notebook, variant: default }, From b3d83470b05e43116af36192b9a2efa85ebff500 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Tue, 13 Feb 2024 01:51:08 +0100 Subject: [PATCH 18/19] fix: use latest NVIDIA documentation link --- images/pytorch-notebook/cuda11/Dockerfile | 2 +- images/pytorch-notebook/cuda12/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/pytorch-notebook/cuda11/Dockerfile b/images/pytorch-notebook/cuda11/Dockerfile index 761659613c..de0ef1ec95 100644 --- a/images/pytorch-notebook/cuda11/Dockerfile +++ b/images/pytorch-notebook/cuda11/Dockerfile @@ -20,6 +20,6 @@ RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# # https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/1.14.5/docker-specialized.html#dockerfiles +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html#dockerfiles ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility diff --git a/images/pytorch-notebook/cuda12/Dockerfile b/images/pytorch-notebook/cuda12/Dockerfile index e44b1b7311..c574e42d8d 100644 --- a/images/pytorch-notebook/cuda12/Dockerfile +++ b/images/pytorch-notebook/cuda12/Dockerfile @@ -20,6 +20,6 @@ RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/1.14.5/docker-specialized.html#dockerfiles +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html#dockerfiles ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility From 5977b63246c65638d7d50a7ec26ac8f572752658 Mon Sep 17 00:00:00 2001 From: zynaa <7562909-zynaa@users.noreply.gitlab.com> Date: Tue, 13 Feb 2024 01:53:04 +0100 Subject: [PATCH 19/19] fix: skip aarch64 tags file for CUDA variants --- .github/workflows/docker-merge-tags.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml index 1245a4151f..07107a4cbd 100644 --- a/.github/workflows/docker-merge-tags.yml +++ b/.github/workflows/docker-merge-tags.yml @@ -41,7 +41,7 @@ jobs: with: name: ${{ inputs.image }}-aarch64-${{ inputs.variant }}-tags path: /tmp/jupyter/tags/ - if: github.repository_owner == 'jupyter' + if: github.repository_owner == 'jupyter' && !contains(inputs.variant, 'cuda') # Docker might be stuck when pulling images # https://github.com/docker/for-mac/issues/2083