Add conda-lock unified lockfile (conda-lock.yml) (#416)

* add conda-lock.yml capability
* fix report image size

Co-authored-by: pangeo-bot <[email protected]>

.github/workflows/Build.yml

@@ -113,9 +113,10 @@ jobs:
           quay.io/${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:master
           quay.io/${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:${{env.DOCKER_TAG}}
 
-    - name: Export Full Conda Environment
+    - name: Inspect Image
       run: |
         docker run ${DOCKER_ORG}/${{ matrix.IMAGE }}:master conda list
+        docker images ls
 
     - name: Test Image
       run: |

.github/workflows/CondaLock.yml

@@ -25,26 +25,26 @@ jobs:
 
       - name: Show environment.ymls
         run: |
-          echo "---\nbase-notebook:\n---\n" && cat base-notebook/environment.yml
-          echo "---\npangeo-notebook:\n---\n" && cat pangeo-notebook/environment.yml
-          echo "---\nml-notebook:\n---\n" && cat ml-notebook/environment.yml
-          echo "---\npytorch-notebook:\n---\n" && cat pytorch-notebook/environment.yml
+          echo -e "---\nbase-notebook:\n---\n" && cat base-notebook/environment.yml
+          echo -e "---\npangeo-notebook:\n---\n" && cat pangeo-notebook/environment.yml
+          echo -e "---\nml-notebook:\n---\n" && cat ml-notebook/environment.yml
+          echo -e "---\npytorch-notebook:\n---\n" && cat pytorch-notebook/environment.yml
+          echo -e "---\nforge:\n---\n" && cat forge/environment.yml
 
       - name: Setup Conda Environment with Mambaforge
         uses: conda-incubator/setup-miniconda@v2
         with:
            # Match base-image/Dockerfile version
            miniforge-version: latest
            miniforge-variant: Mambaforge
-           environment-file: environment-condalock.yml
            activate-environment: condalock
            use-mamba: true
 
       - name: Get Date
         id: get-date
         run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
 
-      - name: Cache Conda env
+      - name: Cache Conda Environment
         id: cache
         uses: actions/cache@v3
         with:
@@ -58,59 +58,67 @@ jobs:
           # Increase this value to reset cache if environment-condalock.yml has not changed
           CACHE_NUMBER: 0
 
-      - name: Update environment
+      - name: Update Conda Environment
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
           mamba env update -n condalock -f environment-condalock.yml
-          conda clean --all --yes --force-pkgs-dirs
+
+      - name: Display Conda Environment
+        run: |
+          mamba list
 
       # Could run as single step in parallel, but would complicate logs...
       - name: Run conda-lock base-notebook
         timeout-minutes: 5
         continue-on-error: true
         run: |
           cd base-notebook
-          conda-lock lock --mamba -k explicit -f environment.yml -p linux-64
+          conda-lock lock -f environment.yml -p linux-64
+          conda-lock render -k explicit
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
       - name: Run conda-lock pangeo-notebook
         timeout-minutes: 5
         continue-on-error: true
         run: |
           cd pangeo-notebook
-          conda-lock lock --mamba -k explicit -f environment.yml -f ../base-notebook/environment.yml -p linux-64
+          conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64 -p osx-64
+          conda-lock render -k explicit
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
       - name: Run conda-lock ml-notebook
         timeout-minutes: 5
         continue-on-error: true
         run: |
           cd ml-notebook
-          conda-lock lock --mamba -k explicit -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml  -p linux-64
+          conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
+          conda-lock render -k explicit
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
       - name: Run conda-lock pytorch-notebook
         timeout-minutes: 5
         continue-on-error: true
         run: |
           cd pytorch-notebook
-          conda-lock lock --mamba -k explicit -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
+          conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
+          conda-lock render -k explicit
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
       - name: Run conda-lock forge
         timeout-minutes: 5
         continue-on-error: true
         run: |
           cd forge
-          conda-lock lock --mamba -k explicit -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
+          conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
+          conda-lock render -k explicit 
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
       # Commit the change to the PR branch (currently fails if no modified files)
       - name: Commit condalock files to PR
         run: |
           git config --global user.name 'pangeo-bot'
           git config --global user.email '[email protected]'
-          git add */conda-linux-64.lock */packages.txt
+          git add */conda*.lock */conda-lock.yml */packages.txt
           git commit -m "[condalock-command] autogenerated conda-lock files"
           git push
 

.github/workflows/Test.yml

@@ -59,8 +59,8 @@ jobs:
 
     - name: Report Image Size and Conda Packages
       run: |
-        docker images
         docker run localhost:5000/${DOCKER_ORG}/${{ matrix.IMAGE }}:PR conda list --export
+        docker images ls
 
     - name: Test Image
       run: |

base-image/Dockerfile

@@ -68,6 +68,7 @@ RUN echo "Installing Mambaforge..." \
     && wget --quiet ${URL} -O installer.sh \
     && /bin/bash installer.sh -u -b -p ${CONDA_DIR} \
     && rm installer.sh \
+    && mamba install conda-lock -y \
     && mamba clean -afy \
     # After installing the packages, we cleanup some unnecessary files
     # to try reduce image size - see https://jcristharif.com/conda-docker-tips.html
@@ -149,14 +150,16 @@ ONBUILD USER ${NB_USER}
 # a list of base packages.
 # After installing the packages, we cleanup some unnecessary files
 # to try reduce image size - see https://jcristharif.com/conda-docker-tips.html
-ONBUILD RUN echo "Checking for 'conda-linux-64.lock' or 'environment.yml'..." \
+ONBUILD RUN echo "Checking for 'conda-lock.yml' 'conda-linux-64.lock' or 'environment.yml'..." \
         ; [ -d binder ] && cd binder \
         ; [ -d .binder ] && cd .binder \
-        ; if test -f "conda-linux-64.lock" ; then \
+        ; if test -f "conda-lock.yml" ; then \
+        conda-lock install --name ${CONDA_ENV} --file conda-lock.yml \
+        ; elif test -f "conda-linux-64.lock" ; then \
         mamba create --name ${CONDA_ENV} --file conda-linux-64.lock \
         ; elif test -f "environment.yml" ; then \
         mamba env create --name ${CONDA_ENV} -f environment.yml  \
-        ; else echo "No conda-linux-64.lock or environment.yml! *creating default env*" ; \
+        ; else echo "No conda-lock.yml, conda-linux-64.lock, or environment.yml! *creating default env*" ; \
         mamba create --name ${CONDA_ENV} pangeo-notebook \
         ; fi \
         && mamba clean -yaf \

environment-condalock.yml

@@ -2,6 +2,8 @@ name: condalock
 channels:
   - conda-forge
 dependencies:
-  - python=3.10
+  - python=3.11
   - conda-lock=1.2
+  # Until https://github.com/conda-incubator/conda-lock/issues/292 resolved
+  - conda<22.11
   - mamba=1