Merge pull request #18 from CMCC-Foundation/add_method_to_image

merging branche add_method_to_image to branch main

.github/workflows/build-push-docker-base.yml

@@ -0,0 +1,41 @@
+name: Build geokube-base Docker image and push to geokube-base container registry
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build-base:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set Docker image tag name
+      run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
+    - name: TAG ECHO
+      run: echo ${{ env.TAG }}
+    - uses: actions/setup-python@v1
+      with:
+        python-version: 3.9
+    - name: Install WHEEL tool
+      run: pip install wheel
+    - name: Build WHEEL file
+      run: python setup.py bdist_wheel
+    - name: Login to Scaleway Container Registry
+      uses: docker/login-action@v3
+      with:
+        username: nologin
+        password: ${{ secrets.SCALEWAY_PASSWORD }}
+        registry: ${{ vars.GEOKUBE_REGISTRY }}
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+    - name: Build and push
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./Dockerfile.base
+        push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        tags: | 
+          ${{ vars.GEOKUBE_REGISTRY }}/geokube-base:${{ env.TAG }}
+          ${{ vars.GEOKUBE_REGISTRY }}/geokube-base:latest

.github/workflows/build-push-docker_pr.yml

@@ -1,16 +1,20 @@
 name: Build geokube Docker image and push to geokube container registry
 
 on:
-  pull_request:
-    types: [opened, synchronize]
+  workflow_run:
+    workflows: [ 'build-base' ]
+    types: [ completed ]
   workflow_dispatch:
+
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
     - name: Set Docker image tag name
       run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
+    - name: TAG ECHO
+      run: echo ${{ env.TAG }}
     - uses: actions/setup-python@v1
       with:
         python-version: 3.9
@@ -19,19 +23,21 @@ jobs:
     - name: Build WHEEL file
       run: python setup.py bdist_wheel
     - name: Login to Scaleway Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: nologin
         password: ${{ secrets.SCALEWAY_PASSWORD }}
         registry: ${{ vars.GEOKUBE_REGISTRY }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2        
+      uses: docker/setup-buildx-action@v2
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile
         push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOKUBE_REGISTRY }}/geokube:${{ env.TAG }}
           ${{ vars.GEOKUBE_REGISTRY }}/geokube:latest

.github/workflows/build-push-docker_release.yml

@@ -26,11 +26,13 @@ jobs:
         username: nologin
         password: ${{ secrets.SCALEWAY_PASSWORD }}
         registry: ${{ vars.GEOKUBE_REGISTRY }}
-    - name: Set up Docker Buildx
-      uses: docker/build-push-action@v4        
+    - name: Build and push
+      uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile
         push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         tags: | 
           ${{ vars.GEOKUBE_REGISTRY }}/geokube:${{ env.RELEASE_TAG }}

.github/workflows/main.yml

@@ -1,63 +1,22 @@
 name: Unit Tests
+
 on:
-  pull_request:
-    types: [opened, reopened, edited, assigned, synchronize]
+  workflow_run:
+    workflows: [ 'build' ]
+    types: [ completed ]
   workflow_dispatch:
 
 jobs:
-  build:
+  pytest:
     runs-on: ubuntu-latest
+    container:
+      image: ${{ vars.GEOKUBE_REGISTRY }}/geokube:latest
+      credentials:
+        username: nologin
+        password: ${{ secrets.SCALEWAY_PASSWORD }}
     steps:
       - uses: actions/checkout@v2
 
-      - uses: actions/setup-python@v1
-        with:
-          python-version: 3.9    
-
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          auto-activate-base: true
-          python-version: 3.9
-
-      - name: Conda create environemnt
-        shell: bash -l {0}
-        run: conda create --name gk   
-
-      - name: Conda activate
-        shell: bash -l {0}
-        run: conda activate gk       
-
-      - run: |
-          conda install -c conda-forge --yes --freeze-installed xesmf cartopy psycopg2 'bokeh>=2.4.2,<3' pandas=1.4.3 netCDF4 scipy xarray=2022.6.0
-          
-      - run: |
-          conda run pip install geopandas       
-          
-      - run: conda run pip install pytest-github-actions-annotate-failures           
-
-      - run: |
-          conda run python setup.py install
-          
-      - run: |
-          conda run pip install black==22.3.0
-
-      - name: Check formatting in `tests` catalog
-        run: conda run black -l 79 --preview tests
-
-      - name: Check formatting in `geokube` catalog
-        run: conda run black -l 79 --preview geokube
-
-      - name: Check for modified files
-        id: git-check
-        run: echo ::set-output name=modified::$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi)    
-
-      - name: Verify code formatting    
-        if: steps.git-check.outputs.modified == 'true'
-        uses: actions/github-script@v3
-        with:
-          script: |
-            core.setFailed('`geokube` or `tests` directory is not well formatted! Use command `black -l 79 --preview geokube` and `black -l 79 --preview tests`')
+      - name: Run test
+        run: pytest tests
 
-      - name: Run pytest
-        run: conda run pytest tests

.gitignore

@@ -115,3 +115,9 @@ _old/
 
 # Netcdf files
 *.nc
+development.py
+Dockerfile.dev
+nuts_shape
+*.png
+*.zarr
+.DS_Store

Dockerfile

@@ -1,15 +1,9 @@
-FROM continuumio/miniconda3:22.11.1
-COPY environment.yaml /tmp/enviroment.yaml
-RUN conda config --set allow_conda_downgrades true 
-RUN conda env update --name base --file /tmp/enviroment.yaml --prune
-RUN conda install -c conda-forge gxx_linux-64==11.1.0
-RUN conda update -n base -c defaults conda
-RUN conda clean -afy \
-    && find /opt/conda/ -follow -type f -name '*.a' -delete \
-    && find /opt/conda/ -follow -type f -name '*.pyc' -delete \
-    && find /opt/conda/ -follow -type f -name '*.js.map' -delete \
-    && find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
-COPY dist/geokube-0.2.6b2-py3-none-any.whl /
-RUN pip install /geokube-0.2.6b2-py3-none-any.whl
-RUN rm /geokube-0.2.6b2-py3-none-any.whl
-ENV LD_LIBRARY_PATH=/opt/conda/x86_64-conda-linux-gnu/lib:/usr/lib/x86_64-linux-gnu
+#FROM rg.fr-par.scw.cloud/geokube/geokube-base:2024.05.03.08.14
+#new version with Zarr support
+FROM rg.fr-par.scw.cloud/geokube/geokube-base:2024.09.26.09.01
+
+COPY dist/geokube-0.2.7-py3-none-any.whl /
+RUN pip install /geokube-0.2.7-py3-none-any.whl --break-system-packages
+RUN rm /geokube-0.2.7-py3-none-any.whl
+
+#ENV LD_LIBRARY_PATH=/opt/conda/x86_64-conda-linux-gnu/lib:/usr/lib/x86_64-linux-gnu

Dockerfile.base

@@ -0,0 +1,27 @@
+FROM ubuntu:24.10
+
+RUN apt update && apt install -y build-essential git gfortran python3-pip python3 libudunits2-dev
+
+ENV UDUNITS2_XML_PATH=/usr/share/xml/udunits/udunits2.xml
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+RUN update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1
+
+WORKDIR /opt
+RUN git clone https://github.com/esmf-org/esmf.git
+
+ENV ESMF_DIR=/opt/esmf
+ENV ESMF_F90COMPILEOPTS=-fallow-argument-mismatch
+
+WORKDIR /opt/esmf
+RUN git checkout v8.6.1
+RUN make -j4
+
+ENV ESMFMKFILE=/opt/esmf/lib/libO/Linux.gfortran.64.mpiuni.default/esmf.mk
+
+RUN pip install numpy setuptools==74.1.2 --break-system-packages
+
+WORKDIR /opt/esmf/src/addon/esmpy
+RUN pip install . --break-system-packages
+
+RUN pip install git+https://github.com/pangeo-data/xesmf.git --break-system-packages #@v0.6.3

environment.yaml

@@ -8,7 +8,7 @@ dependencies:
   - pandas=1.4.3
   - netCDF4
   - scipy 
-  - xarray=2022.6.0
+  - xarray=2022.10.0
   - dask=2023.1.1
   - esmpy=8.2.0
   - xesmf=0.6.3