merge: attempt fixing main and Update.Rpipeline merge conflict

.github/workflows/ci.yaml

@@ -0,0 +1,132 @@
+name: CI
+
+# Enable Buildkit and let compose use it to speed up image building
+env:
+  DOCKER_BUILDKIT: 1
+  COMPOSE_DOCKER_CLI_BUILD: 1
+  CONDA_ENV_NAME: clim-recal
+  MIN_PYTHON_VERSION: 3.9
+  PYTHON_MODULE_FOLDER: /home/jovyan/python/
+  COVERAGE_SVG_FOLDER: docs/assets/
+  COVERAGE_SVG_FILE_NAME: coverage.svg
+  # replace below with references to previous config lines
+  COVERAGE_SVG_PATH: docs/assets/coverage.svg
+
+on:
+  pull_request:
+    branches: ['main', 'docker-config']
+    paths-ignore: ['docs/**']
+
+  push:
+    branches: ['main', 'docker-config']
+    paths-ignore: ['docs/**']
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+
+jobs:
+  linter:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code Repository
+        uses: actions/checkout@main
+
+      - name: Set up Python
+        uses: actions/setup-python@main
+        with:
+          python-version: ${{ env.MIN_PYTHON_VERSION }}
+
+      - name: Run pre-commit
+        uses: pre-commit/action@main
+
+      - name: Update pre-commit
+        uses: pre-commit-ci/[email protected]
+        if: always()
+
+  # With no caching at all the entire ci process takes 4m 30s to complete!
+  pytest:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code Repository
+        uses: actions/checkout@main
+
+      - name: Build, Test and Save Test Coverage
+        run: |
+          docker compose build
+          docker compose up --detach
+          docker compose exec jupyter bash -c "conda run -n $CONDA_ENV_NAME --cwd python pytest -p no:sugar"
+          export JUPYTER_ID=$(docker compose ps -q jupyter)
+          echo "jupyter_id=$JUPYTER_ID" >> $GITHUB_ENV
+          echo "jupyter_id=$JUPYTER_ID"
+
+      - name: Check accessing saved jupyter_id
+        run: |
+          echo ${{ env.jupyter_id }}
+
+      - name: Copy test coverage results
+        run: |
+          docker cp ${{ env.jupyter_id }}:${{ env.PYTHON_MODULE_FOLDER }}${{ env.COVERAGE_SVG_PATH }} ${{ env.COVERAGE_SVG_FOLDER }}
+
+      - name: Archive coverage svg
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-badge
+          path: ${{ env.COVERAGE_SVG_PATH }}
+
+      - name: Tear down the Stack
+        run: docker compose down
+
+  docs:
+    needs: [linter, pytest]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.MIN_PYTHON_VERSION }}
+
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+        name: Update cache_id
+
+      - name: Download coverage svg
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage-badge
+          path: ${{ env.COVERAGE_SVG_PATH }}
+
+      - name: Build quarto
+        run: |
+          docker compose build
+          # docker cp ${{ env.jupyter_id }}:app/docs/assets/coverage.svg docs/assets/
+          docker compose up --detach
+          docker cp $(docker compose ps -q docs):/usr/local/apache2/htdocs/ .
+
+      # - name: Build docker quarto
+      #   run: |
+      #     docker compose build
+      #     docker cp ${{ env.jupyter_id }}:app/docs/assets/coverage.svg docs/assets/
+      #     docker compose up --detach
+      #     docker cp $(docker compose ps -q docs):/usr/local/apache2/htdocs/ .
+          # docker cp $(docker compose ps -q docs)/app/_site/ /usr/local/apache2/htdocs/
+      # - name: Apply mkdocs cache
+      #   uses: actions/cache@v3
+      #   with:
+      #     key: mkdocs-material-${{ env.cache_id }}
+      #     path: .cache
+      #     restore-keys: |
+      #       mkdocs-material-
+      #
+      # - name: Install doc dependencies via poetry
+      #   run: |
+      #     pip install poetry
+      #     poetry install --with docs
+      #
+      # - name: Build docs with gh-deploy --force
+      #   run: |
+      #     poetry run mkdocs gh-deploy --force

.gitignore

@@ -246,7 +246,11 @@ vignettes/*.pdf
 .Renviron
 
 # pkgdown site
-docs/
+# Commented out by @griff-rees in configuring quarto
+# docs/
+
+# Add as a replacement for use of quarto
+docs/reference
 
 # translation temp files
 po/*~
@@ -260,5 +264,7 @@ rsconnect/
 
 # End of https://www.toptal.com/developers/gitignore/api/macos,python,r
 
-# Some custom adds 
+# Some custom adds
 data/*
+
+/.quarto/

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+exclude: "R|data"
+repos:
+  - repo: https://github.com/psf/black
+    rev: "23.9.1"
+    hooks:
+      - id: black-jupyter
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: "v1.10.0"
+    hooks:
+      - id: rst-backticks
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+
+  - repo: https://github.com/hadialqattan/pycln
+    rev: v2.2.2
+    hooks:
+      - id: pycln
+        args: ["python/"]
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
+        args: ["--profile", "black"]
+
+  # Currently fails to build.
+  # See: https://github.com/lorenzwalthert/precommit/issues/476
+  # - repo: https://github.com/lorenzwalthert/precommit
+  #   rev: v0.3.2
+  #   hooks:
+  #     - id: readme-rmd-rendered

R/README.md

@@ -3,3 +3,12 @@
 - **/Resampling** - code for Resampling data to different extents (grid sizes) 
 - **/bias-correction-methods** - bias correction methods implemented in R
 - **/comparing-r-and-python** - Comparing various pipeline aspects between R and python 
+
+<!--
+### R 
+
+In the `R` subdirectory you can find code for replicating the different data processing and debiasing steps as above, along with comparisons of methods between the two languages. 
+- **bias-correction-methods** for bias correction (debiasing) methods available specifically in `R` libraries
+- **comparing-r-and-python** for replication of resampling and reviewing the bias correction methods applied in `python`.
+- **Resampling** for resampling the HADsUK datasets from 1km to 2.2km grid in `R`.
+-->