Use new nf-test features

.github/conda_skip.yml

@@ -0,0 +1,145 @@
+# TODO What do we do with these?
+# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now
+# - path: modules/nf-core/nf-test
+# - profile: conda
+#   path: modules/nf-core/angsd/gl
+# - profile: conda
+#   path: modules/nf-core/annotsv/installannotations
+# - profile: conda
+#   path: modules/nf-core/happy/sompy
+# - profile: conda
+#   path: modules/nf-core/backsub
+# - profile: conda
+#   path: modules/nf-core/bakta/bakta
+# - profile: conda
+#   path: modules/nf-core/bakta/baktadbdownload
+# - profile: conda
+#   path: modules/nf-core/bases2fastq
+# - profile: conda
+#   path: modules/nf-core/bcl2fastq
+# - profile: conda
+#   path: modules/nf-core/bclconvert
+# - profile: conda
+#   path: modules/nf-core/celesta
+# - profile: conda
+#   path: modules/nf-core/cellpose
+# - profile: conda
+#   path: modules/nf-core/cellranger/count
+# - profile: conda
+#   path: modules/nf-core/cellranger/mkfastq
+# - profile: conda
+#   path: modules/nf-core/cellranger/mkgtf
+# - profile: conda
+#   path: modules/nf-core/cellranger/mkref
+# - profile: conda
+#   path: modules/nf-core/cellranger/mkvdjref
+# - profile: conda
+#   path: modules/nf-core/cellranger/multi
+# - profile: conda
+#   path: modules/nf-core/cellranger/vdj
+# - profile: conda
+#   path: modules/nf-core/checkqc
+# - profile: conda
+#   path: modules/nf-core/custom/dumpsoftwareversions
+# - profile: conda
+#   path: modules/nf-core/deepcell/mesmer
+# - profile: conda
+#   path: modules/nf-core/deepvariant
+- profile: conda
+  path: modules/nf-core/deepvariant/callvariants
+- profile: conda
+  path: modules/nf-core/deepvariant/makeexamples
+- profile: conda
+  path: modules/nf-core/deepvariant/postprocessvariants
+- profile: conda
+  path: modules/nf-core/deepvariant/rundeepvariant
+# - profile: conda
+#   path: modules/nf-core/ensemblvep/vep
+# - profile: conda
+#   path: modules/nf-core/fastk/fastk
+# - profile: conda
+#   path: modules/nf-core/fastk/histex
+# - profile: conda
+#   path: modules/nf-core/fastk/merge
+# - profile: conda
+#   path: modules/nf-core/fcs/fcsadaptor
+# - profile: conda
+#   path: modules/nf-core/fcs/fcsgx
+# - profile: conda
+#   path: modules/nf-core/ganon/buildcustom
+# - profile: conda
+#   path: modules/nf-core/ganon/classify
+# - profile: conda
+#   path: modules/nf-core/ganon/report
+# - profile: conda
+#   path: modules/nf-core/ganon/table
+# - profile: conda
+#   path: modules/nf-core/gatk4/cnnscorevariants
+# - profile: conda
+#   path: modules/nf-core/gatk4/determinegermlinecontigploidy
+# - profile: conda
+#   path: modules/nf-core/genescopefk
+# - profile: conda
+#   path: modules/nf-core/ilastik/multicut
+# - profile: conda
+#   path: modules/nf-core/ilastik/pixelclassification
+# - profile: conda
+#   path: modules/nf-core/imputeme/vcftoprs
+# - profile: conda
+#   path: modules/nf-core/mcquant
+# - profile: conda
+#   path: modules/nf-core/merquryfk/katcomp
+# - profile: conda
+#   path: modules/nf-core/merquryfk/katgc
+# - profile: conda
+#   path: modules/nf-core/merquryfk/merquryfk
+# - profile: conda
+#   path: modules/nf-core/merquryfk/ploidyplot
+# - profile: conda
+#   path: modules/nf-core/molkartgarage/clahe
+# - profile: conda
+#   path: modules/nf-core/quartonotebook
+# - profile: conda
+#   path: modules/nf-core/scimap/spatiallda
+# - profile: conda
+#   path: modules/nf-core/sentieon/bwaindex
+# - profile: conda
+#   path: modules/nf-core/sentieon/bwamem
+- profile: conda
+  path: modules/nf-core/sentieon/datametrics
+# - profile: conda
+#   path: modules/nf-core/sentieon/dedup
+# - profile: conda
+#   path: modules/nf-core/sentieon/qualcal
+# - profile: conda
+#   path: modules/nf-core/spaceranger/mkgtf
+# - profile: conda
+#   path: modules/nf-core/spaceranger/mkref
+# - profile: conda
+#   path: modules/nf-core/spaceranger/count
+# - profile: conda
+#   path: modules/nf-core/spotiflow
+# - profile: conda
+#   path: modules/nf-core/svanalyzer/svbenchmark
+# - profile: conda
+#   path: modules/nf-core/universc
+# - profile: singularity
+#   path: modules/nf-core/universc
+# - profile: conda
+#   path: modules/nf-core/vt/decompose
+# - profile: singularity
+#   path: modules/nf-core/bases2fastq
+# - profile: conda
+#   path: modules/nf-core/wittyer
+# - profile: conda
+#   path: modules/nf-core/islandpath
+# - profile: conda
+#   path: subworkflows/nf-core/vcf_annotate_ensemblvep
+# - profile: conda
+#   path: subworkflows/nf-core/bcl_demultiplex
+- profile: conda
+  path: subworkflows/nf-core/deepvariant
+# - profile: conda
+#   path: subworkflows/nf-core/fastq_align_bamcmp_bwa
+# - profile: conda
+#   path: subworkflows/nf-core/fastq_align_bwa

.github/workflows/lint.yml

@@ -0,0 +1,168 @@
+name: Run Linting
+on:
+  pull_request:
+    branches: [master]
+  merge_group:
+    types: [checks_requested]
+    branches: [master]
+  workflow_dispatch:
+    inputs:
+      runners:
+        description: "Runners to test on"
+        type: choice
+        options:
+          - "ubuntu-latest"
+          - "self-hosted"
+        default: "self-hosted"
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5
+      - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
+        # FIXME Flip this off once we get to less than a couple hundred. Adding
+        # this so it will only run against changed files. It'll make it much
+        # easier to fix these as they come up rather than everything at once.
+        with:
+          extra_args: ""
+
+  ###################
+  # nf-core linting #
+  ###################
+  # TODO Move these to pre-commit
+  # https://github.com/nf-core/tools/pull/3141
+  nf-core-changes:
+    name: nf-core-changes
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.filter.outputs.changes }}
+      modules: ${{ steps.tags.outputs.modules }}
+      subworkflows: ${{ steps.tags.outputs.subworkflows }}
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        with:
+          fetch-depth: 2 # To retrieve the preceding commit.
+
+      # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented
+      - uses: mirpedrol/paths-filter@main
+        id: filter
+        with:
+          filters: "tests/config/pytest_modules.yml"
+          token: ""
+
+      - name: Fetch module tags
+        id: tags
+        run: |
+          echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT
+          echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT
+
+      - name: debug
+        run: |
+          echo ${{ steps.tags.outputs.modules }}
+          echo ${{ steps.tags.outputs.subworkflows }}
+
+  nf-core-lint-modules:
+    runs-on: ${{ github.event.inputs.runners  || 'self-hosted' }}
+    name: nf-core lint modules
+    needs: nf-core-changes
+    if: ${{ (needs.nf-core-changes.outputs.modules != '[]') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: "${{ fromJson(needs.nf-core-changes.outputs.modules) }}"
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - name: Set up Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
+        with:
+          python-version: "3.11"
+
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
+        id: cache-pip
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip
+          restore-keys: |
+            ${{ runner.os }}-pip
+
+      - name: Install pip
+        run: python -m pip install --upgrade pip
+
+      - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@v2
+
+      - name: Install nf-core tools development version
+        run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
+
+      - name: Lint module ${{ matrix.tags }}
+        run: nf-core modules lint ${{ matrix.tags }}
+
+  nf-core-lint-subworkflows:
+    runs-on: ubuntu-latest
+    name: nf-core lint subworkflows
+    needs: nf-core-changes
+    if: ${{ (needs.nf-core-changes.outputs.subworkflows != '[]') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: "${{ fromJson(needs.nf-core-changes.outputs.subworkflows) }}"
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - name: Set up Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
+        with:
+          python-version: "3.11"
+
+      - name: Install pip
+        run: python -m pip install --upgrade pip
+
+      - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2
+
+      - name: Install nf-core tools development version
+        run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
+
+      - name: Lint module ${{ matrix.tags }}
+        run: nf-core subworkflows lint ${{ matrix.tags }}
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-core-lint-modules, nf-core-lint-subworkflows]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"