Now using nf-test for pipeline level testing

Plant-Food-Research-Open · Oct 10, 2024 · 97928c3 · 97928c3
1 parent 66f76ce
commit 97928c3
Show file tree

Hide file tree

Showing 28 changed files with 1,023 additions and 129 deletions.
diff --git a/.github/include.yaml b/.github/include.yaml
@@ -0,0 +1,10 @@
+".":
+  - ./.github/workflows/**
+  - ./nf-test.config
+  - ./nextflow.config
+tests:
+  - ./assets/*
+  - ./bin/*
+  - ./conf/*
+  - ./main.nf
+  - ./nextflow_schema.json
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,88 +1,106 @@
 name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
-  push:
-    branches:
-      - dev
   pull_request:
-  release:
-    types: [published]
-  workflow_dispatch:
 
 env:
   NXF_ANSI_LOG: false
+  NFT_VER: "0.9.0"
+  NFT_WORKDIR: "~"
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
-
 jobs:
-  test:
-    name: Run pipeline with test data
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'Plant-Food-Research-Open/assemblyqc') }}"
+  nf-test-changes:
+    name: Check for changes
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        NXF_VER:
-          - "24.04.2"
-        TEST_PARAMS:
-          - minimal
-          - invalid
-          - stub
-          - noltr
-          - hicparam
-        include:
-          - OPTION_STUB: ""
-          - OPTION_STUB: "-stub"
-            TEST_PARAMS: stub
-          - OPTION_STUB: "-stub"
-            TEST_PARAMS: hicparam
+    outputs:
+      nf_test_files: ${{ steps.list.outputs.components }}
     steps:
-      - name: Check out pipeline code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
-
-      - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v2
+      - uses: actions/[email protected]
         with:
-          version: ${{ matrix.NXF_VER }}
+          fetch-depth: 0
 
-      - name: Disk space cleanup
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/[email protected]
+        with:
+          head: ${{ github.sha }}
+          base: origin/${{ github.base_ref }}
+          include: .github/include.yaml
 
-      - name: Run pipeline with test data
+      - name: print list of nf-test files
         run: |
-          nextflow run \
-            ${GITHUB_WORKSPACE} \
-            -profile test,docker \
-            -params-file ./tests/${{ matrix.TEST_PARAMS }}/params.json \
-            ${{ matrix.OPTION_STUB }} \
-            --outdir ./results
-
-  nf-test:
-    name: Run nf-tests
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'Plant-Food-Research-Open/assemblyqc') }}"
+          echo ${{ steps.list.outputs.components }}
+
+  test:
+    name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }}
+    needs: [nf-test-changes]
+    if: needs.nf-test-changes.outputs.nf_test_files != '[]'
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
           - "24.04.2"
+
+        nf_test_files: ["${{ fromJson(needs.nf-test-changes.outputs.nf_test_files) }}"]
+        profile:
+          - "docker"
+
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        uses: actions/checkout@v4.2.1
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
-          version: ${{ matrix.NXF_VER }}
+          version: "${{ matrix.NXF_VER }}"
+
+      - uses: actions/[email protected]
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+
+      - name: Install pdiff to see diff between nf-test snapshots
+        run: |
+          python -m pip install --upgrade pip
+          pip install pdiff
+
+      - uses: nf-core/[email protected]
+
+      - name: Run nf-test
+        run: |
+          nf-test test --verbose ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}" --tap=test.tap
+
+      - uses: pcolby/[email protected]
+        with:
+          path: >-
+            test.tap
 
-      - name: Install nf-test
-        uses: nf-core/[email protected]
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
 
-      - name: Run nf-tests
+      - name: debug-print
+        if: always()
         run: |
-          nf-test \
-            test \
-            --verbose \
-            tests
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 1. Added Gfastats [#126](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/126)
 2. Updated nf-core/template to 3.0.1 [#149](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/149)
 3. Updated `samtools faidx` to 1.21
+4. Now using nf-test for pipeline level testing [#153](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/153)
 
 ### `Fixed`
 

diff --git a/nf-test.config b/nf-test.config
@@ -1,8 +1,10 @@
 config {
+    testsDir "."
+    workDir System.getenv("NFT_WORKDIR") ?: ".nf-test"
+    configFile "tests/nextflow.config"
 
-    testsDir "tests"
-    workDir ".nf-test"
-    configFile "nextflow.config"
-    profile ""
-
+    plugins {
+        load "[email protected]"
+        load "[email protected]"
+    }
 }
diff --git a/tests/README.md b/tests/README.md
@@ -14,14 +14,17 @@ Or using [singularity](https://docs.sylabs.io/guides/3.0/user-guide/installation
 nextflow run plant-food-research-open/assemblyqc -r main -profile singularity,test --outdir results
 ```
 
-## Local Testing
+## nf-test and Continuous Integration (CI)
 
-The test sets included in this directory can be executed by first downloading the pipeline from GitHub and then executing the following command:
+The GitHub [CI action](../.github/workflows/ci.yml) included with the pipeline continuously tests the pipeline using [nf-test](https://www.nf-test.com).
 
-```bash
-./main.nf -profile docker -params-file tests/minimal/params.json --outdir results
-```
+## Testing Merqury Datasets
 
-## Continuous Integration (CI)
+Three Merqury datasets are included here which can be tested by pointing to one of the parameters file.
 
-The GitHub [CI action](../.github/workflows/ci.yml) included with the pipeline continuously tests the pipeline with the various test sets listed in this directory.
+```bash
+./main.nf \
+  -profile <docker,singularity> \
+  -params-file tests/merqury/<mixed2x,phased2x,phased2x.mp>/params.json \
+  --outdir results
+```
diff --git a/tests/hicparam/assemblysheet.csv b/tests/hicparam/assemblysheet.csv
@@ -1,2 +1,2 @@
 tag,fasta
-test,tests/hicparam/test_genome.fa.gz
+test,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/genome.fasta
diff --git a/tests/hicparam/main.nf.test b/tests/hicparam/main.nf.test
@@ -0,0 +1,35 @@
+nextflow_pipeline {
+
+    name "Test with hic param"
+    script "main.nf"
+
+    test("hic param - stub") {
+
+        options '-stub'
+
+        when {
+            params {
+                input   = "$baseDir/tests/hicparam/assemblysheet.csv"
+                hic     = "$baseDir/tests/hicparam/hic/Dummy_hic_{1,2}.merged.fq.gz"
+                outdir  = "$outputDir"
+            }
+        }
+
+        then {
+            def stable_path = getAllFilesFromDir(params.outdir, false, ['pipeline_info/*.{html,json,txt,yml}', 'report.{html,json}'], null, ['**'])
+
+            assertAll(
+                { assert workflow.success},
+                { assert snapshot(
+                    [
+                        'successful tasks': workflow.trace.succeeded().size(),
+                        'versions': removeNextflowVersion("$outputDir/pipeline_info/software_versions.yml"),
+                        'stable paths': stable_path
+                    ]
+                ).match() }
+            )
+        }
+
+    }
+
+}
diff --git a/tests/hicparam/main.nf.test.snap b/tests/hicparam/main.nf.test.snap
@@ -0,0 +1,102 @@
+{
+    "hic param - stub": {
+        "content": [
+            {
+                "successful tasks": 21,
+                "versions": {
+                    "AGP2ASSEMBLY": {
+                        "juicebox_scripts": "0.1.0"
+                    },
+                    "ASSEMBLATHON_STATS": {
+                        "assemblathon_stats": "github/PlantandFoodResearch/assemblathon2-analysis/a93cba2"
+                    },
+                    "ASSEMBLY2BEDPE": {
+                        "python": "3.11.3",
+                        "pandas": "2.1.1"
+                    },
+                    "BWA_INDEX": {
+                        "bwa": "0.7.18-r1243-dirty"
+                    },
+                    "BWA_MEM": {
+                        "bwa": "0.7.18-r1243-dirty",
+                        "samtools": 1.2
+                    },
+                    "FASTAVALIDATOR": {
+                        "py_fasta_validator": 0.6
+                    },
+                    "FASTP": {
+                        "fastp": "0.23.4"
+                    },
+                    "FASTQC_RAW": {
+                        "fastqc": "0.12.1"
+                    },
+                    "FASTQC_TRIM": {
+                        "fastqc": "0.12.1"
+                    },
+                    "HIC2HTML": {
+                        "python": "3.11.3"
+                    },
+                    "HICQC": {
+                        "hic_qc.py": "0+untagged.261.g6881c33"
+                    },
+                    "JUICER_SORT": {
+                        "sort": 8.3
+                    },
+                    "MAKEAGPFROMFASTA": {
+                        "juicebox_scripts": "0.1.0"
+                    },
+                    "MATLOCK_BAM2_JUICER": {
+                        "matlock": 20181227
+                    },
+                    "RUNASSEMBLYVISUALIZER": {
+                        "run-assembly-visualizer.sh": "18 July 2016"
+                    },
+                    "SAMBLASTER": {
+                        "samblaster": "0.1.26",
+                        "samtools": "1.19.2"
+                    },
+                    "SAMTOOLS_FAIDX": {
+                        "samtools": 1.21
+                    },
+                    "SEQKIT_RMDUP": {
+                        "seqkit": "v2.8.0"
+                    },
+                    "SEQKIT_SORT": {
+                        "seqkit": "v2.8.0"
+                    },
+                    "TAG_ASSEMBLY": {
+                        "pigz": "2.3.4"
+                    },
+                    "Workflow": {
+                        "plant-food-research-open/assemblyqc": "v2.2.0+dev"
+                    }
+                },
+                "stable paths": [
+                    "test_stats.csv:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "test.agp.assembly:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "test.assembly.bedpe:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "Dummy_hic.fastp.html:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "Dummy_hic.fastp.json:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "Dummy_hic.fastp.log:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "Dummy_hic.paired.fail.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940",
+                    "Dummy_hic_1.fail.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940",
+                    "Dummy_hic_1.fastp.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940",
+                    "Dummy_hic_2.fail.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940",
+                    "Dummy_hic_2.fastp.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940",
+                    "Dummy_hic.html:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "Dummy_hic.zip:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "Dummy_hic.html:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "Dummy_hic.zip:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "Dummy_hic.on.test.pdf:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "test.hic:md5,d41d8cd98f00b204e9800998ecf8427e",
+                    "test.html:md5,bbd8f07f11522eb75bb9429e86e95713"
+                ]
+            }
+        ],
+        "meta": {
+            "nf-test": "0.9.0",
+            "nextflow": "24.04.4"
+        },
+        "timestamp": "2024-10-10T16:11:00.660108"
+    }
+}
diff --git a/tests/hicparam/params.json b/tests/hicparam/params.json
diff --git a/tests/hicparam/test_genome.fa.gz b/tests/hicparam/test_genome.fa.gz
diff --git a/tests/invalid/assemblysheet.csv b/tests/invalid/assemblysheet.csv
@@ -1,5 +1,5 @@
 tag,fasta,gff3
 FI1,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/003/814/445/GCA_003814445.1_ASM381444v1/GCA_003814445.1_ASM381444v1_genomic.fna.gz,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/003/814/445/GCA_003814445.1_ASM381444v1/GCA_003814445.1_ASM381444v1_genomic.gff.gz
 TT_2021a,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/021/950/295/GCA_021950295.1_ASM2195029v1/GCA_021950295.1_ASM2195029v1_genomic.fna.gz,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/003/814/445/GCA_003814445.1_ASM381444v1/GCA_003814445.1_ASM381444v1_genomic.gff.gz
-MISC,tests/invalid/invalid.fsa.gz
-DUPSEQ,tests/invalid/dupseq.fsa.gz
+MISC,https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/dev/tests/invalid/invalid.fsa.gz
+DUPSEQ,https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/dev/tests/invalid/dupseq.fsa.gz