Skip to content

DESeq2 and Limma can be a low-resource processes (#4423) #5

DESeq2 and Limma can be a low-resource processes (#4423)

DESeq2 and Limma can be a low-resource processes (#4423) #5

Workflow file for this run

name: Run tests
on:
push:
branches: [master]
pull_request:
branches: [master]
merge_group:
types: [checks_requested]
branches: [master]
# Cancel if a newer run is started
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
jobs:
prettier:
runs-on: self-hosted
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Install NodeJS
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install Prettier
run: npm install -g prettier
- name: Run Prettier --check
run: prettier --check .
editorconfig:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install editorconfig-checker
run: npm install -g editorconfig-checker
- name: Run ECLint check
run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test)
pytest-changes:
name: pytest-changes
runs-on: self-hosted
outputs:
# Expose matched filters as job 'modules' output variable
modules: ${{ steps.filter.outputs.changes }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2 # To retrieve the preceding commit.
# TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented
- uses: mirpedrol/paths-filter@main
id: filter
with:
filters: "tests/config/pytest_modules.yml"
token: ""
nf-test-changes:
name: nf-test-changes
runs-on: ubuntu-latest
outputs:
# Expose matched filters as job 'modules' output variable
modules: ${{ steps.filter.outputs.changes }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2 # To retrieve the preceding commit.
- name: Combine all tags.yml files
id: get_tags
run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml
- name: debug
run: cat .github/tags.yml
# TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented
- uses: mirpedrol/paths-filter@main
id: filter
with:
filters: ".github/tags.yml"
token: ""
nf-core-lint:
runs-on: self-hosted
name: nf-core-lint
needs: [pytest-changes, nf-test-changes]
if: ${{ (needs.pytest-changes.outputs.modules != '[]') || (needs.nf-test-changes.outputs.modules != '[]') }}
strategy:
fail-fast: false
matrix:
tags:
[
"${{ fromJson(needs.pytest-changes.outputs.modules) }}",
"${{ fromJson(needs.nf-test-changes.outputs.modules) }}",
]
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install pip
run: python -m pip install --upgrade pip
- uses: actions/setup-java@v3
with:
distribution: "temurin"
java-version: "17"
- name: Setup Nextflow
uses: nf-core/setup-nextflow@v1
- name: Install nf-core tools development version
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
- name: Lint module ${{ matrix.tags }}
run: nf-core modules lint ${{ matrix.tags }}
if: ${{ !startsWith(matrix.tags, 'subworkflows/') }}
- name: Remove substring from matrix.tags
id: remove_substring
run: echo subworkflow_names=$(echo "${{ matrix.tags }}" | sed 's/subworkflows\///g') >> $GITHUB_OUTPUT
- name: Lint subworkflow ${{ matrix.tags }}
run: nf-core subworkflows lint ${{steps.remove_substring.outputs.subworkflow_names}}
if: ${{ startsWith(matrix.tags, 'subworkflows/') }}
migration:
runs-on: ubuntu-latest
name: nf-test Migration Notice
needs: [pytest-changes]
if: needs.pytest-changes.outputs.modules != '[]'
# Needed for Deprecation comment
permissions:
pull-requests: write
steps:
- name: Migration Comment
uses: mshick/add-pr-comment@v2
if: ${{ github.event_name == 'pull_request' }}
with:
message: |
Hello!
We're in the process of migrating to [nf-test](https://code.askimed.com/nf-test)!🚀
We would appreciate this test being rewritten in nf-test.
We have documented a step-by-step process to do so [here](https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test).
If you aren't able to do that right now [raise an issue](https://github.com/nf-core/modules/issues/new?assignees=&labels=bug&projects=&template=bug_report.yml).
pytest:
runs-on: ubuntu-latest
name: pytest
needs: [pytest-changes]
if: needs.pytest-changes.outputs.modules != '[]'
strategy:
fail-fast: false
matrix:
tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"]
profile: ["docker", "singularity", "conda"]
exclude:
- tags: "nf-test"
- profile: "conda"
tags: annotsv
- profile: "conda"
tags: happy/sompy
- profile: "conda"
tags: cellpose
- profile: "conda"
tags: mcquant
- profile: "conda"
tags: bases2fastq
- profile: "conda"
tags: backsub
- profile: "conda"
tags: basicpy
- profile: "conda"
tags: bcl2fastq
- profile: "conda"
tags: bclconvert
- profile: "conda"
tags: bwa/aln
- profile: "conda"
tags: bwa/index
- profile: "conda"
tags: bwa/mem
- profile: "conda"
tags: bwa/sampe
- profile: "conda"
tags: bwa/samse
- profile: "conda"
tags: cellranger/count
- profile: "conda"
tags: cellranger/mkfastq
- profile: "conda"
tags: cellranger/mkgtf
- profile: "conda"
tags: cellranger/mkref
- profile: "conda"
tags: cellranger/mkvdjref
- profile: "conda"
tags: cellranger/multi
- profile: "conda"
tags: cellranger/vdj
- profile: "conda"
tags: cellrangeratac/count
- profile: "conda"
tags: cellrangeratac/mkfastq
- profile: "conda"
tags: cellrangeratac/mkref
- profile: "conda"
tags: coreograph
- profile: "conda"
tags: deepcell/mesmer
- profile: "conda"
tags: deepvariant
- profile: "conda"
tags: fastk/fastk
- profile: "conda"
tags: fastani
- profile: "conda"
tags: fastk/histex
- profile: "conda"
tags: fastk/merge
- profile: "conda"
tags: fcs/fcsadaptor
- profile: "conda"
tags: fcs/fcsgx
- profile: "conda"
tags: gatk4/cnnscorevariants
- profile: "conda"
tags: gatk4/determinegermlinecontigploidy
- profile: "conda"
tags: gatk4/germlinecnvcaller
- profile: "conda"
tags: gatk4/postprocessgermlinecnvcalls
- profile: "conda"
tags: genescopefk
- profile: "conda"
tags: hlala/preparegraph
- profile: "conda"
tags: ilastik/multicut
- profile: "conda"
tags: ilastik/pixelclassification
- profile: "conda"
tags: imputeme/vcftoprs
- profile: "conda"
tags: medaka
- profile: "conda"
tags: merquryfk/katcomp
- profile: "conda"
tags: merquryfk/katgc
- profile: "conda"
tags: merquryfk/merquryfk
- profile: "conda"
tags: merquryfk/ploidyplot
- profile: "conda"
tags: minimap2/align
- profile: "conda"
tags: mitohifi/mitohifi
- profile: "conda"
tags: mitohifi/findmitoreference
- profile: "conda"
tags: nanoplot
- profile: "conda"
tags: ncbitools/vecscreen
- profile: "conda"
tags: parabricks/applybqsr
- profile: "conda"
tags: parabricks/fq2bam
- profile: "conda"
tags: picard/collecthsmetrics
- profile: "conda"
tags: picard/collectwgsmetrics
- profile: "conda"
tags: scimap/mcmicro
- profile: "conda"
tags: sentieon/applyvarcal
- profile: "conda"
tags: sentieon/bwaindex
- profile: "conda"
tags: sentieon/bwamem
- profile: "conda"
tags: sentieon/datametrics
- profile: "conda"
tags: sentieon/dedup
- profile: "conda"
tags: sentieon/dnamodelapply
- profile: "conda"
tags: sentieon/dnascope
- profile: "conda"
tags: sentieon/gvcftyper
- profile: "conda"
tags: sentieon/haplotyper
- profile: "conda"
tags: sentieon/readwriter
- profile: "conda"
tags: sentieon/tnfilter
- profile: "conda"
tags: sentieon/tnhaplotyper2
- profile: "conda"
tags: sentieon/tnscope
- profile: "conda"
tags: sentieon/varcal
- profile: "conda"
tags: sentieon/wgsmetrics
- profile: "conda"
tags: spaceranger/count
- profile: "conda"
tags: spaceranger/mkgtf
- profile: "conda"
tags: spaceranger/mkref
- profile: "conda"
tags: universc
- profile: "singularity"
tags: universc
- profile: "conda"
tags: vt/decompose
- profile: "singularity"
tags: gatk4/determinegermlinecontigploidy
- profile: "conda"
tags: subworkflows/bcl_demultiplex
- profile: "conda"
tags: subworkflows/fasta_clean_fcs
- profile: "conda"
tags: islandpath
- profile: "conda"
tags: subworkflows/bam_qc_picard
env:
NXF_ANSI_LOG: false
SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Install Python dependencies
run: python -m pip install --upgrade pip pytest-workflow cryptography
- uses: actions/setup-java@v3
with:
distribution: "temurin"
java-version: "17"
- name: Setup Nextflow ${{ matrix.NXF_VER }}
uses: nf-core/setup-nextflow@v1
with:
version: "${{ matrix.NXF_VER }}"
- name: Setup apptainer
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-apptainer@main
- name: Set up Singularity
if: matrix.profile == 'singularity'
run: |
mkdir -p $NXF_SINGULARITY_CACHEDIR
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
- name: Set up miniconda
uses: conda-incubator/setup-miniconda@v2
with:
miniconda-version: "latest"
channels: conda-forge,bioconda,defaults
python-version: ${{ matrix.python-version }}
- name: Conda setup
run: |
conda clean -a
conda install -n base conda-libmamba-solver
conda config --set solver libmamba
echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
echo $(realpath python) >> $GITHUB_PATH
# Set up secrets
- name: Set up nextflow secrets
if: env.SENTIEON_LICENSE_BASE64 != null
run: |
nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
SENTIEON_AUTH_DATA=$(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
# Test the module
- name: Run pytest-workflow
# only use one thread for pytest-workflow to avoid race condition on conda cache.
run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes
- name: Output log on failure
if: failure()
run: |
sudo apt-get update > /dev/null
sudo apt-get install bat > /dev/null
batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
- name: Setting global variables
uses: actions/github-script@v6
id: parsed
with:
script: |
return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"')
result-encoding: string
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v3
with:
name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }}
path: |
/home/runner/pytest_workflow_*/*/.nextflow.log
/home/runner/pytest_workflow_*/*/log.out
/home/runner/pytest_workflow_*/*/log.err
/home/runner/pytest_workflow_*/*/work
!/home/runner/pytest_workflow_*/*/work/conda
!/home/runner/pytest_workflow_*/*/work/singularity
!${{ github.workspace }}/.singularity
nf-test:
runs-on: ubuntu-latest
name: nf-test
needs: [nf-test-changes]
if: needs.nf-test-changes.outputs.modules != '[]'
strategy:
fail-fast: false
matrix:
tags: ["${{ fromJson(needs.nf-test-changes.outputs.modules) }}"]
profile: ["docker", "singularity", "conda"]
exclude:
- tags: "nf-test"
- profile: "conda"
tags: annotsv
- profile: "conda"
tags: happy/sompy
- profile: "conda"
tags: backsub
- profile: "conda"
tags: bases2fastq
- profile: "conda"
tags: bcl2fastq
- profile: "conda"
tags: bclconvert
- profile: "conda"
tags: cellranger/count
- profile: "conda"
tags: cellranger/mkfastq
- profile: "conda"
tags: cellranger/mkgtf
- profile: "conda"
tags: cellranger/mkref
- profile: "conda"
tags: custom/dumpsoftwareversions
- profile: "conda"
tags: deepvariant
- profile: "conda"
tags: ensemblvep/vep
- profile: "conda"
tags: fastk/fastk
- profile: "conda"
tags: fastk/histex
- profile: "conda"
tags: fastk/merge
- profile: "conda"
tags: fcs/fcsadaptor
- profile: "conda"
tags: fcs/fcsgx
- profile: "conda"
tags: gatk4/cnnscorevariants
- profile: "conda"
tags: gatk4/determinegermlinecontigploidy
- profile: "conda"
tags: genescopefk
- profile: "conda"
tags: ilastik/multicut
- profile: "conda"
tags: ilastik/pixelclassification
- profile: "conda"
tags: imputeme/vcftoprs
- profile: "conda"
tags: merquryfk/katcomp
- profile: "conda"
tags: merquryfk/katgc
- profile: "conda"
tags: merquryfk/merquryfk
- profile: "conda"
tags: merquryfk/ploidyplot
- profile: "conda"
tags: sentieon/bwaindex
- profile: "conda"
tags: sentieon/bwamem
- profile: "conda"
tags: universc
- profile: "singularity"
tags: universc
- profile: "conda"
tags: vt/decompose
- profile: "conda"
tags: subworkflows/vcf_annotate_ensemblvep
env:
NXF_ANSI_LOG: false
SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
NFTEST_VER: "0.8.1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
distribution: "temurin"
java-version: "17"
- name: Setup Nextflow
uses: nf-core/setup-nextflow@v1
- name: Cache nf-test installation
id: cache-software
uses: actions/cache@v3
with:
path: |
/usr/local/bin/nf-test
/home/runner/.nf-test/nf-test.jar
key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
- name: Install nf-test
if: steps.cache-software.outputs.cache-hit != 'true'
run: |
wget -qO- https://code.askimed.com/install/nf-test | bash
sudo mv nf-test /usr/local/bin/
- name: Setup apptainer
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-apptainer@main
- name: Set up Singularity
if: matrix.profile == 'singularity'
run: |
mkdir -p $NXF_SINGULARITY_CACHEDIR
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
- name: Set up miniconda
uses: conda-incubator/setup-miniconda@v2
with:
miniconda-version: "latest"
auto-update-conda: true
channels: conda-forge,bioconda,defaults
python-version: ${{ matrix.python-version }}
- name: Conda setup
run: |
conda clean -a
conda install -n base conda-libmamba-solver
conda config --set solver libmamba
echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
echo $(realpath python) >> $GITHUB_PATH
# Set up secrets
- name: Set up nextflow secrets
if: env.SENTIEON_LICENSE_BASE64 != null
run: |
nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
SENTIEON_AUTH_DATA=$(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
# Test the module
- name: Run nf-test
run: |
NXF_TEST_DIR=~ \
nf-test test \
--profile=${{ matrix.profile }} \
--tag ${{ matrix.tags }} \
--tap=test.tap \
--verbose
- uses: pcolby/tap-summary@v1
with:
path: >-
test.tap
confirm-pass:
runs-on: self-hosted
needs: [prettier, editorconfig, pytest-changes, nf-core-lint, pytest, nf-test-changes, nf-test]
if: always()
steps:
- name: All tests ok
if: ${{ success() || !contains(needs.*.result, 'failure') }}
run: exit 0
- name: One or more tests failed
if: ${{ contains(needs.*.result, 'failure') }}
run: exit 1
- name: debug-print
if: always()
run: |
echo "toJSON(needs) = ${{ toJSON(needs) }}"
echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"