Merge pull request #1581 from nf-core/dev

Dev -> Master for v2.4 release

.github/workflows/create-lint-wf.yml

@@ -27,7 +27,7 @@ jobs:
 
       # Set up nf-core/tools
       - name: Set up Python 3.8
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: 3.8
 

.github/workflows/create-test-wf.yml

@@ -25,7 +25,7 @@ jobs:
         name: Check out source-code repository
 
       - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: 3.7
 

.github/workflows/deploy-pypi.yml

@@ -12,7 +12,7 @@ jobs:
         name: Check out source-code repository
 
       - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: 3.7
 

.github/workflows/fix-linting.yml

@@ -0,0 +1,49 @@
+name: Fix linting from a comment
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  deploy:
+    # Only run if comment is on a PR with the main repo, and if it contains the magic keywords
+    if: >
+      contains(github.event.comment.html_url, '/pull/') &&
+      contains(github.event.comment.body, '@nf-core-bot fix linting') &&
+      github.repository == 'nf-core/tools'
+    runs-on: ubuntu-latest
+    steps:
+      # Use the @nf-core-bot token to check out so we can push later
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.nf_core_bot_auth_token }}
+
+      # Action runs on the issue comment, so we don't get the PR by default
+      # Use the gh cli to check out the PR
+      - name: Checkout Pull Request
+        run: gh pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
+
+      - uses: actions/setup-node@v2
+
+      - name: Install Prettier
+        run: npm install -g prettier @prettier/plugin-php
+
+      - name: Run 'prettier --write'
+        run: prettier --write ${GITHUB_WORKSPACE}
+
+      - name: Run Black
+        uses: psf/black@stable
+        with:
+          # Override to remove the default --check flag so that we make changes
+          options: "--color"
+
+      - name: Commit & push changes
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "nf-core-bot"
+          git config push.default upstream
+          git add .
+          git status
+          git commit -m "[automated] Fix linting with Prettier"
+          git push

.github/workflows/pytest.yml

@@ -20,7 +20,7 @@ jobs:
         name: Check out source-code repository
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/sync.yml

@@ -38,7 +38,7 @@ jobs:
           fetch-depth: "0"
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: 3.8
 

.github/workflows/tools-api-docs-dev.yml

@@ -12,7 +12,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: 3.7
 

.github/workflows/tools-api-docs-release.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: 3.7
 

.prettierignore

@@ -1,3 +1,4 @@
+email_template.html
 docs/api/_build
 testing
 

CHANGELOG.md

@@ -1,5 +1,46 @@
 # nf-core/tools: Changelog
 
+## [v2.4 - Cobolt Koala](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
+
+### Template
+
+- Read entire lines when sniffing the samplesheet format (fix [#1561](https://github.com/nf-core/tools/issues/1561))
+- Add actions workflow to respond to `@nf-core-bot fix linting` comments on pipeline PRs
+- Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509))
+- Fix bug in pipeline readme logo URL
+- Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system ([#1512](https://github.com/nf-core/tools/pull/1512)).
+- Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself
+- Add `.prettierignore` file to stop Prettier linting tests from running over test files
+- Made module template test command match the default used in `nf-core modules create-test-yml` ([#1562](https://github.com/nf-core/tools/issues/1562))
+- Removed black background from Readme badges now that GitHub has a dark mode, added Tower launch badge.
+- Don't save md5sum for `versions.yml` when running `nf-core modules create-test-yml` ([#1511](https://github.com/nf-core/tools/pull/1511))
+
+### General
+
+- Add actions workflow to respond to `@nf-core-bot fix linting` comments on nf-core/tools PRs
+- Use [`$XDG_CONFIG_HOME`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) or `~/.config/nf-core` instead of `~/.nfcore` for API cache (the latter can be safely deleted)
+- Consolidate GitHub API calls into a shared function that uses authentication from the [`gh` GitHub cli tool](https://cli.github.com/) or `GITHUB_AUTH_TOKEN` to avoid rate limiting ([#1499](https://github.com/nf-core/tools/pull/1499))
+- Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors.
+- Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419))
+- Bumped the minimum version of `rich` from `v10` to `v10.7.0`
+
+### Modules
+
+- Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image.
+- Add a new command `nf-core modules test` which runs pytests locally.
+- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520))
+- Allow follow links when generating `test.yml` file with `nf-core modules create-test-yml` ([1570](https://github.com/nf-core/tools/pull/1570))
+- Escaped test run output before logging it, to avoid a rich ` MarkupError`
+
+### Linting
+
+- Don't allow a `.nf-core.yaml` file, should be `.yml` ([#1515](https://github.com/nf-core/tools/pull/1515)).
+- `shell` blocks now recognised to avoid error `when: condition has too many lines` ([#1557](https://github.com/nf-core/tools/issues/1557))
+- Fixed error when using comments after `input` tuple lines ([#1542](https://github.com/nf-core/tools/issues/1542))
+- Don't lint the `shell` block when `script` is used ([1558](https://github.com/nf-core/tools/pull/1558))
+- Check that `template` is used in `script` blocks
+- Tweaks to CLI output display of lint results
+
 ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24]
 
 Very minor patch release to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release.

README.md

@@ -35,7 +35,9 @@ A python package with helper tools for the nf-core community.
   - [`modules create` - Create a module from the template](#create-a-new-module)
   - [`modules create-test-yml` - Create the `test.yml` file for a module](#create-a-module-test-config-file)
   - [`modules lint` - Check a module against nf-core guidelines](#check-a-module-against-nf-core-guidelines)
+  - [`modules test` - Run the tests for a module](#run-the-tests-for-a-module-using-pytest)
   - [`modules bump-versions` - Bump software versions of modules](#bump-bioconda-and-container-versions-of-modules-in)
+  - [`modules mulled` - Generate the name for a multi-tool container image](#generate-the-name-for-a-multi-tool-container-image)
 
 - [Citation](#citation)
 
@@ -1296,6 +1298,57 @@ INFO     Linting module: star/align
 ╰──────────────────────╯
 ```
 
+### Run the tests for a module using pytest
+
+To run unit tests of a module that you have installed or the test created by the command [`nf-core mdoules create-test-yml`](#create-a-module-test-config-file), you can use `nf-core modules test` command. This command runs the tests specified in `modules/tests/software/<tool>/<subtool>/test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/).
+
+You can specify the module name in the form TOOL/SUBTOOL in command line or provide it later by prompts.
+
+```console
+$ nf-core modules test fastqc
+                                          ,--./,-.
+          ___     __   __   __   ___     /,-._.--~\
+    |\ | |__  __ /  ` /  \ |__) |__         }  {
+    | \| |       \__, \__/ |  \ |___     \`-._,-`-,
+                                          `._,._,'
+
+    nf-core/tools version 2.4
+
+? Choose software profile Docker
+INFO     Setting environment variable '$PROFILE' to 'docker'
+INFO     Running pytest for module 'fastqc'
+
+=============================================================== test session starts ================================================================
+platform darwin -- Python 3.9.12, pytest-7.1.2, pluggy-1.0.0
+rootdir: ~/modules, configfile: pytest.ini
+plugins: workflow-1.6.0
+collecting ...
+collected 761 items
+
+fastqc single-end:
+        command:   nextflow run ./tests/modules/fastqc/ -entry test_fastqc_single_end -c ./tests/config/nextflow.config -c ./tests/modules/fastqc/nextflow.config -c ./tests/modules/fastqc/nextflow.config
+        directory: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end
+        stdout:    /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end/log.out
+        stderr:    /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end/log.err
+'fastqc single-end' done.
+
+fastqc paired-end:
+        command:   nextflow run ./tests/modules/fastqc/ -entry test_fastqc_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/fastqc/nextflow.config -c ./tests/modules/fastqc/nextflow.config
+        directory: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end
+        stdout:    /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end/log.out
+        stderr:    /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end/log.err
+'fastqc paired-end' done.
+
+tests/test_versions_yml.py sssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 17%]
+ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 38%]
+ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 59%]
+sssssssssssssssssssssssssssssssssssssssssssssssssssssssssss..ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss ssss [ 80%]
+ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss     [ 98%]
+tests/modules/fastqc/test.yml ........
+Keeping temporary directories and logs. Use '--kwd' or '--keep-workflow-wd' to disable this behaviour.
+=================================================== 10 passed, 751 skipped, 479 warnings in 50.76s ===================================================
+```
+
 ### Bump bioconda and container versions of modules in
 
 If you are contributing to the `nf-core/modules` repository and want to bump bioconda and container versions of certain modules, you can use the `nf-core modules bump-versions` helper tool. This will bump the bioconda version of a single or all modules to the latest version and also fetch the correct Docker and Singularity container tags.
@@ -1339,6 +1392,25 @@ bump-versions:
   star/align: "2.6.1d"
 ```
 
+### Generate the name for a multi-tool container image
+
+When you want to use an image of a multi-tool container and you know the specific dependencies and their versions of that container, for example, by looking them up in the [BioContainers hash.tsv](https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv), you can use the `nf-core modules mulled` helper tool. This tool generates the name of a BioContainers mulled image.
+
+```console
+$ nf-core modules mulled pysam==0.16.0.1 biopython==1.78
+
+                                          ,--./,-.
+          ___     __   __   __   ___     /,-._.--~\
+    |\ | |__  __ /  ` /  \ |__) |__         }  {
+    | \| |       \__, \__/ |  \ |___     \`-._,-`-,
+                                          `._,._,'
+
+    nf-core/tools version 2.4
+
+
+mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0
+```
+
 ## Citation
 
 If you use `nf-core tools` in your work, please cite the `nf-core` publication as follows:

nf_core/__main__.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 """ nf-core: Helper tools for use with nf-core Nextflow pipelines. """
-
 from rich import print
 import logging
 import os
@@ -49,7 +48,7 @@
         },
         {
             "name": "Developing new modules",
-            "commands": ["create", "create-test-yml", "lint", "bump-versions"],
+            "commands": ["create", "create-test-yml", "lint", "bump-versions", "mulled", "test"],
         },
     ],
 }
@@ -674,6 +673,69 @@ def bump_versions(ctx, tool, dir, all, show_all):
         sys.exit(1)
 
 
+# nf-core modules mulled
+@modules.command()
+@click.argument("specifications", required=True, nargs=-1, metavar="<tool==version> <...>")
+@click.option(
+    "--build-number",
+    type=int,
+    default=0,
+    show_default=True,
+    metavar="<number>",
+    help="The build number for this image. This is an incremental value that starts at zero.",
+)
+def mulled(specifications, build_number):
+    """
+    Generate the name of a BioContainers mulled image version 2.
+
+    When you know the specific dependencies and their versions of a multi-tool container image and you need the name of
+    that image, this command can generate it for you.
+
+    """
+    from nf_core.modules.mulled import MulledImageNameGenerator
+
+    try:
+        image_name = MulledImageNameGenerator.generate_image_name(
+            MulledImageNameGenerator.parse_targets(specifications), build_number=build_number
+        )
+    except ValueError as e:
+        log.error(e)
+        sys.exit(1)
+    if not MulledImageNameGenerator.image_exists(image_name):
+        log.error("The generated multi-tool container image name does not seem to exist yet.")
+        log.info(
+            "Please double check that your provided combination of tools and versions exists in the file: "
+            "[link=https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv]BioContainers/multi-package-containers 'combinations/hash.tsv'[/link]"
+        )
+        log.info(
+            "If it does not, please add your desired combination as detailed at: "
+            "https://github.com/BioContainers/multi-package-containers"
+        )
+        sys.exit(1)
+    log.info("Mulled container hash:")
+    print(image_name)
+
+
+# nf-core modules test
+@modules.command("test")
+@click.pass_context
+@click.argument("tool", type=str, required=False, metavar="<tool> or <tool/subtool>")
+@click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting")
+@click.option("-a", "--pytest_args", type=str, required=False, multiple=True, help="Additional pytest arguments")
+def test_module(ctx, tool, no_prompts, pytest_args):
+    """
+    Run module tests locally.
+
+    Given the name of a module, runs the Nextflow test command.
+    """
+    try:
+        meta_builder = nf_core.modules.ModulesTest(tool, no_prompts, pytest_args)
+        meta_builder.run()
+    except UserWarning as e:
+        log.critical(e)
+        sys.exit(1)
+
+
 # nf-core schema subcommands
 @nf_core_cli.group()
 def schema():

nf_core/create.py

@@ -39,8 +39,8 @@ def __init__(self, name, description, author, version="1.0dev", no_git=False, fo
         self.name = f"nf-core/{self.short_name}"
         self.name_noslash = self.name.replace("/", "-")
         self.name_docker = self.name.replace("nf-core", "nfcore")
-        self.logo_light = f"{self.name}_logo_light.png"
-        self.logo_dark = f"{self.name}_logo_dark.png"
+        self.logo_light = f"{self.name_noslash}_logo_light.png"
+        self.logo_dark = f"{self.name_noslash}_logo_dark.png"
         self.description = description
         self.author = author
         self.version = version

nf_core/gitpod/gitpod.Dockerfile

@@ -26,13 +26,12 @@ RUN conda update -n base -c defaults conda && \
     conda config --add channels conda-forge && \
     conda install \
         openjdk=11.0.13 \
-        nextflow=21.10.6 \
+        nextflow=22.04.0 \
         pytest-workflow=1.6.0 \
-        mamba=0.22.1 \
+        mamba=0.23.1 \
         pip=22.0.4 \
         black=22.1.0 \
         -n base && \
-    nextflow self-update && \
     conda clean --all -f -y
 
 # Install nf-core

nf_core/launch.py

@@ -420,7 +420,7 @@ def prompt_param(self, param_id, param_obj, is_required, answers):
 
         # If required and got an empty reponse, ask again
         while type(answer[param_id]) is str and answer[param_id].strip() == "" and is_required:
-            log.error("'–-{}' is required".format(param_id))
+            log.error("'--{}' is required".format(param_id))
             answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style)
 
         # Ignore if empty
@@ -692,6 +692,7 @@ def build_command(self):
             if self.use_params_file:
                 with open(self.params_out, "w") as fp:
                     json.dump(self.schema_obj.input_params, fp, indent=4)
+                    fp.write("\n")
                 self.nextflow_cmd += ' {} "{}"'.format("-params-file", os.path.relpath(self.params_out))
 
             # Call nextflow with a list of command line flags