Merge branch 'dev' into renovate/prompt_toolkit-3.x

.github/workflows/create-lint-wf.yml

@@ -38,7 +38,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "23.10.0"
           - "latest-everything"
     steps:
       - name: go to subdirectory and change nextflow workdir

.github/workflows/create-test-lint-wf-template.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Create Matrix
         id: create_matrix
         run: |
-          echo "matrix=$(yq 'keys | tojson(0)' nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT
+          echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/template_features.yml)" >> $GITHUB_OUTPUT
 
   RunTestWorkflow:
     runs-on: ${{ matrix.runner }}
@@ -66,8 +66,6 @@ jobs:
             runner: ubuntu-latest
             profile: "docker"
         exclude:
-          - TEMPLATE: github
-          - TEMPLATE: is_nfcore
           - TEMPLATE: nf_core_configs
             profile: "self_hosted_runner"
       fail-fast: false

.github/workflows/create-test-wf.yml

@@ -39,7 +39,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "23.10.0"
           - "latest-everything"
     steps:
       - name: go to working directory

.gitpod.yml

@@ -6,17 +6,14 @@ tasks:
       python -m pip install -r requirements-dev.txt
       pre-commit install --install-hooks
       nextflow self-update
-  - name: unset JAVA_TOOL_OPTIONS
-    command: |
-      unset JAVA_TOOL_OPTIONS
 
 vscode:
-  extensions: # based on nf-core.nf-core-extensionpack
+  extensions:
     - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code
     - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files
     - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar
     - mechatroner.rainbow-csv # Highlight columns in csv files in different colors
-    # - nextflow.nextflow                    # Nextflow syntax highlighting
+    - nextflow.nextflow # Nextflow syntax highlighting
     - oderwat.indent-rainbow # Highlight indentation level
     - streetsidesoftware.code-spell-checker # Spelling checker for source code
     - charliermarsh.ruff # Code linter Ruff

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.6
+    rev: v0.6.0
     hooks:
       - id: ruff # linter
         args: [--fix, --exit-non-zero-on-fix] # sort imports and fix

CHANGELOG.md

@@ -1,6 +1,18 @@
 # nf-core/tools: Changelog
 
-## v2.14.2dev
+## v3.0.0dev
+
+**Highlights**
+
+- Pipeline commands are renamed from `nf-core <comand>` to `nf-core pipelines <command>` to follow the same command structure as modules and subworkflows commands.
+- More customisation for pipeline templates. The template has been divided into features which can be skipped, e.g. you can create a new pipeline without any traces of FastQC in it.
+- A new Text User Interface app when running `nf-core pipelines create` to help us guide you through the process better (no worries, you can still use the cli if you give all values as parameters)
+- We replaced nf-validation with nf-schema in the pipeline template
+- CI tests now lint with the nf-core tools version matching the template version of the pipeline, to minimise errors in opened PRs with every new tools release.
+- New command `nf-core pipelines ro-crate` to create a [Research Object (RO) crate](https://www.researchobject.org/ro-crate/) for a pipeline
+- `nf-core licences` command is deprecated.
+- The structure of nf-core/tools pytests has been updated
+- The structure of the API docs has been updated
 
 ### Template
 
@@ -9,66 +21,79 @@
 - Remove deprecated syntax ([#3046](https://github.com/nf-core/tools/pull/3046))
 - Use filename in code block for `params.yml` ([#3055](https://github.com/nf-core/tools/pull/3055))
 - Remove release announcement for non nf-core pipelines ([#3072](https://github.com/nf-core/tools/pull/3072))
+- handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108), [#3112](https://github.com/nf-core/tools/pull/3112))
 - add option to exclude code linters for custom pipeline template ([#3084](https://github.com/nf-core/tools/pull/3084))
 - add option to exclude citations for custom pipeline template ([#3101](https://github.com/nf-core/tools/pull/3101))
 - add option to exclude gitpod for custom pipeline template ([#3100](https://github.com/nf-core/tools/pull/3100))
 - add option to exclude codespaces from pipeline template ([#3105](https://github.com/nf-core/tools/pull/3105))
 - add option to exclude multiqc from pipeline template ([#3103](https://github.com/nf-core/tools/pull/3103))
 - add option to exclude changelog from custom pipeline template ([#3104](https://github.com/nf-core/tools/pull/3104))
-- handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108))
-- add templatefeatures.yml to python package ([#3112](https://github.com/nf-core/tools/pull/3112))
+- add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125))
+- add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126))
+- Use nf-schema instead of nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116))
+- add option to exclude nf-schema from the template ([#3116](https://github.com/nf-core/tools/pull/3116))
+- add option to exclude fastqc from pipeline template ([#3129](https://github.com/nf-core/tools/pull/3129))
+- add option to exclude documentation from pipeline template ([#3130](https://github.com/nf-core/tools/pull/3130))
+- add option to exclude test configs from pipeline template ([#3133](https://github.com/nf-core/tools/pull/3133))
+- add option to exclude tower.yml from pipeline template ([#3134](https://github.com/nf-core/tools/pull/3134))
+- run nf-core lint `--release` on PRs to master ([#3148](https://github.com/nf-core/tools/pull/3148))
+- Add tests to ensure all files are part of a template customisation group and all groups are tested ([#3099](https://github.com/nf-core/tools/pull/3099))
 
 ### Linting
 
 - Fix linting fail on nfcore_external_java_deps if nf_schema is used ([#2976](https://github.com/nf-core/tools/pull/2976))
 - Conda module linting: Include package name in log file ([#3014](https://github.com/nf-core/tools/pull/3014))
+- Remove defaults from conda `environment.yml` file. ([#3029](https://github.com/nf-core/tools/pull/3029))
 - Restructure pipeline tests and move pipeline linting into subfolder ([#3070](https://github.com/nf-core/tools/pull/3070))
 - Fix module linting warning for process_high_memory ([#3086](https://github.com/nf-core/tools/issues/3086))
+- Linting will now fail when an unpinned plugin is used ([#3116](https://github.com/nf-core/tools/pull/3116))
+- Linting will now check if the schema is correct for the used validation plugin ([#3116])(https://github.com/nf-core/tools/pull/3116)
+- Linting will now check the use of the right validation plugin include statements in the workflow scripts ([#3116])(https://github.com/nf-core/tools/pull/3116)
+- Full linting for correct use of nf-schema and nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116))
+- Handle cases where the directory path contains the name of the component ([#3147](https://github.com/nf-core/tools/pull/3147))
+- Don't test conda `environment.yml` `name` attribute (which should no longer be there) ([#3161](https://github.com/nf-core/tools/pull/3161))
 
-### Download
+### Pipeline create command
+
+- Create: allow more special characters on the pipeline name for non-nf-core pipelines ([#3008](https://github.com/nf-core/tools/pull/3008))
+- Create: Mock git cretentials to generate stable textual snapshots ([#3007](https://github.com/nf-core/tools/pull/3007))
+- Create app: display input textbox with equally spaced grid ([#3038](https://github.com/nf-core/tools/pull/3038))
+- Pipelines: allow numbers in custom pipeline name ([#3094](https://github.com/nf-core/tools/pull/3094))
 
 ### Components
 
 - The `modules_nfcore` tag in the `main.nf.test` file of modules/subworkflows now displays the organization name in custom modules repositories ([#3005](https://github.com/nf-core/tools/pull/3005))
+- Add `--migrate_pytest` option to `nf-core <modules|subworkflows> test` command ([#3085](https://github.com/nf-core/tools/pull/3085))
+- Components: allow spaces at the beginning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115))
 
 ### General
 
-- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.4 ([#2974](https://github.com/nf-core/tools/pull/2974))
-- Update gitpod/workspace-base Docker digest to 92dd1bc ([#2982](https://github.com/nf-core/tools/pull/2982))
 - Update output of generation script for API docs to new structure ([#2988](https://github.com/nf-core/tools/pull/2988))
 - Add no clobber and put bash options on their own line ([#2991](https://github.com/nf-core/tools/pull/2991))
 - update minimal textual version and snapshots ([#2998](https://github.com/nf-core/tools/pull/2998))
 - move pipeline subcommands for v3.0 ([#2983](https://github.com/nf-core/tools/pull/2983))
 - return directory if base_dir is the root directory ([#3003](https://github.com/nf-core/tools/pull/3003))
-- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.6 ([#3006](https://github.com/nf-core/tools/pull/3006))
-- Create: allow more special characters on the pipeline name for non-nf-core pipelines ([#3008](https://github.com/nf-core/tools/pull/3008))
 - Remove nf-core licences command ([#3012](https://github.com/nf-core/tools/pull/3012))
 - README - absolute image paths ([#3013](https://github.com/nf-core/tools/pull/3013))
-- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.7 ([#3015](https://github.com/nf-core/tools/pull/3015))
-- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.8 ([#3017](https://github.com/nf-core/tools/pull/3017))
-- Update python:3.12-slim Docker digest to e3ae8cf ([#3020](https://github.com/nf-core/tools/pull/3020))
-- Update python:3.12-slim Docker digest to 2fba8e7 ([#3023](https://github.com/nf-core/tools/pull/3023))
-- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.10 ([#3031](https://github.com/nf-core/tools/pull/3031))
 - Add warning deprecation message to top-level commands ([#3036](https://github.com/nf-core/tools/pull/3036))
-- Create: Mock git cretentials to generate stable textual snapshots ([#3007](https://github.com/nf-core/tools/pull/3007))
 - move pipeline commands to functions to avoid duplication ([#3039](https://github.com/nf-core/tools/pull/3039))
-- Create app: display input textbox with equally spaced grid ([#3038](https://github.com/nf-core/tools/pull/3038))
-- Update python:3.12-slim Docker digest to da2d7af ([#3041](https://github.com/nf-core/tools/pull/3041))
-- Update gitpod/workspace-base Docker digest to 0f38224 ([#3048](https://github.com/nf-core/tools/pull/3048))
 - update output_dir for api docs to new website structure ([#3051](https://github.com/nf-core/tools/pull/3051))
-- Update pre-commit hook astral-sh/ruff-pre-commit to v0.5.1 ([#3052](https://github.com/nf-core/tools/pull/3052))
 - Add `--limit-output` argument for modules/subworkflow update ([#3047](https://github.com/nf-core/tools/pull/3047))
 - update api docs to new structure ([#3054](https://github.com/nf-core/tools/pull/3054))
-- Update to pytest v8 and move it to dev dependencies ([#3058](https://github.com/nf-core/tools/pull/3058))
 - handle new jsonschema error type ([#3061](https://github.com/nf-core/tools/pull/3061))
-- Update python:3.12-slim Docker digest to f11725a ([#3071](https://github.com/nf-core/tools/pull/3071))
 - Fix number of arguments for pipelines_create within the command_create function ([#3074](https://github.com/nf-core/tools/pull/3074))
-- Update python:3.12-slim Docker digest to 740d94a ([#3079](https://github.com/nf-core/tools/pull/3079))
-- Add `--migrate_pytest` option to `nf-core <modules|subworkflows> test` command ([#3085](https://github.com/nf-core/tools/pull/3085))
-- Update pre-commit hook pre-commit/mirrors-mypy to v1.11.1 ([#3091](https://github.com/nf-core/tools/pull/3091))
-- Pipelines: allow numbers in custom pipeline name ([#3094](https://github.com/nf-core/tools/pull/3094))
 - Add bot action to update textual snapshots and write bot documentation ([#3102](https://github.com/nf-core/tools/pull/3102))
-- Components: allow spaces at the betinning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115))
+- Update gitpod setup ([#3136](https://github.com/nf-core/tools/pull/3136))
+- fix syncing a pipeline from current directory ([#3143](https://github.com/nf-core/tools/pull/3143))
+- Patch gitpod conda setup to not use defaults channel ([#3159](https://github.com/nf-core/tools/pull/3159))
+
+## Version updates
+
+- Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 ([#3122](https://github.com/nf-core/tools/pull/3122))
+- Update gitpod/workspace-base Docker digest to 92dd1bc ([#2982](https://github.com/nf-core/tools/pull/2982))
+- Update python:3.12-slim Docker digest to 59c7332 ([#3124](https://github.com/nf-core/tools/pull/3124))
+- Update pre-commit hook pre-commit/mirrors-mypy to v1.11.1 ([#3091](https://github.com/nf-core/tools/pull/3091))
+- Update to pytest v8 and move it to dev dependencies ([#3058](https://github.com/nf-core/tools/pull/3058))
 - Update dependency prompt_toolkit to <=3.0.47 ([#3121](https://github.com/nf-core/tools/pull/3121))
 
 ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09]

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim@sha256:740d94a19218c8dd584b92f804b1158f85b0d241e5215ea26ed2dcade2b9d138
+FROM python:3.12-slim@sha256:59c7332a4a24373861c4a5f0eec2c92b87e3efeb8ddef011744ef9a751b1d11c
 LABEL authors="[email protected],[email protected]" \
     description="Docker image containing requirements for nf-core/tools"
 

MANIFEST.in

@@ -10,4 +10,4 @@ include nf_core/assets/logo/nf-core-repo-logo-base-darkbg.png
 include nf_core/assets/logo/placeholder_logo.svg
 include nf_core/assets/logo/MavenPro-Bold.ttf
 include nf_core/pipelines/create/create.tcss
-include nf_core/pipelines/create/templatefeatures.yml
+include nf_core/pipelines/create/template_features.yml

docs/api/_src/pipeline_lint_tests/plugin_includes.md

@@ -0,0 +1,5 @@
+# plugin_includes
+
+```{eval-rst}
+.. automethod:: nf_core.pipelines.lint.PipelineLint.plugin_includes
+```

nf_core/__main__.py

@@ -1136,6 +1136,13 @@ def command_modules_create(
 @modules.command("test")
 @click.pass_context
 @click.argument("tool", type=str, callback=normalize_case, required=False, metavar="<tool> or <tool/subtool>")
+@click.option(
+    "-v",
+    "--verbose",
+    is_flag=True,
+    default=False,
+    help="Print verbose output to the console. Sets `--debug` inside the nf-test command.",
+)
 @click.option(
     "-d",
     "--dir",
@@ -1171,10 +1178,12 @@ def command_modules_create(
     default=False,
     help="Migrate a module with pytest tests to nf-test",
 )
-def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest):
+def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest, verbose):
     """
     Run nf-test for a module.
     """
+    if verbose:
+        ctx.obj["verbose"] = verbose
     modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest)
 
 

nf_core/commands_pipelines.py

@@ -144,9 +144,12 @@ def pipelines_lint(
             ctx.obj["hide_progress"],
         )
         swf_failed = 0
+        module_failed = 0
         if subworkflow_lint_obj is not None:
             swf_failed = len(subworkflow_lint_obj.failed)
-        if len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed > 0:
+        if module_lint_obj is not None:
+            module_failed = len(module_lint_obj.failed)
+        if len(lint_obj.failed) + module_failed + swf_failed > 0:
             sys.exit(1)
     except AssertionError as e:
         log.critical(e)

nf_core/components/lint/__init__.py

@@ -114,9 +114,7 @@ def __init__(
                         )
                     )
             if not self.all_remote_components:
-                raise LookupError(
-                    f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline."
-                )
+                log.warning(f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline.")
             local_component_dir = Path(self.directory, self.component_type, "local")
 
             if local_component_dir.exists():
@@ -146,7 +144,7 @@ def __init__(
             ]
             self.all_local_components = []
             if not self.all_remote_components:
-                raise LookupError(f"No {self.component_type} in '{self.component_type}' directory")
+                log.warning(f"No {self.component_type} in '{self.component_type}' directory")
 
             # This could be better, perhaps glob for all nextflow.config files in?
             self.config = nf_core.utils.fetch_wf_config(self.directory / "tests" / "config", cache_config=True)

nf_core/components/nfcore_component.py

@@ -64,7 +64,15 @@ def __init__(
             self.process_name = ""
             self.environment_yml: Optional[Path] = Path(self.component_dir, "environment.yml")
 
-            repo_dir = self.component_dir.parts[: self.component_dir.parts.index(self.component_name.split("/")[0])][-1]
+            component_list = self.component_name.split("/")
+
+            name_index = len(self.component_dir.parts) - 1 - self.component_dir.parts[::-1].index(component_list[0])
+            if len(component_list) != 1 and component_list[0] == component_list[1]:
+                # Handle cases where the subtool has the same name as the tool
+                name_index -= 1
+
+            repo_dir = self.component_dir.parts[:name_index][-1]
+
             self.org = repo_dir
             self.nftest_testdir = Path(self.component_dir, "tests")
             self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test")

nf_core/gitpod/gitpod.Dockerfile

@@ -1,6 +1,7 @@
 # Test build locally before making a PR
 #   docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile .
 
+# See https://docs.renovatebot.com/docker/#digest-pinning for why a digest is used.
 FROM gitpod/workspace-base@sha256:f189a4195c3861365356f9c1b438ab26fd88e1ff46ce2843afc62861fc982e0c
 
 USER root
@@ -40,12 +41,11 @@ RUN chown -R gitpod:gitpod /opt/conda /usr/src/nf_core
 
 # Change user to gitpod
 USER gitpod
-# Install nextflow, nf-core, Mamba, and pytest-workflow
-RUN conda config --add channels defaults && \
-    conda config --add channels bioconda && \
+# Install nextflow, nf-core, nf-test, and other useful tools
+RUN conda config --add channels bioconda && \
     conda config --add channels conda-forge && \
     conda config --set channel_priority strict && \
-    conda install --quiet --yes --name base \
+    conda install --quiet --yes --update-all --name base \
     nextflow \
     nf-test \
     prettier \
@@ -63,3 +63,4 @@ RUN nextflow self-update && \
 # Setup pdiff for nf-test diffs
 ENV NFT_DIFF="pdiff"
 ENV NFT_DIFF_ARGS="--line-numbers --expand-tabs=2"
+ENV JAVA_TOOL_OPTIONS=

nf_core/module-template/environment.yml

@@ -1,9 +1,7 @@
 ---
 # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json
-name: "{{ component_name_underscore }}"
 channels:
   - conda-forge
   - bioconda
-  - defaults
 dependencies:
   - "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}"

nf_core/modules/lint/__init__.py

@@ -110,7 +110,7 @@ def lint(
         """
         # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class
         # Prompt for module or all
-        if module is None and not all_modules:
+        if module is None and not all_modules and len(self.all_remote_components) > 0:
             questions = [
                 {
                     "type": "list",