From 1e47693b295eea4c2a5f7b6b0351305c10133a31 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 19 Jun 2024 15:03:44 +0200 Subject: [PATCH 001/151] Linting: Remove 'defaults' from conda environment.yml file. See nf-core/modules#5829 --- nf_core/module-template/environment.yml | 1 - .../pipeline-template/modules/nf-core/fastqc/environment.yml | 1 - .../pipeline-template/modules/nf-core/multiqc/environment.yml | 1 - nf_core/pipeline-template/nextflow.config | 2 +- .../subworkflows/nf-core/utils_nextflow_pipeline/main.nf | 2 +- tests/test_utils.py | 2 +- 6 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nf_core/module-template/environment.yml b/nf_core/module-template/environment.yml index dcf510affb..f234f85422 100644 --- a/nf_core/module-template/environment.yml +++ b/nf_core/module-template/environment.yml @@ -4,6 +4,5 @@ name: "{{ component_name_underscore }}" channels: - conda-forge - bioconda - - defaults dependencies: - "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/environment.yml index 1787b38a9a..0d5be45f26 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/environment.yml @@ -2,6 +2,5 @@ name: fastqc channels: - conda-forge - bioconda - - defaults dependencies: - bioconda::fastqc=0.12.1 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml index ca39fb67e2..329ddb4870 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml @@ -2,6 +2,5 @@ name: multiqc channels: - conda-forge - bioconda - - defaults dependencies: - bioconda::multiqc=1.21 diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2e6a56b001..6202aa8319 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -98,7 +98,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false - conda.channels = ['conda-forge', 'bioconda', 'defaults'] + conda.channels = ['conda-forge', 'bioconda'] apptainer.enabled = false } mamba { diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index ac31f28f66..e770d91b97 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -102,7 +102,7 @@ def checkCondaChannels() { // Check that all channels are present // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def required_channels_in_order = ['conda-forge', 'bioconda'] def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order diff --git a/tests/test_utils.py b/tests/test_utils.py index 85f4e3c548..48288be817 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -105,7 +105,7 @@ def test_load_pipeline_config(self): # def test_load_conda_env(self): # """Load the pipeline Conda environment.yml file""" # self.pipeline_obj._load_conda_environment() - # assert self.pipeline_obj.conda_config["channels"] == ["conda-forge", "bioconda", "defaults"] + # assert self.pipeline_obj.conda_config["channels"] == ["conda-forge", "bioconda"] def test_list_files_git(self): """Test listing pipeline files using `git ls`""" From d6191cce7843918cf256da0f9b0fe0940e813288 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 19 Jun 2024 13:05:27 +0000 Subject: [PATCH 002/151] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9da8c1ab24..3e2be8fd91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Fix linting fail on nfcore_external_java_deps if nf_schema is used ([#2976](https://github.com/nf-core/tools/pull/2976)) - Conda module linting: Include package name in log file ([#3014](https://github.com/nf-core/tools/pull/3014)) +- Remove defaults from conda environment.yml file. ([#3029](https://github.com/nf-core/tools/pull/3029)) ### Download From b2336adaf288e3c64d02c28fb7e3e58cd22bd4cc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 23 Jul 2024 14:24:21 +0200 Subject: [PATCH 003/151] remove remaining mention to 'defaults' --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 48d1c3ca3d..d5bf328b59 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -635,7 +635,7 @@ def anaconda_package(dep, dep_channels=None): """ if dep_channels is None: - dep_channels = ["conda-forge", "bioconda", "defaults"] + dep_channels = ["conda-forge", "bioconda"] # Check if each dependency is the latest available version if "=" in dep: From a754fe0845f0e28ae0398625d8132a6047aa991e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 15:07:38 +0200 Subject: [PATCH 004/151] add schema checking for draft 2020-12 --- nf_core/pipelines/lint/schema_lint.py | 32 ++++++++++++++++++++++----- nf_core/pipelines/schema.py | 21 +++++++++++++----- requirements.txt | 2 +- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/nf_core/pipelines/lint/schema_lint.py b/nf_core/pipelines/lint/schema_lint.py index 6786c5012d..4007bf8fe5 100644 --- a/nf_core/pipelines/lint/schema_lint.py +++ b/nf_core/pipelines/lint/schema_lint.py @@ -16,26 +16,26 @@ def schema_lint(self): The lint test checks the schema for the following: * Schema should be a valid JSON file - * Schema should adhere to `JSONSchema `_, Draft 7. + * Schema should adhere to `JSONSchema `_, Draft 7 or Draft 2020-12. * Parameters can be described in two places: * As ``properties`` in the top-level schema object - * As ``properties`` within subschemas listed in a top-level ``definitions`` objects + * As ``properties`` within subschemas listed in a top-level ``definitions``(draft 7) or ``$defs``(draft 2020-12) objects * The schema must describe at least one parameter * There must be no duplicate parameter IDs across the schema and definition subschema - * All subschema in ``definitions`` must be referenced in the top-level ``allOf`` key + * All subschema in ``definitions`` or ``$defs`` must be referenced in the top-level ``allOf`` key * The top-level ``allOf`` key must not describe any non-existent definitions * Default parameters in the schema must be valid * Core top-level schema attributes should exist and be set as follows: - * ``$schema``: ``https://json-schema.org/draft-07/schema`` + * ``$schema``: ``https://json-schema.org/draft-07/schema`` or ``https://json-schema.org/draft/2020-12/schema`` * ``$id``: URL to the raw schema file, eg. ``https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json`` * ``title``: ``YOURPIPELINE pipeline parameters`` * ``description``: The pipeline config ``manifest.description`` * That the ``input`` property is defined and has a mimetype. A list of common mimetypes can be found `here `_. - For example, an *extremely* minimal schema could look like this: + For example, an *extremely* minimal schema could look like this (draft 7): .. code-block:: json @@ -57,6 +57,28 @@ def schema_lint(self): "allOf": [{"$ref": "#/definitions/my_first_group"}] } + Or this (draft 2020-12): + + .. code-block:: json + + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "$defs": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{"$ref": "#/$defs/my_first_group"}] + } + .. tip:: You can check your pipeline schema without having to run the entire pipeline lint by running ``nf-core pipelines schema lint`` instead of ``nf-core pipelines lint`` """ diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 7f562bff38..854bddccc0 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -359,11 +359,22 @@ def validate_schema(self, schema=None): """ if schema is None: schema = self.schema - try: - jsonschema.Draft7Validator.check_schema(schema) - log.debug("JSON Schema Draft7 validated") - except jsonschema.exceptions.SchemaError as e: - raise AssertionError(f"Schema does not validate as Draft 7 JSON Schema:\n {e}") + + schema_draft = schema["$schema"] + if schema_draft == "https://json-schema.org/draft-07/schema": + try: + jsonschema.Draft7Validator.check_schema(schema) + log.debug("JSON Schema Draft7 validated") + except jsonschema.exceptions.SchemaError as e: + raise AssertionError(f"Schema does not validate as Draft 7 JSON Schema:\n {e}") + elif schema_draft == "https://json-schema.org/draft/2020-12/schema": + try: + jsonschema.Draft202012Validator.check_schema(schema) + log.debug("JSON Schema Draft2020-12 validated") + except jsonschema.exceptions.SchemaError as e: + raise AssertionError(f"Schema does not validate as Draft 2020-12 JSON Schema:\n {e}") + else: + raise AssertionError(f"Unsupported JSON schema draft detected: ${schema_draft}. Use draft 7 or 2020-12 instead.") param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) diff --git a/requirements.txt b/requirements.txt index fb658be2fb..eba6460f03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ filetype GitPython PyGithub jinja2 -jsonschema>=3.0 +jsonschema>=4.0 markdown>=3.3 packaging pillow From 464f16e982337699382e66e3b1da21af4f01dd7a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 15:26:52 +0200 Subject: [PATCH 005/151] update validate schema --- nf_core/pipelines/schema.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 854bddccc0..abdad719f6 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -378,17 +378,32 @@ def validate_schema(self, schema=None): param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) - for d_key, d_schema in schema.get("definitions", {}).items(): + schema_defs = dict() + defs_notation = "" + if "$defs" in schema: + schema_defs = schema.get("$defs", {}).items() + defs_notation = "$defs" + elif "definitions" in schema: + schema_defs = schema.get("definitions", {}).items() + defs_notation = "definitions" + elif "defs" in schema: + # nf-schema v2.0.0 only supported defs. this has been changed to $defs in nf-schema v2.1.0 + # this line prevents the breakage of schemas created for v2.0.0 + schema_defs = schema.get("defs", {}).items() + defs_notation = "defs" + + for d_key, d_schema in schema_defs: # Check that this definition is mentioned in allOf if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") in_allOf = False for allOf in schema.get("allOf", []): - if allOf["$ref"] == f"#/definitions/{d_key}": + if allOf["$ref"] == f"#/{defs_notation}/{d_key}": in_allOf = True if not in_allOf: - raise AssertionError(f"Definition subschema `{d_key}` not included in schema `allOf`") + raise AssertionError(f"Definition subschema `#{defs_notation}/{d_key}` not included in schema `allOf`") + # TODO add support for nested parameters for d_param_id in d_schema.get("properties", {}): # Check that we don't have any duplicate parameter IDs in different definitions if d_param_id in param_keys: @@ -398,11 +413,11 @@ def validate_schema(self, schema=None): # Check that everything in allOf exists for allOf in schema.get("allOf", []): - if "definitions" not in schema: - raise AssertionError("Schema has allOf, but no definitions") - def_key = allOf["$ref"][14:] - if def_key not in schema.get("definitions", {}): - raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `definitions`") + _, allOf_defs_notation, def_key = allOf["$ref"].split("/") # "#//" + if allOf_defs_notation not in schema: + raise AssertionError(f"Schema has allOf, but no {allOf_defs_notation}") + if def_key not in schema.get(allOf_defs_notation, {}): + raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `{allOf_defs_notation}`") # Check that the schema describes at least one parameter if num_params == 0: From 6658449cc892021782a22885c3db63c0215254b6 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 15:51:27 +0200 Subject: [PATCH 006/151] update more code to support draft 2020-12 --- nf_core/pipelines/schema.py | 55 ++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index abdad719f6..398783d5a8 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -116,8 +116,26 @@ def load_schema(self): self.schema = json.load(fh) self.schema_defaults = {} self.schema_params = {} + if "$schema" not in self.schema: + raise AssertionError("Schema missing top-level `$schema` attribute") + self.schema_draft = self.schema["$schema"] + self.defs_notation = self.get_defs_notation() log.debug(f"JSON file loaded: {self.schema_filename}") + def get_defs_notation(self, schema=None): + if schema is None: + schema = self.schema + + if "$defs" in schema: + defs_notation = "$defs" + elif "definitions" in schema: + defs_notation = "definitions" + elif "defs" in schema: + # nf-schema v2.0.0 only supported defs. this has been changed to $defs in nf-schema v2.1.0 + # this line prevents the breakage of schemas created for v2.0.0 + defs_notation = "defs" + return defs_notation + def sanitise_param_default(self, param): """ Given a param, ensure that the default value is the correct variable type @@ -168,10 +186,11 @@ def get_schema_defaults(self) -> None: if param["default"] is not None: self.schema_defaults[p_key] = param["default"] + # TODO add support for nested parameters # Grouped schema properties in subschema definitions - for defn_name, definition in self.schema.get("definitions", {}).items(): + for defn_name, definition in self.schema.get(self.defs_notation, {}).items(): for p_key, param in definition.get("properties", {}).items(): - self.schema_params[p_key] = ("definitions", defn_name, "properties", p_key) + self.schema_params[p_key] = (self.defs_notation, defn_name, "properties", p_key) if "default" in param: param = self.sanitise_param_default(param) if param["default"] is not None: @@ -248,13 +267,14 @@ def validate_default_params(self): if self.schema is None: log.error("[red][✗] Pipeline schema not found") try: + # TODO add support for nested parameters # Make copy of schema and remove required flags schema_no_required = copy.deepcopy(self.schema) if "required" in schema_no_required: schema_no_required.pop("required") - for group_key, group in schema_no_required.get("definitions", {}).items(): + for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): if "required" in group: - schema_no_required["definitions"][group_key].pop("required") + schema_no_required[self.defs_notation][group_key].pop("required") jsonschema.validate(self.schema_defaults, schema_no_required) except jsonschema.exceptions.ValidationError as e: raise AssertionError(f"Default parameters are invalid: {e.message}") @@ -360,6 +380,8 @@ def validate_schema(self, schema=None): if schema is None: schema = self.schema + if "$schema" not in schema: + raise AssertionError("Schema missing top-level `$schema` attribute") schema_draft = schema["$schema"] if schema_draft == "https://json-schema.org/draft-07/schema": try: @@ -374,25 +396,14 @@ def validate_schema(self, schema=None): except jsonschema.exceptions.SchemaError as e: raise AssertionError(f"Schema does not validate as Draft 2020-12 JSON Schema:\n {e}") else: - raise AssertionError(f"Unsupported JSON schema draft detected: ${schema_draft}. Use draft 7 or 2020-12 instead.") + raise AssertionError(f"Schema `$schema` should be `https://json-schema.org/draft/2020-12/schema` or `https://json-schema.org/draft-07/schema` \n Found `{schema_draft}`") param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) schema_defs = dict() - defs_notation = "" - if "$defs" in schema: - schema_defs = schema.get("$defs", {}).items() - defs_notation = "$defs" - elif "definitions" in schema: - schema_defs = schema.get("definitions", {}).items() - defs_notation = "definitions" - elif "defs" in schema: - # nf-schema v2.0.0 only supported defs. this has been changed to $defs in nf-schema v2.1.0 - # this line prevents the breakage of schemas created for v2.0.0 - schema_defs = schema.get("defs", {}).items() - defs_notation = "defs" + defs_notation = self.get_defs_notation(schema) - for d_key, d_schema in schema_defs: + for d_key, d_schema in schema.get(defs_notation, {}).items(): # Check that this definition is mentioned in allOf if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") @@ -428,7 +439,7 @@ def validate_schema(self, schema=None): def validate_schema_title_description(self, schema=None): """ Extra validation command for linting. - Checks that the schema "$id", "title" and "description" attributes match the piipeline config. + Checks that the schema "$id", "title" and "description" attributes match the pipeline config. """ if schema is None: schema = self.schema @@ -436,12 +447,6 @@ def validate_schema_title_description(self, schema=None): log.debug("Pipeline schema not set - skipping validation of top-level attributes") return None - if "$schema" not in self.schema: - raise AssertionError("Schema missing top-level `$schema` attribute") - schema_attr = "http://json-schema.org/draft-07/schema" - if self.schema["$schema"] != schema_attr: - raise AssertionError(f"Schema `$schema` should be `{schema_attr}`\n Found `{self.schema['$schema']}`") - if self.pipeline_manifest == {}: self.get_wf_params() From 6ecde202d05baa2944f582f0b0835ea73765006b Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:11:47 +0200 Subject: [PATCH 007/151] fix draft 7 schema using http --- nf_core/pipelines/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 398783d5a8..e1931b9007 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -383,7 +383,7 @@ def validate_schema(self, schema=None): if "$schema" not in schema: raise AssertionError("Schema missing top-level `$schema` attribute") schema_draft = schema["$schema"] - if schema_draft == "https://json-schema.org/draft-07/schema": + if schema_draft == "https://json-schema.org/draft-07/schema" or schema_draft == "http://json-schema.org/draft-07/schema": try: jsonschema.Draft7Validator.check_schema(schema) log.debug("JSON Schema Draft7 validated") From 36fad24c7eda672fce4a885fe2e7d5610b022a9c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:18:08 +0200 Subject: [PATCH 008/151] ruff --- nf_core/pipelines/schema.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index e1931b9007..188861fff1 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -400,7 +400,6 @@ def validate_schema(self, schema=None): param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) - schema_defs = dict() defs_notation = self.get_defs_notation(schema) for d_key, d_schema in schema.get(defs_notation, {}).items(): @@ -424,11 +423,11 @@ def validate_schema(self, schema=None): # Check that everything in allOf exists for allOf in schema.get("allOf", []): - _, allOf_defs_notation, def_key = allOf["$ref"].split("/") # "#//" - if allOf_defs_notation not in schema: - raise AssertionError(f"Schema has allOf, but no {allOf_defs_notation}") - if def_key not in schema.get(allOf_defs_notation, {}): - raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `{allOf_defs_notation}`") + _, allof_defs_notation, def_key = allOf["$ref"].split("/") # "#//" + if allof_defs_notation not in schema: + raise AssertionError(f"Schema has allOf, but no {allof_defs_notation}") + if def_key not in schema.get(allof_defs_notation, {}): + raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `{allof_defs_notation}`") # Check that the schema describes at least one parameter if num_params == 0: From 6777694b20726f333dd8eae416917c3324c3e158 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:33:00 +0200 Subject: [PATCH 009/151] fix tests + add draft 2020-12 as default draft --- nf_core/pipelines/schema.py | 2 ++ tests/pipelines/test_schema.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 188861fff1..1555a4198d 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -46,6 +46,8 @@ def __init__(self): self.web_schema_build_url = "https://nf-co.re/pipeline_schema_builder" self.web_schema_build_web_url = None self.web_schema_build_api_url = None + self.schema_draft = "https://json-schema.org/draft/2020-12/schema" + self.defs_notation = "$defs" def get_schema_path( self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None diff --git a/tests/pipelines/test_schema.py b/tests/pipelines/test_schema.py index 633de3db69..777d959e66 100644 --- a/tests/pipelines/test_schema.py +++ b/tests/pipelines/test_schema.py @@ -175,6 +175,7 @@ def test_validate_schema_fail_duplicate_ids(self): Check that the schema validation fails when we have duplicate IDs in definition subschema """ self.schema_obj.schema = { + "$schema": "http://json-schema.org/draft-07/schema", "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"foo": {}}}}, "allOf": [{"$ref": "#/definitions/groupOne"}, {"$ref": "#/definitions/groupTwo"}], } @@ -187,18 +188,20 @@ def test_validate_schema_fail_missing_def(self): Check that the schema validation fails when we a definition in allOf is not in definitions """ self.schema_obj.schema = { + "$schema": "http://json-schema.org/draft-07/schema", "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, "allOf": [{"$ref": "#/definitions/groupOne"}], } with pytest.raises(AssertionError) as exc_info: self.schema_obj.validate_schema(self.schema_obj.schema) - assert exc_info.value.args[0] == "Definition subschema `groupTwo` not included in schema `allOf`" + assert exc_info.value.args[0] == "Definition subschema `#/definitions/groupTwo` not included in schema `allOf`" def test_validate_schema_fail_unexpected_allof(self): """ Check that the schema validation fails when we an unrecognised definition is in allOf """ self.schema_obj.schema = { + "$schema": "http://json-schema.org/draft-07/schema", "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, "allOf": [ {"$ref": "#/definitions/groupOne"}, From 0dfd15ded935054fc8d185b5fd36223505441ff7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:34:01 +0200 Subject: [PATCH 010/151] fix typo --- nf_core/pipelines/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 1555a4198d..ba5af51f96 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -413,7 +413,7 @@ def validate_schema(self, schema=None): if allOf["$ref"] == f"#/{defs_notation}/{d_key}": in_allOf = True if not in_allOf: - raise AssertionError(f"Definition subschema `#{defs_notation}/{d_key}` not included in schema `allOf`") + raise AssertionError(f"Definition subschema `#/{defs_notation}/{d_key}` not included in schema `allOf`") # TODO add support for nested parameters for d_param_id in d_schema.get("properties", {}): From 7f29f373a82ed1ada16b6b493f10d724a4b879cc Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:37:06 +0200 Subject: [PATCH 011/151] ruff --- nf_core/pipelines/schema.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index ba5af51f96..43058d4e6f 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -119,7 +119,7 @@ def load_schema(self): self.schema_defaults = {} self.schema_params = {} if "$schema" not in self.schema: - raise AssertionError("Schema missing top-level `$schema` attribute") + raise AssertionError("Schema missing top-level `$schema` attribute") self.schema_draft = self.schema["$schema"] self.defs_notation = self.get_defs_notation() log.debug(f"JSON file loaded: {self.schema_filename}") @@ -385,7 +385,10 @@ def validate_schema(self, schema=None): if "$schema" not in schema: raise AssertionError("Schema missing top-level `$schema` attribute") schema_draft = schema["$schema"] - if schema_draft == "https://json-schema.org/draft-07/schema" or schema_draft == "http://json-schema.org/draft-07/schema": + if ( + schema_draft == "https://json-schema.org/draft-07/schema" + or schema_draft == "http://json-schema.org/draft-07/schema" + ): try: jsonschema.Draft7Validator.check_schema(schema) log.debug("JSON Schema Draft7 validated") @@ -398,7 +401,9 @@ def validate_schema(self, schema=None): except jsonschema.exceptions.SchemaError as e: raise AssertionError(f"Schema does not validate as Draft 2020-12 JSON Schema:\n {e}") else: - raise AssertionError(f"Schema `$schema` should be `https://json-schema.org/draft/2020-12/schema` or `https://json-schema.org/draft-07/schema` \n Found `{schema_draft}`") + raise AssertionError( + f"Schema `$schema` should be `https://json-schema.org/draft/2020-12/schema` or `https://json-schema.org/draft-07/schema` \n Found `{schema_draft}`" + ) param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) @@ -425,7 +430,7 @@ def validate_schema(self, schema=None): # Check that everything in allOf exists for allOf in schema.get("allOf", []): - _, allof_defs_notation, def_key = allOf["$ref"].split("/") # "#//" + _, allof_defs_notation, def_key = allOf["$ref"].split("/") # "#//" if allof_defs_notation not in schema: raise AssertionError(f"Schema has allOf, but no {allof_defs_notation}") if def_key not in schema.get(allof_defs_notation, {}): From 730c6779779d4e14512df25c043eb02efe6d1659 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 17:34:53 +0200 Subject: [PATCH 012/151] add checks for nf-validation and nf-schema plugin config --- nf_core/pipelines/lint/nextflow_config.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 96323af94d..8fe933d102 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -336,6 +336,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) # Check for the availability of the "test" configuration profile by parsing nextflow.config + # Also check for the presence of nf-validation/nf-schema and check if they have pinned versions with open(Path(self.wf_path, "nextflow.config")) as f: content = f.read() @@ -363,6 +364,38 @@ def nextflow_config(self) -> Dict[str, List[str]]: else: failed.append("nextflow.config does not contain configuration profile `test`") + match_plugins = re.search(r"\bplugins\s*\{([^}]+)}", cleaned_content, re.MULTILINE) + if not match_plugins: + failed.append( + "nextflow.config does not contain `plugins` scope, but `nf-validation` or `nf-schema` plugins are required" + ) + else: + found_plugins = {} + for line in match_plugins.group(1).split("\n"): + cleaned_line = line.split("//")[0].strip().replace("\"", "'") + if "id" not in line: continue + match_line = re.search(r"\bid\s'([^']+)'", cleaned_line) + if not match_line: + failed.append(f"nextflow.config contains an invalid plugins identifier: {cleaned_line}") + continue + plugin = match_line.group(1) + name = plugin.split("@")[0] + version = "" + if "@" in plugin: + version = plugin.split("@")[1] + found_plugins[name] = version + + if len(found_plugins) == 0: + failed.append("nextflow.config contains an empty plugins scope") + elif "nf-validation" in found_plugins and "nf-schema" in found_plugins: + failed.append("nextflow.config contains both nf-validation and nf-schema") + elif "nf-validation" in found_plugins and found_plugins["nf-validation"] == "": + failed.append("nextflow.config contains an unpinned version of nf-validation") + elif "nf-schema" in found_plugins and found_plugins["nf-schema"] == "": + failed.append("nextflow.config contains an unpinned version of nf-schema") + elif "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: + failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json ignore_defaults = [] for item in ignore_configs: From 7fbbff7a412c64ef67417dd60a722d67f6d37647 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 17:39:10 +0200 Subject: [PATCH 013/151] linting fully works now! --- nf_core/pipelines/lint/schema_description.py | 5 +++-- nf_core/pipelines/schema.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/pipelines/lint/schema_description.py b/nf_core/pipelines/lint/schema_description.py index d617e40949..b586cc5242 100644 --- a/nf_core/pipelines/lint/schema_description.py +++ b/nf_core/pipelines/lint/schema_description.py @@ -36,8 +36,9 @@ def schema_description(self): warned.append(f"Ungrouped param in schema: `{up}`") # Iterate over groups and add warning for parameters without a description - for group_key in self.schema_obj.schema["definitions"].keys(): - group = self.schema_obj.schema["definitions"][group_key] + defs_notation = self.schema_obj.defs_notation + for group_key in self.schema_obj.schema[defs_notation].keys(): + group = self.schema_obj.schema[defs_notation][group_key] for param_key, param in group["properties"].items(): if param_key in ignore_params: ignored.append(f"Ignoring description check for param in schema: `{param_key}`") diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 43058d4e6f..c70332beb2 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -203,7 +203,7 @@ def get_schema_types(self) -> None: for name, param in self.schema.get("properties", {}).items(): if "type" in param: self.schema_types[name] = param["type"] - for _, definition in self.schema.get("definitions", {}).items(): + for _, definition in self.schema.get(self.defs_notation, {}).items(): for name, param in definition.get("properties", {}).items(): if "type" in param: self.schema_types[name] = param["type"] From 1c69f9d84e5c342a54ed733c4497de53ee8c1cea Mon Sep 17 00:00:00 2001 From: laurencekuhl Date: Thu, 15 Aug 2024 15:37:21 +0200 Subject: [PATCH 014/151] Add a verbose click option --- nf_core/__main__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 153b8b7e75..7da60180cd 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1136,6 +1136,13 @@ def command_modules_create( @modules.command("test") @click.pass_context @click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.option( + "-v", + "--verbose", + is_flag=True, + default=False, + help="Print verbose output to the console.", +) @click.option( "-d", "--dir", @@ -1171,11 +1178,11 @@ def command_modules_create( default=False, help="Migrate a module with pytest tests to nf-test", ) -def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest): +def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest, verbose): """ Run nf-test for a module. """ - modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest) + modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest, verbose) # nf-core modules lint From 9f6c1f4710ee10d094b0b9b1e011c891d62e5b8c Mon Sep 17 00:00:00 2001 From: laurencekuhl Date: Thu, 15 Aug 2024 16:44:24 +0200 Subject: [PATCH 015/151] Overwrite the verbose in ctx in case the user uses it --- nf_core/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 7da60180cd..64956e5f3f 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1182,7 +1182,8 @@ def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile """ Run nf-test for a module. """ - modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest, verbose) + ctx.obj['verbose'] = verbose + modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest) # nf-core modules lint From 7d873b5a456e6a13f77c4a8a6ff8f947b5eafc04 Mon Sep 17 00:00:00 2001 From: laurencekuhl Date: Thu, 15 Aug 2024 16:44:46 +0200 Subject: [PATCH 016/151] Overwrite the verbose in ctx in case the user uses it --- nf_core/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 64956e5f3f..3b56bda66c 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1182,7 +1182,8 @@ def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile """ Run nf-test for a module. """ - ctx.obj['verbose'] = verbose + if verbose: + ctx.obj['verbose'] = verbose modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest) From 22a18a4273bd4db1c2343a90742c1b5b9a3b62eb Mon Sep 17 00:00:00 2001 From: laurencekuhl Date: Thu, 15 Aug 2024 16:48:05 +0200 Subject: [PATCH 017/151] add matthias' suggestion woops --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3b56bda66c..23930ccd3c 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1141,7 +1141,7 @@ def command_modules_create( "--verbose", is_flag=True, default=False, - help="Print verbose output to the console.", + help="Print verbose output to the console. Sets `--debug` inside the nf-test command.", ) @click.option( "-d", From 7aead16a057bc0e30344aec26d4e12d9512bfa11 Mon Sep 17 00:00:00 2001 From: laurencekuhl Date: Thu, 15 Aug 2024 17:26:25 +0200 Subject: [PATCH 018/151] Ran pre commit --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 23930ccd3c..0efea13ec9 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1183,7 +1183,7 @@ def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile Run nf-test for a module. """ if verbose: - ctx.obj['verbose'] = verbose + ctx.obj["verbose"] = verbose modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest) From e5714d3cccdfce236c6c3ee3878282761d598be2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 04:56:51 +0000 Subject: [PATCH 019/151] Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 815e980270..4f08d8419d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.0 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 5da0e06d4f7b55e1ec8851c85380b36dbad7919e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 16 Aug 2024 04:57:40 +0000 Subject: [PATCH 020/151] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc78a1628a..ddc346b9dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ - Pipelines: allow numbers in custom pipeline name ([#3094](https://github.com/nf-core/tools/pull/3094)) - Add bot action to update textual snapshots and write bot documentation ([#3102](https://github.com/nf-core/tools/pull/3102)) - Components: allow spaces at the betinning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 ([#3122](https://github.com/nf-core/tools/pull/3122)) ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09] From 85461c5e47c7527e4879513e90c7150b19bb9013 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:34:04 +0000 Subject: [PATCH 021/151] Update python:3.12-slim Docker digest to 59c7332 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c88abcb1ce..fb1a867937 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:740d94a19218c8dd584b92f804b1158f85b0d241e5215ea26ed2dcade2b9d138 +FROM python:3.12-slim@sha256:59c7332a4a24373861c4a5f0eec2c92b87e3efeb8ddef011744ef9a751b1d11c LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From 129858342d0529ef59f1f04f7da0332fec038a1a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 17 Aug 2024 16:36:57 +0000 Subject: [PATCH 022/151] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddc346b9dc..0a450a6728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ - Add bot action to update textual snapshots and write bot documentation ([#3102](https://github.com/nf-core/tools/pull/3102)) - Components: allow spaces at the betinning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 ([#3122](https://github.com/nf-core/tools/pull/3122)) +- Update python:3.12-slim Docker digest to 59c7332 ([#3124](https://github.com/nf-core/tools/pull/3124)) ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09] From 0838d6bb040b8218493fad13c5fc33fa355b84af Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 14:58:27 +0200 Subject: [PATCH 023/151] warning for nf-validation --- nf_core/pipelines/lint/nextflow_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 8fe933d102..84bb492cb0 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -396,6 +396,9 @@ def nextflow_config(self) -> Dict[str, List[str]]: elif "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + if "nf-validation" in found_plugins: + warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") + # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json ignore_defaults = [] for item in ignore_configs: From 8559c2d6a18624c5096957e6a941011c8725e580 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 16:54:38 +0200 Subject: [PATCH 024/151] fetch plugin on schema filename setting --- nf_core/pipelines/lint/nextflow_config.py | 1 + nf_core/pipelines/schema.py | 81 +++++++++++++++-------- 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 84bb492cb0..a5b7cf325f 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -364,6 +364,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: else: failed.append("nextflow.config does not contain configuration profile `test`") + # Lint for nf-validation and nf-schema match_plugins = re.search(r"\bplugins\s*\{([^}]+)}", cleaned_content, re.MULTILINE) if not match_plugins: failed.append( diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index c70332beb2..77f421724e 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -5,6 +5,7 @@ import logging import tempfile import webbrowser +import re from pathlib import Path from typing import Union @@ -32,7 +33,7 @@ def __init__(self): self.schema = {} self.pipeline_dir = "" - self.schema_filename = "" + self._schema_filename = "" self.schema_defaults = {} self.schema_types = {} self.schema_params = {} @@ -46,8 +47,46 @@ def __init__(self): self.web_schema_build_url = "https://nf-co.re/pipeline_schema_builder" self.web_schema_build_web_url = None self.web_schema_build_api_url = None - self.schema_draft = "https://json-schema.org/draft/2020-12/schema" - self.defs_notation = "$defs" + self.validation_plugin = None + self.schema_draft = None + self.defs_notation = None + + # Update the validation plugin code everytime the schema gets changed + def set_schema_filename(self, schema: str) -> None: + self._schema_filename = schema + basepath = "/".join(str(schema).split("/")[:-1]) + config = f"{basepath}/nextflow.config" if basepath != "" else "nextflow.config" + self._update_validation_plugin_from_config(config) + + def get_schema_filename(self) -> str: + return self._schema_filename + + def del_schema_filename(self) -> None: + del self._schema_filename + + schema_filename = property(get_schema_filename, set_schema_filename, del_schema_filename) + + def _update_validation_plugin_from_config(self, config: str) -> None: + plugin = "nf-schema" + with open(Path(config)) as conf: + nf_schema_pattern = re.compile("id\s*[\"']nf-schema", re.MULTILINE) + nf_validation_pattern = re.compile("id\s*[\"']nf-validation", re.MULTILINE) + config_content = conf.read() + if re.search(nf_validation_pattern, config_content): + plugin = "nf-validation" + elif re.search(nf_schema_pattern, config_content): + plugin = "nf-schema" + else: + log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") + + self.validation_plugin = plugin + # Previous versions of nf-schema used "defs", but it's advised to use "$defs" + if plugin == "nf-schema": + self.defs_notation = "$defs" + self.schema_draft = "https://json-schema.org/draft/2020-12/schema" + else: + self.defs_notation = "definitions" + self.schema_draft = "https://json-schema.org/draft-07/schema" def get_schema_path( self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None @@ -120,24 +159,8 @@ def load_schema(self): self.schema_params = {} if "$schema" not in self.schema: raise AssertionError("Schema missing top-level `$schema` attribute") - self.schema_draft = self.schema["$schema"] - self.defs_notation = self.get_defs_notation() log.debug(f"JSON file loaded: {self.schema_filename}") - def get_defs_notation(self, schema=None): - if schema is None: - schema = self.schema - - if "$defs" in schema: - defs_notation = "$defs" - elif "definitions" in schema: - defs_notation = "definitions" - elif "defs" in schema: - # nf-schema v2.0.0 only supported defs. this has been changed to $defs in nf-schema v2.1.0 - # this line prevents the breakage of schemas created for v2.0.0 - defs_notation = "defs" - return defs_notation - def sanitise_param_default(self, param): """ Given a param, ensure that the default value is the correct variable type @@ -385,16 +408,15 @@ def validate_schema(self, schema=None): if "$schema" not in schema: raise AssertionError("Schema missing top-level `$schema` attribute") schema_draft = schema["$schema"] - if ( - schema_draft == "https://json-schema.org/draft-07/schema" - or schema_draft == "http://json-schema.org/draft-07/schema" - ): + if self.schema_draft != schema_draft: + raise AssertionError(f"Schema is using the wrong draft: {schema_draft}, should be {self.schema_draft}") + if self.schema_draft == "https://json-schema.org/draft-07/schema": try: jsonschema.Draft7Validator.check_schema(schema) log.debug("JSON Schema Draft7 validated") except jsonschema.exceptions.SchemaError as e: raise AssertionError(f"Schema does not validate as Draft 7 JSON Schema:\n {e}") - elif schema_draft == "https://json-schema.org/draft/2020-12/schema": + elif self.schema_draft == "https://json-schema.org/draft/2020-12/schema": try: jsonschema.Draft202012Validator.check_schema(schema) log.debug("JSON Schema Draft2020-12 validated") @@ -407,18 +429,21 @@ def validate_schema(self, schema=None): param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) - defs_notation = self.get_defs_notation(schema) - for d_key, d_schema in schema.get(defs_notation, {}).items(): + print(self.defs_notation) + + for d_key, d_schema in schema.get(self.defs_notation, {}).items(): + print(d_key) + print(d_schema) # Check that this definition is mentioned in allOf if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") in_allOf = False for allOf in schema.get("allOf", []): - if allOf["$ref"] == f"#/{defs_notation}/{d_key}": + if allOf["$ref"] == f"#/{self.defs_notation}/{d_key}": in_allOf = True if not in_allOf: - raise AssertionError(f"Definition subschema `#/{defs_notation}/{d_key}` not included in schema `allOf`") + raise AssertionError(f"Definition subschema `#/{self.defs_notation}/{d_key}` not included in schema `allOf`") # TODO add support for nested parameters for d_param_id in d_schema.get("properties", {}): From 0296f7d8fa6c5cc7a8dac62b450a8babcd77bab7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 17:00:33 +0200 Subject: [PATCH 025/151] add a check for defs usage instead of $defs --- nf_core/pipelines/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 77f421724e..9c65f34fde 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -430,11 +430,11 @@ def validate_schema(self, schema=None): param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) - print(self.defs_notation) + # Add a small check for older nf-schema JSON schemas + if "defs" in schema: + raise AssertionError(f'Using "defs" for schema definitions is not supported. Please use {self.defs_notation} instead') for d_key, d_schema in schema.get(self.defs_notation, {}).items(): - print(d_key) - print(d_schema) # Check that this definition is mentioned in allOf if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") From 0b8d04f89eb7f348bce87cc93f04d983e48e218f Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 17:16:31 +0200 Subject: [PATCH 026/151] change all definitions to self.defs_notation --- nf_core/pipelines/schema.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 9c65f34fde..9db9457516 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -235,7 +235,7 @@ def save_schema(self, suppress_logging=False): """Save a pipeline schema to a file""" # Write results to a JSON file num_params = len(self.schema.get("properties", {})) - num_params += sum(len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()) + num_params += sum(len(d.get("properties", {})) for d in self.schema.get(self.defs_notation, {}).values()) if not suppress_logging: log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") dump_json_with_prettier(self.schema_filename, self.schema) @@ -321,7 +321,7 @@ def validate_default_params(self): params_ignore = [] # Go over group keys - for group_key, group in schema_no_required.get("definitions", {}).items(): + for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): group_properties = group.get("properties") for param in group_properties: if param in params_ignore: @@ -527,9 +527,9 @@ def check_for_input_mimetype(self): if "input" not in self.schema_params: raise LookupError("Parameter `input` not found in schema") # Check that the input parameter is defined in the right place - if "input" not in self.schema.get("definitions", {}).get("input_output_options", {}).get("properties", {}): + if "input" not in self.schema.get(self.defs_notation, {}).get("input_output_options", {}).get("properties", {}): raise LookupError("Parameter `input` is not defined in the correct subschema (input_output_options)") - input_entry = self.schema["definitions"]["input_output_options"]["properties"]["input"] + input_entry = self.schema[self.defs_notation]["input_output_options"]["properties"]["input"] if "mimetype" not in input_entry: return None mimetype = input_entry["mimetype"] @@ -581,7 +581,7 @@ def schema_to_markdown(self, columns): out = f"# {self.schema['title']}\n\n" out += f"{self.schema['description']}\n" # Grouped parameters - for definition in self.schema.get("definitions", {}).values(): + for definition in self.schema.get(self.defs_notation, {}).values(): out += f"\n## {definition.get('title', {})}\n\n" out += f"{definition.get('description', '')}\n\n" required = definition.get("required", []) @@ -763,15 +763,15 @@ def remove_schema_empty_definitions(self): """ # Identify and remove empty definitions from the schema empty_definitions = [] - for d_key, d_schema in list(self.schema.get("definitions", {}).items()): + for d_key, d_schema in list(self.schema.get(self.defs_notation, {}).items()): if not d_schema.get("properties"): - del self.schema["definitions"][d_key] + del self.schema[self.defs_notation][d_key] empty_definitions.append(d_key) log.warning(f"Removing empty group: '{d_key}'") # Remove "allOf" group with empty definitions from the schema for d_key in empty_definitions: - allOf = {"$ref": f"#/definitions/{d_key}"} + allOf = {"$ref": f"#/{self.defs_notation}/{d_key}"} if allOf in self.schema.get("allOf", []): self.schema["allOf"].remove(allOf) @@ -780,8 +780,8 @@ def remove_schema_empty_definitions(self): del self.schema["allOf"] # If we don't have anything left in "definitions", remove it - if self.schema.get("definitions") == {}: - del self.schema["definitions"] + if self.schema.get(self.defs_notation) == {}: + del self.schema[self.defs_notation] def remove_schema_notfound_configs(self): """ @@ -791,9 +791,9 @@ def remove_schema_notfound_configs(self): # Top-level properties self.schema, params_removed = self.remove_schema_notfound_configs_single_schema(self.schema) # Sub-schemas in definitions - for d_key, definition in self.schema.get("definitions", {}).items(): + for d_key, definition in self.schema.get(self.defs_notation, {}).items(): cleaned_schema, p_removed = self.remove_schema_notfound_configs_single_schema(definition) - self.schema["definitions"][d_key] = cleaned_schema + self.schema[self.defs_notation][d_key] = cleaned_schema params_removed.extend(p_removed) return params_removed From 544381310c6e5ac57664524f63105441246c34fd Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 18:33:52 +0200 Subject: [PATCH 027/151] migrate template part 1 --- nf_core/pipeline-template/conf/colors.config | 61 ++++++++ nf_core/pipeline-template/nextflow.config | 44 ++++-- .../pipeline-template/nextflow_schema.json | 55 +++---- .../utils_nfcore_pipeline_pipeline/main.nf | 17 +- .../nf-core/utils_nfcore_pipeline/main.nf | 46 ------ .../nf-core/utils_nfvalidation_plugin/main.nf | 25 +-- .../tests/main.nf.test | 145 +----------------- .../tests/nextflow.config | 1 + .../pipeline-template/workflows/pipeline.nf | 2 +- 9 files changed, 130 insertions(+), 266 deletions(-) create mode 100644 nf_core/pipeline-template/conf/colors.config create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config diff --git a/nf_core/pipeline-template/conf/colors.config b/nf_core/pipeline-template/conf/colors.config new file mode 100644 index 0000000000..00f2963267 --- /dev/null +++ b/nf_core/pipeline-template/conf/colors.config @@ -0,0 +1,61 @@ +// For now, there is no easy way to set monochromeLogs +colors { + // Reset / Meta + reset = "\033[0m" + bold = "\033[1m" + dim = "\033[2m" + underlined = "\033[4m" + blink = "\033[5m" + reverse = "\033[7m" + hidden = "\033[8m" + + // Regular Colors + black = "\033[0;30m" + red = "\033[0;31m" + green = "\033[0;32m" + yellow = "\033[0;33m" + blue = "\033[0;34m" + purple = "\033[0;35m" + cyan = "\033[0;36m" + white = "\033[0;37m" + + // Bold + bblack = "\033[1;30m" + bred = "\033[1;31m" + bgreen = "\033[1;32m" + byellow = "\033[1;33m" + bblue = "\033[1;34m" + bpurple = "\033[1;35m" + bcyan = "\033[1;36m" + bwhite = "\033[1;37m" + + // Underline + ublack = "\033[4;30m" + ured = "\033[4;31m" + ugreen = "\033[4;32m" + uyellow = "\033[4;33m" + ublue = "\033[4;34m" + upurple = "\033[4;35m" + ucyan = "\033[4;36m" + uwhite = "\033[4;37m" + + // High Intensity + iblack = "\033[0;90m" + ired = "\033[0;91m" + igreen = "\033[0;92m" + iyellow = "\033[0;93m" + iblue = "\033[0;94m" + ipurple = "\033[0;95m" + icyan = "\033[0;96m" + iwhite = "\033[0;97m" + + // Bold High Intensity + biblack = "\033[1;90m" + bired = "\033[1;91m" + bigreen = "\033[1;92m" + biyellow = "\033[1;93m" + biblue = "\033[1;94m" + bipurple = "\033[1;95m" + bicyan = "\033[1;96m" + biwhite = "\033[1;97m" +} \ No newline at end of file diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 0469d6cfec..3d3841ba49 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -38,6 +38,8 @@ params { monochrome_logs = false hook_url = null help = false + helpFull = false + showHidden = false version = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' @@ -59,10 +61,6 @@ params { max_time = '240.h' // Schema validation default options - validationFailUnrecognisedParams = false - validationLenientMode = false - validationSchemaIgnoreParams = 'genomes,igenomes_base' - validationShowHiddenParams = false validate_params = true } @@ -200,11 +198,6 @@ podman.registry = 'quay.io' singularity.registry = 'quay.io' charliecloud.registry = 'quay.io' -// Nextflow plugins -plugins { - id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet -} - {% if igenomes -%} // Load igenomes.config if required if (!params.igenomes_ignore) { @@ -267,6 +260,39 @@ manifest { doi = '' } +// Nextflow plugins +plugins { + id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} + +includeConfig "conf/colors.config" + +validation { + parametersSchema = "${projectDir}/nextflow_schema.json" + help { + enabled = true + command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " + beforeText = """ +-${colors.dim}----------------------------------------------------${colors.reset}- + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} +${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} +${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} +${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} +${colors.purple} ${manifest.name} ${manifest.version}${colors.reset} +-${colors.dim}----------------------------------------------------${colors.reset}- +""" + afterText = """${manifest.doi ? "* The pipeline\n" : ""}${manifest.doi.tokenize(",").collect { " https://doi.org/${it.trim().replace('https://doi.org/','')}"}.join("\n")}${manifest.doi ? "\n" : ""} +* The nf-core framework + https://doi.org/10.1038/s41587-020-0439-x + +* Software dependencies + https://github.com/${manifest.name}/blob/master/CITATIONS.md +""" + } +} + + // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 4a1a22c3ee..14120bbabc 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/{{ name }}/master/nextflow_schema.json", "title": "{{ name }} pipeline parameters", "description": "{{ description }}", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -73,6 +73,20 @@ "fa_icon": "fas fa-ban", "hidden": true, "help_text": "Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`." + }, + "igenomes_base": { + "type": "string", + "format": "directory-path", + "description": "The base path to the igenomes reference files", + "fa_icon": "fas fa-ban", + "hidden": true, + "default": "s3://ngi-igenomes/igenomes/" + }, + "genomes": { + "type": "object", + "description": "An object containing all reference data availabe in igenomes", + "fa_icon": "fas fa-ban", + "hidden": true } } }, @@ -169,12 +183,6 @@ "description": "Less common options for the pipeline, typically set in a config file.", "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", "properties": { - "help": { - "type": "boolean", - "description": "Display help text.", - "fa_icon": "fas fa-question-circle", - "hidden": true - }, "version": { "type": "boolean", "description": "Display version and exit.", @@ -253,27 +261,6 @@ "fa_icon": "fas fa-check-square", "hidden": true }, - "validationShowHiddenParams": { - "type": "boolean", - "fa_icon": "far fa-eye-slash", - "description": "Show all params when using `--help`", - "hidden": true, - "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "validationFailUnrecognisedParams": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters fails when an unrecognised parameter is found.", - "hidden": true, - "help_text": "By default, when an unrecognised parameter is found, it returns a warinig." - }, - "validationLenientMode": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters in lenient more.", - "hidden": true, - "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." - }, "pipelines_testdata_base_path": { "type": "string", "fa_icon": "far fa-check-circle", @@ -286,19 +273,19 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, {% if igenomes %}{ - "$ref": "#/definitions/reference_genome_options" + "$ref": "#/$defs/reference_genome_options" },{% endif %} {% if nf_core_configs %}{ - "$ref": "#/definitions/institutional_config_options" + "$ref": "#/$defs/institutional_config_options" },{% endif %} { - "$ref": "#/definitions/max_job_request_options" + "$ref": "#/$defs/max_job_request_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 72c9be85eb..c83d2ae0cb 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -9,8 +9,8 @@ */ include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' -include { paramsSummaryMap } from 'plugin/nf-validation' -include { fromSamplesheet } from 'plugin/nf-validation' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' @@ -30,7 +30,6 @@ workflow PIPELINE_INITIALISATION { take: version // boolean: Display version and exit - help // boolean: Display help text validate_params // boolean: Boolean whether to validate parameters against the schema at runtime monochrome_logs // boolean: Do not use coloured log outputs nextflow_cli_args // array: List of positional nextflow CLI args @@ -54,16 +53,8 @@ workflow PIPELINE_INITIALISATION { // // Validate parameters and generate parameter summary to stdout // - pre_help_text = nfCoreLogo(monochrome_logs) - post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) - def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " UTILS_NFVALIDATION_PLUGIN ( - help, - workflow_command, - pre_help_text, - post_help_text, - validate_params, - "nextflow_schema.json" + validate_params ) // @@ -84,7 +75,7 @@ workflow PIPELINE_INITIALISATION { // Create channel from input file provided through params.input // Channel - .fromSamplesheet("input") + .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")) .map { meta, fastq_1, fastq_2 -> if (!fastq_2) { diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 14558c3927..a5dcd5f83d 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -61,25 +61,6 @@ def checkProfileProvided(nextflow_cli_args) { } } -// -// Citation string for pipeline -// -def workflowCitation() { - def temp_doi_ref = "" - String[] manifest_doi = workflow.manifest.doi.tokenize(",") - // Using a loop to handle multiple DOIs - // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers - // Removing ` ` since the manifest.doi is a string and not a proper list - for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" - return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + - "* The pipeline\n" + - temp_doi_ref + "\n" + - "* The nf-core framework\n" + - " https://doi.org/10.1038/s41587-020-0439-x\n\n" + - "* Software dependencies\n" + - " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" -} - // // Generate workflow version string // @@ -157,33 +138,6 @@ def paramsSummaryMultiqc(summary_params) { return yaml_file_text } -// -// nf-core logo -// -def nfCoreLogo(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) - String.format( - """\n - ${dashedLine(monochrome_logs)} - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} - ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} - ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} - ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} - ${dashedLine(monochrome_logs)} - """.stripIndent() - ) -} - -// -// Return dashed line -// -def dashedLine(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) - return "-${colors.dim}----------------------------------------------------${colors.reset}-" -} - // // ANSII colours used for terminal logging // diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf index 2585b65d1b..453894a64a 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf @@ -8,9 +8,8 @@ ======================================================================================== */ -include { paramsHelp } from 'plugin/nf-validation' -include { paramsSummaryLog } from 'plugin/nf-validation' -include { validateParameters } from 'plugin/nf-validation' +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' /* ======================================================================================== @@ -21,12 +20,7 @@ include { validateParameters } from 'plugin/nf-validation' workflow UTILS_NFVALIDATION_PLUGIN { take: - print_help // boolean: print help - workflow_command // string: default commmand used to run pipeline - pre_help_text // string: string to be printed before help text and summary log - post_help_text // string: string to be printed after help text and summary log validate_params // boolean: validate parameters - schema_filename // path: JSON schema file, null to use default value main: @@ -37,24 +31,11 @@ workflow UTILS_NFVALIDATION_PLUGIN { post_help_text = post_help_text ?: '' workflow_command = workflow_command ?: '' - // - // Print help message if needed - // - if (print_help) { - log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text - System.exit(0) - } - - // - // Print parameter summary to stdout - // - log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text - // // Validate parameters relative to the parameter JSON schema // if (validate_params){ - validateParameters(parameters_schema: schema_filename) + validateParameters() } emit: diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test index 5784a33f2f..354b817545 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test @@ -10,30 +10,21 @@ nextflow_workflow { tag "utils_nfvalidation_plugin" tag "subworkflows/utils_nfvalidation_plugin" + config "./nextflow.config" + test("Should run nothing") { when { params { - monochrome_logs = true test_data = '' } workflow { """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename + input[0] = validate_params """ } } @@ -45,147 +36,19 @@ nextflow_workflow { } } - test("Should run help") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with command") { - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with extra text") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = "pre-help-text" - post_help_text = "post-help-text" - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('pre-help-text') } }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } }, - { assert workflow.stdout.any { it.contains('post-help-text') } } - ) - } - } - test("Should validate params") { when { params { - monochrome_logs = true test_data = '' outdir = 1 } workflow { """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null validate_params = true - schema_filename = "$moduleTestDir/nextflow_schema.json" - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename + input[0] = validate_params """ } } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config new file mode 100644 index 0000000000..8d047ed59e --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config @@ -0,0 +1 @@ +validation.monochromeLogs = true \ No newline at end of file diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index d98c392f0c..4bb0913211 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -6,7 +6,7 @@ include { FASTQC } from '../modules/nf-core/fastqc/main' {% if multiqc %}include { MULTIQC } from '../modules/nf-core/multiqc/main'{% endif %} -include { paramsSummaryMap } from 'plugin/nf-validation' +include { paramsSummaryMap } from 'plugin/nf-schema' {% if multiqc %}include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline'{% endif %} include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' {% if citations or multiqc %}include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'{% endif %} From 7931c7f658712e15d522531616c3e001849e338b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 20 Aug 2024 12:43:06 +0200 Subject: [PATCH 028/151] add option to exclude license from pipeline template --- nf_core/pipelines/create/templatefeatures.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index b97bf347ab..dc3f072710 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -242,3 +242,18 @@ changelog: - "CHANGELOG.md" nfcore_pipelines: False custom_pipelines: True +license: + skippable_paths: + - "LICENSE" + short_description: "Add a license" + description: "Add the MIT license file." + help_text: | + To protect the copyright of the pipeline, you can add a LICENSE file. + This option ads the MIT License. You can read the conditions here: https://opensource.org/license/MIT + linting: + files_exist: + - "LICENSE" + files_unchanged: + - "LICENSE" + nfcore_pipelines: False + custom_pipelines: True From 6ff0929c0956b148839c231133b440f5f0cd876a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 20 Aug 2024 10:46:47 +0000 Subject: [PATCH 029/151] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a450a6728..b530f597bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - add option to exclude changelog from custom pipeline template ([#3104](https://github.com/nf-core/tools/pull/3104)) - handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108)) - add templatefeatures.yml to python package ([#3112](https://github.com/nf-core/tools/pull/3112)) +- add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125)) ### Linting From 78d469fd893a6e1c45aa59330378f213a994fdbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 20 Aug 2024 12:56:04 +0200 Subject: [PATCH 030/151] Update nf_core/pipelines/create/templatefeatures.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipelines/create/templatefeatures.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index dc3f072710..a8a6ce5656 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -245,7 +245,7 @@ changelog: license: skippable_paths: - "LICENSE" - short_description: "Add a license" + short_description: "Add a license File" description: "Add the MIT license file." help_text: | To protect the copyright of the pipeline, you can add a LICENSE file. From 34565c1e12ce20cda823e4e92eed6c084501821f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 20 Aug 2024 13:00:04 +0200 Subject: [PATCH 031/151] update textual snapshots --- .../__snapshots__/test_create_app.ambr | 512 +++++++++--------- 1 file changed, 256 insertions(+), 256 deletions(-) diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index f4eb255087..66c08faba0 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-3220763577-matrix { + .terminal-2278814444-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3220763577-title { + .terminal-2278814444-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3220763577-r1 { fill: #c5c8c6 } - .terminal-3220763577-r2 { fill: #e3e3e3 } - .terminal-3220763577-r3 { fill: #989898 } - .terminal-3220763577-r4 { fill: #e1e1e1 } - .terminal-3220763577-r5 { fill: #4ebf71;font-weight: bold } - .terminal-3220763577-r6 { fill: #1e1e1e } - .terminal-3220763577-r7 { fill: #507bb3 } - .terminal-3220763577-r8 { fill: #e2e2e2 } - .terminal-3220763577-r9 { fill: #808080 } - .terminal-3220763577-r10 { fill: #dde6ed;font-weight: bold } - .terminal-3220763577-r11 { fill: #001541 } - .terminal-3220763577-r12 { fill: #0178d4 } - .terminal-3220763577-r13 { fill: #454a50 } - .terminal-3220763577-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-3220763577-r15 { fill: #000000 } - .terminal-3220763577-r16 { fill: #e4e4e4 } - .terminal-3220763577-r17 { fill: #14191f } - .terminal-3220763577-r18 { fill: #7ae998 } - .terminal-3220763577-r19 { fill: #0a180e;font-weight: bold } - .terminal-3220763577-r20 { fill: #008139 } - .terminal-3220763577-r21 { fill: #fea62b;font-weight: bold } - .terminal-3220763577-r22 { fill: #a7a9ab } - .terminal-3220763577-r23 { fill: #e2e3e3 } + .terminal-2278814444-r1 { fill: #c5c8c6 } + .terminal-2278814444-r2 { fill: #e3e3e3 } + .terminal-2278814444-r3 { fill: #989898 } + .terminal-2278814444-r4 { fill: #e1e1e1 } + .terminal-2278814444-r5 { fill: #4ebf71;font-weight: bold } + .terminal-2278814444-r6 { fill: #1e1e1e } + .terminal-2278814444-r7 { fill: #507bb3 } + .terminal-2278814444-r8 { fill: #e2e2e2 } + .terminal-2278814444-r9 { fill: #808080 } + .terminal-2278814444-r10 { fill: #dde6ed;font-weight: bold } + .terminal-2278814444-r11 { fill: #001541 } + .terminal-2278814444-r12 { fill: #0178d4 } + .terminal-2278814444-r13 { fill: #454a50 } + .terminal-2278814444-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-2278814444-r15 { fill: #000000 } + .terminal-2278814444-r16 { fill: #e4e4e4 } + .terminal-2278814444-r17 { fill: #14191f } + .terminal-2278814444-r18 { fill: #7ae998 } + .terminal-2278814444-r19 { fill: #0a180e;font-weight: bold } + .terminal-2278814444-r20 { fill: #008139 } + .terminal-2278814444-r21 { fill: #fea62b;font-weight: bold } + .terminal-2278814444-r22 { fill: #a7a9ab } + .terminal-2278814444-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Hide help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - - Nf-core pipelines are configured to use a copy of the most common reference  - genome files.▂▂ - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Hide help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + + Nf-core pipelines are configured to use a copy of the most common reference  + genome files. + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + ▅▅ + For more information about reference genomes in nf-core pipelines, see the  + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-537214554-matrix { + .terminal-2282176583-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-537214554-title { + .terminal-2282176583-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-537214554-r1 { fill: #c5c8c6 } - .terminal-537214554-r2 { fill: #e3e3e3 } - .terminal-537214554-r3 { fill: #989898 } - .terminal-537214554-r4 { fill: #e1e1e1 } - .terminal-537214554-r5 { fill: #4ebf71;font-weight: bold } - .terminal-537214554-r6 { fill: #1e1e1e } - .terminal-537214554-r7 { fill: #507bb3 } - .terminal-537214554-r8 { fill: #e2e2e2 } - .terminal-537214554-r9 { fill: #808080 } - .terminal-537214554-r10 { fill: #dde6ed;font-weight: bold } - .terminal-537214554-r11 { fill: #001541 } - .terminal-537214554-r12 { fill: #14191f } - .terminal-537214554-r13 { fill: #454a50 } - .terminal-537214554-r14 { fill: #7ae998 } - .terminal-537214554-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-537214554-r16 { fill: #0a180e;font-weight: bold } - .terminal-537214554-r17 { fill: #000000 } - .terminal-537214554-r18 { fill: #008139 } - .terminal-537214554-r19 { fill: #fea62b;font-weight: bold } - .terminal-537214554-r20 { fill: #a7a9ab } - .terminal-537214554-r21 { fill: #e2e3e3 } + .terminal-2282176583-r1 { fill: #c5c8c6 } + .terminal-2282176583-r2 { fill: #e3e3e3 } + .terminal-2282176583-r3 { fill: #989898 } + .terminal-2282176583-r4 { fill: #e1e1e1 } + .terminal-2282176583-r5 { fill: #4ebf71;font-weight: bold } + .terminal-2282176583-r6 { fill: #1e1e1e } + .terminal-2282176583-r7 { fill: #507bb3 } + .terminal-2282176583-r8 { fill: #e2e2e2 } + .terminal-2282176583-r9 { fill: #808080 } + .terminal-2282176583-r10 { fill: #dde6ed;font-weight: bold } + .terminal-2282176583-r11 { fill: #001541 } + .terminal-2282176583-r12 { fill: #14191f } + .terminal-2282176583-r13 { fill: #454a50 } + .terminal-2282176583-r14 { fill: #7ae998 } + .terminal-2282176583-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-2282176583-r16 { fill: #0a180e;font-weight: bold } + .terminal-2282176583-r17 { fill: #000000 } + .terminal-2282176583-r18 { fill: #008139 } + .terminal-2282176583-r19 { fill: #fea62b;font-weight: bold } + .terminal-2282176583-r20 { fill: #a7a9ab } + .terminal-2282176583-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help ▇▇ - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use code lintersThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - prettier. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Include citationsInclude pipeline tools Show help  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Show help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▇▇ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use code lintersThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + editor-config and  + prettier. + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Include citationsInclude pipeline tools Show help  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  From ceb78a84c90d7955e903f3ffbfa26000d62557fb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 20 Aug 2024 15:11:26 +0200 Subject: [PATCH 032/151] add option to exclude email from pipeline template --- nf_core/pipeline-template/.editorconfig | 2 + nf_core/pipeline-template/.prettierignore | 2 + nf_core/pipeline-template/docs/output.md | 2 +- nf_core/pipeline-template/main.nf | 4 +- nf_core/pipeline-template/nextflow.config | 2 + .../pipeline-template/nextflow_schema.json | 8 +- .../utils_nfcore_pipeline_pipeline/main.nf | 6 + nf_core/pipelines/create/templatefeatures.yml | 18 + .../__snapshots__/test_create_app.ambr | 510 +++++++++--------- 9 files changed, 293 insertions(+), 261 deletions(-) diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index 72dda289a0..5efa586109 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -25,8 +25,10 @@ insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset +{%- if email %} [/assets/email*] indent_size = unset +{%- endif %} # ignore python and markdown [*.{py,md}] diff --git a/nf_core/pipeline-template/.prettierignore b/nf_core/pipeline-template/.prettierignore index 437d763d0c..3b3d5ff08e 100644 --- a/nf_core/pipeline-template/.prettierignore +++ b/nf_core/pipeline-template/.prettierignore @@ -1,4 +1,6 @@ +{%- if email %} email_template.html +{%- endif %} adaptivecard.json slackreport.json .nextflow* diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index edd48b83f4..76195a6825 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -55,7 +55,7 @@ Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQ - `pipeline_info/` - Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. - - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. + {% if email %}- Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. {% endif %} - Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. - Parameters used by the pipeline run: `params.json`. diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index fddfc5489e..0cf688a7e9 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -20,7 +20,7 @@ include { {{ short_name|upper }} } from './workflows/{{ short_name }}' include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' -{% if igenomes %} +{%- if igenomes %} include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' /* @@ -95,9 +95,11 @@ workflow { // SUBWORKFLOW: Run completion tasks // PIPELINE_COMPLETION ( + {%- if email %} params.email, params.email_on_fail, params.plaintext_email, + {%- endif %} params.outdir, params.monochrome_logs, params.hook_url, diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 0469d6cfec..daa1c5e369 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -32,9 +32,11 @@ params { // Boilerplate options outdir = null publish_dir_mode = 'copy' + {%- if email %} email = null email_on_fail = null plaintext_email = false + {%- endif %} monochrome_logs = false hook_url = null help = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 4a1a22c3ee..4a376330bd 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -28,14 +28,14 @@ "format": "directory-path", "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", "fa_icon": "fas fa-folder-open" - }, + }{% if email %}, "email": { "type": "string", "description": "Email address for completion summary.", "fa_icon": "fas fa-envelope", "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.", "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$" - }{% if multiqc %}, + }{% endif %}{% if multiqc %}, "multiqc_title": { "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", @@ -189,7 +189,7 @@ "fa_icon": "fas fa-copy", "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], "hidden": true - }, + },{% if email %} "email_on_fail": { "type": "string", "description": "Email address for completion summary, only when pipeline fails.", @@ -203,7 +203,7 @@ "description": "Send plain-text email instead of HTML.", "fa_icon": "fas fa-remove-format", "hidden": true - }, + },{% endif %} {%- if multiqc %} "max_multiqc_email_size": { "type": "string", diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 72c9be85eb..62ef73d65d 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -12,7 +12,9 @@ include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plu include { paramsSummaryMap } from 'plugin/nf-validation' include { fromSamplesheet } from 'plugin/nf-validation' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +{%- if email %} include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' +{%- endif %} include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' @@ -117,9 +119,11 @@ workflow PIPELINE_INITIALISATION { workflow PIPELINE_COMPLETION { take: + {%- if email %} email // string: email address email_on_fail // string: email address sent on pipeline failure plaintext_email // boolean: Send plain-text email instead of HTML + {% endif %} outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output hook_url // string: hook URL for notifications @@ -133,6 +137,7 @@ workflow PIPELINE_COMPLETION { // Completion email and summary // workflow.onComplete { + {%- if email %} if (email || email_on_fail) { {%- if multiqc %} completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, multiqc_report.toList()) @@ -140,6 +145,7 @@ workflow PIPELINE_COMPLETION { completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, []) {%- endif %} } + {%- endif %} completionSummary(monochrome_logs) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index a8a6ce5656..5dafd7e312 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -257,3 +257,21 @@ license: - "LICENSE" nfcore_pipelines: False custom_pipelines: True +email: + skippable_paths: + - "assets/email_template.html" + - "assets/sendmail_template.txt" + - "assets/email_template.txt" + short_description: "Enable email updates" + description: "Enable sending emails on pipeline completion." + help_text: | + Enable the option of sending an email which will include pipeline execution reports on pipeline completion. + linting: + files_exist: + - "assets/email_template.html" + - "assets/sendmail_template.txt" + - "assets/email_template.txt" + files_unchanged: + - ".prettierignore" + nfcore_pipelines: False + custom_pipelines: True diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index 66c08faba0..f2a0578474 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-2278814444-matrix { + .terminal-2220422576-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2278814444-title { + .terminal-2220422576-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2278814444-r1 { fill: #c5c8c6 } - .terminal-2278814444-r2 { fill: #e3e3e3 } - .terminal-2278814444-r3 { fill: #989898 } - .terminal-2278814444-r4 { fill: #e1e1e1 } - .terminal-2278814444-r5 { fill: #4ebf71;font-weight: bold } - .terminal-2278814444-r6 { fill: #1e1e1e } - .terminal-2278814444-r7 { fill: #507bb3 } - .terminal-2278814444-r8 { fill: #e2e2e2 } - .terminal-2278814444-r9 { fill: #808080 } - .terminal-2278814444-r10 { fill: #dde6ed;font-weight: bold } - .terminal-2278814444-r11 { fill: #001541 } - .terminal-2278814444-r12 { fill: #0178d4 } - .terminal-2278814444-r13 { fill: #454a50 } - .terminal-2278814444-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-2278814444-r15 { fill: #000000 } - .terminal-2278814444-r16 { fill: #e4e4e4 } - .terminal-2278814444-r17 { fill: #14191f } - .terminal-2278814444-r18 { fill: #7ae998 } - .terminal-2278814444-r19 { fill: #0a180e;font-weight: bold } - .terminal-2278814444-r20 { fill: #008139 } - .terminal-2278814444-r21 { fill: #fea62b;font-weight: bold } - .terminal-2278814444-r22 { fill: #a7a9ab } - .terminal-2278814444-r23 { fill: #e2e3e3 } + .terminal-2220422576-r1 { fill: #c5c8c6 } + .terminal-2220422576-r2 { fill: #e3e3e3 } + .terminal-2220422576-r3 { fill: #989898 } + .terminal-2220422576-r4 { fill: #e1e1e1 } + .terminal-2220422576-r5 { fill: #4ebf71;font-weight: bold } + .terminal-2220422576-r6 { fill: #1e1e1e } + .terminal-2220422576-r7 { fill: #507bb3 } + .terminal-2220422576-r8 { fill: #e2e2e2 } + .terminal-2220422576-r9 { fill: #808080 } + .terminal-2220422576-r10 { fill: #dde6ed;font-weight: bold } + .terminal-2220422576-r11 { fill: #001541 } + .terminal-2220422576-r12 { fill: #0178d4 } + .terminal-2220422576-r13 { fill: #454a50 } + .terminal-2220422576-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-2220422576-r15 { fill: #000000 } + .terminal-2220422576-r16 { fill: #e4e4e4 } + .terminal-2220422576-r17 { fill: #14191f } + .terminal-2220422576-r18 { fill: #7ae998 } + .terminal-2220422576-r19 { fill: #0a180e;font-weight: bold } + .terminal-2220422576-r20 { fill: #008139 } + .terminal-2220422576-r21 { fill: #fea62b;font-weight: bold } + .terminal-2220422576-r22 { fill: #a7a9ab } + .terminal-2220422576-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - + - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Hide help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - - Nf-core pipelines are configured to use a copy of the most common reference  - genome files. - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Hide help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + + Nf-core pipelines are configured to use a copy of the most common reference ▇▇ + genome files. + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + ▅▅ + For more information about reference genomes in nf-core pipelines, see the  + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-2282176583-matrix { + .terminal-1370375189-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2282176583-title { + .terminal-1370375189-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2282176583-r1 { fill: #c5c8c6 } - .terminal-2282176583-r2 { fill: #e3e3e3 } - .terminal-2282176583-r3 { fill: #989898 } - .terminal-2282176583-r4 { fill: #e1e1e1 } - .terminal-2282176583-r5 { fill: #4ebf71;font-weight: bold } - .terminal-2282176583-r6 { fill: #1e1e1e } - .terminal-2282176583-r7 { fill: #507bb3 } - .terminal-2282176583-r8 { fill: #e2e2e2 } - .terminal-2282176583-r9 { fill: #808080 } - .terminal-2282176583-r10 { fill: #dde6ed;font-weight: bold } - .terminal-2282176583-r11 { fill: #001541 } - .terminal-2282176583-r12 { fill: #14191f } - .terminal-2282176583-r13 { fill: #454a50 } - .terminal-2282176583-r14 { fill: #7ae998 } - .terminal-2282176583-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-2282176583-r16 { fill: #0a180e;font-weight: bold } - .terminal-2282176583-r17 { fill: #000000 } - .terminal-2282176583-r18 { fill: #008139 } - .terminal-2282176583-r19 { fill: #fea62b;font-weight: bold } - .terminal-2282176583-r20 { fill: #a7a9ab } - .terminal-2282176583-r21 { fill: #e2e3e3 } + .terminal-1370375189-r1 { fill: #c5c8c6 } + .terminal-1370375189-r2 { fill: #e3e3e3 } + .terminal-1370375189-r3 { fill: #989898 } + .terminal-1370375189-r4 { fill: #e1e1e1 } + .terminal-1370375189-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1370375189-r6 { fill: #1e1e1e } + .terminal-1370375189-r7 { fill: #507bb3 } + .terminal-1370375189-r8 { fill: #e2e2e2 } + .terminal-1370375189-r9 { fill: #808080 } + .terminal-1370375189-r10 { fill: #dde6ed;font-weight: bold } + .terminal-1370375189-r11 { fill: #001541 } + .terminal-1370375189-r12 { fill: #14191f } + .terminal-1370375189-r13 { fill: #454a50 } + .terminal-1370375189-r14 { fill: #7ae998 } + .terminal-1370375189-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-1370375189-r16 { fill: #0a180e;font-weight: bold } + .terminal-1370375189-r17 { fill: #000000 } + .terminal-1370375189-r18 { fill: #008139 } + .terminal-1370375189-r19 { fill: #fea62b;font-weight: bold } + .terminal-1370375189-r20 { fill: #a7a9ab } + .terminal-1370375189-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▇▇ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use code lintersThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - prettier. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Include citationsInclude pipeline tools Show help  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Show help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges▁▁ + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use code lintersThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + editor-config and  + prettier. + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Include citationsInclude pipeline tools Show help  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  From 1390a6d5e2bb259c6004a845e212abe2b80ff167 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 20 Aug 2024 13:19:14 +0000 Subject: [PATCH 033/151] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b530f597bc..98fb925338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108)) - add templatefeatures.yml to python package ([#3112](https://github.com/nf-core/tools/pull/3112)) - add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125)) +- add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126)) ### Linting From 5519c83d5b582af797ff4ae242fa98113bbe847a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 15:24:51 +0200 Subject: [PATCH 034/151] some small updates for the new release --- nf_core/pipeline-template/conf/colors.config | 61 ------------------- nf_core/pipeline-template/modules.json | 4 +- nf_core/pipeline-template/nextflow.config | 22 ++++--- .../nf-core/utils_nfschema_plugin/main.nf | 32 ++++++++++ .../nf-core/utils_nfschema_plugin/meta.yml | 22 +++++++ .../tests/main.nf.test | 27 ++++---- .../tests/nextflow.config | 8 +++ .../tests/nextflow_schema.json | 8 +-- .../nf-core/utils_nfvalidation_plugin/main.nf | 43 ------------- .../utils_nfvalidation_plugin/meta.yml | 44 ------------- .../tests/nextflow.config | 1 - .../utils_nfvalidation_plugin/tests/tags.yml | 2 - 12 files changed, 93 insertions(+), 181 deletions(-) delete mode 100644 nf_core/pipeline-template/conf/colors.config create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml rename nf_core/pipeline-template/subworkflows/nf-core/{utils_nfvalidation_plugin => utils_nfschema_plugin}/tests/main.nf.test (62%) create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config rename nf_core/pipeline-template/subworkflows/nf-core/{utils_nfvalidation_plugin => utils_nfschema_plugin}/tests/nextflow_schema.json (95%) delete mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf delete mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml delete mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config delete mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml diff --git a/nf_core/pipeline-template/conf/colors.config b/nf_core/pipeline-template/conf/colors.config deleted file mode 100644 index 00f2963267..0000000000 --- a/nf_core/pipeline-template/conf/colors.config +++ /dev/null @@ -1,61 +0,0 @@ -// For now, there is no easy way to set monochromeLogs -colors { - // Reset / Meta - reset = "\033[0m" - bold = "\033[1m" - dim = "\033[2m" - underlined = "\033[4m" - blink = "\033[5m" - reverse = "\033[7m" - hidden = "\033[8m" - - // Regular Colors - black = "\033[0;30m" - red = "\033[0;31m" - green = "\033[0;32m" - yellow = "\033[0;33m" - blue = "\033[0;34m" - purple = "\033[0;35m" - cyan = "\033[0;36m" - white = "\033[0;37m" - - // Bold - bblack = "\033[1;30m" - bred = "\033[1;31m" - bgreen = "\033[1;32m" - byellow = "\033[1;33m" - bblue = "\033[1;34m" - bpurple = "\033[1;35m" - bcyan = "\033[1;36m" - bwhite = "\033[1;37m" - - // Underline - ublack = "\033[4;30m" - ured = "\033[4;31m" - ugreen = "\033[4;32m" - uyellow = "\033[4;33m" - ublue = "\033[4;34m" - upurple = "\033[4;35m" - ucyan = "\033[4;36m" - uwhite = "\033[4;37m" - - // High Intensity - iblack = "\033[0;90m" - ired = "\033[0;91m" - igreen = "\033[0;92m" - iyellow = "\033[0;93m" - iblue = "\033[0;94m" - ipurple = "\033[0;95m" - icyan = "\033[0;96m" - iwhite = "\033[0;97m" - - // Bold High Intensity - biblack = "\033[1;90m" - bired = "\033[1;91m" - bigreen = "\033[1;92m" - biyellow = "\033[1;93m" - biblue = "\033[1;94m" - bipurple = "\033[1;95m" - bicyan = "\033[1;96m" - biwhite = "\033[1;97m" -} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index eb9391b29a..1a3b96490d 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -30,9 +30,9 @@ "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", "installed_by": ["subworkflows"] }, - "utils_nfvalidation_plugin": { + "utils_nfschema_plugin": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "a3e87febb28bd0461c22a917c5b2c1492053ef85", "installed_by": ["subworkflows"] } } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3d3841ba49..346292c980 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -265,22 +265,20 @@ plugins { id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } -includeConfig "conf/colors.config" - validation { parametersSchema = "${projectDir}/nextflow_schema.json" help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " beforeText = """ --${colors.dim}----------------------------------------------------${colors.reset}- - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} -${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} -${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} -${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} -${colors.purple} ${manifest.name} ${manifest.version}${colors.reset} --${colors.dim}----------------------------------------------------${colors.reset}- +-\033[2m----------------------------------------------------\033[0m- + \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m +\033[0;34m ___ __ __ __ ___ \033[0;32m/,-._.--~\'\033[0m +\033[0;34m |\\ | |__ __ / ` / \\ |__) |__ \033[0;33m} {\033[0m +\033[0;34m | \\| | \\__, \\__/ | \\ |___ \033[0;32m\\`-._,-`-,\033[0m + \033[0;32m`._,._,\'\033[0m +\033[0;35m ${manifest.name} ${manifest.version}\033[0m +-\033[2m----------------------------------------------------\033[0m- """ afterText = """${manifest.doi ? "* The pipeline\n" : ""}${manifest.doi.tokenize(",").collect { " https://doi.org/${it.trim().replace('https://doi.org/','')}"}.join("\n")}${manifest.doi ? "\n" : ""} * The nf-core framework @@ -290,6 +288,10 @@ ${colors.purple} ${manifest.name} ${manifest.version}${colors.reset} https://github.com/${manifest.name}/blob/master/CITATIONS.md """ } + summary { + beforeText = validation.help.beforeText + afterText = validation.help.afterText + } } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf new file mode 100644 index 0000000000..5d1dc7c928 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -0,0 +1,32 @@ +// +// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary +// + +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' + +workflow UTILS_NFSCHEMA_PLUGIN { + + take: + validate_params // boolean: validate the parameters + + main: + + // + // Print parameter summary to stdout. This will display the parameters + // that differ from the default given in the JSON schema + // + log.info paramsSummaryLog(workflow) + + // + // Validate the parameters using nextflow_schema.json or the schema + // given via the validation.parametersSchema configuration option + // + if(validate_params) { + validateParameters() + } + + emit: + dummy_emit = true +} + diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml new file mode 100644 index 0000000000..90b8cb1898 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "utils_nfschema_plugin" +description: Run nf-schema to validate parameters and create a summary of changed parameters +keywords: + - validation + - JSON schema + - plugin + - parameters + - summary +components: [] +input: + - validate_params: + type: boolean + description: Validate the parameters and error if invalid. +output: + - dummy_emit: + type: boolean + description: Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nvnieuwk" +maintainers: + - "@nvnieuwk" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test similarity index 62% rename from nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test rename to nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test index 354b817545..703d3a9b86 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -1,14 +1,14 @@ +// TODO nf-core: Once you have added the required tests, please run the following command to build this file: +// nf-core subworkflows test utils_nfschema_plugin nextflow_workflow { - name "Test Workflow UTILS_NFVALIDATION_PLUGIN" + name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" script "../main.nf" - workflow "UTILS_NFVALIDATION_PLUGIN" + workflow "UTILS_NFSCHEMA_PLUGIN" + tag "subworkflows" tag "subworkflows_nfcore" - tag "plugin/nf-validation" - tag "'plugin/nf-validation'" - tag "utils_nfvalidation_plugin" - tag "subworkflows/utils_nfvalidation_plugin" + tag "subworkflows/utils_nfschema_plugin" config "./nextflow.config" @@ -17,13 +17,12 @@ nextflow_workflow { when { params { - test_data = '' + test_data = '' } workflow { """ - validate_params = false - + validate_params = false input[0] = validate_params """ } @@ -41,13 +40,13 @@ nextflow_workflow { when { params { - test_data = '' - outdir = 1 + test_data = '' + outdir = 1 } + workflow { """ - validate_params = true - + validate_params = true input[0] = validate_params """ } @@ -56,7 +55,7 @@ nextflow_workflow { then { assertAll( { assert workflow.failed }, - { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } ) } } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config new file mode 100644 index 0000000000..0907ac58f0 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -0,0 +1,8 @@ +plugins { + id "nf-schema@2.1.0" +} + +validation { + parametersSchema = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + monochromeLogs = true +} \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json similarity index 95% rename from nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json rename to nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json index 7626c1c93e..331e0d2f44 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", "title": ". pipeline parameters", "description": "", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -87,10 +87,10 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf deleted file mode 100644 index 453894a64a..0000000000 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf +++ /dev/null @@ -1,43 +0,0 @@ -// -// Subworkflow that uses the nf-validation plugin to render help text and parameter summary -// - -/* -======================================================================================== - IMPORT NF-VALIDATION PLUGIN -======================================================================================== -*/ - -include { paramsSummaryLog } from 'plugin/nf-schema' -include { validateParameters } from 'plugin/nf-schema' - -/* -======================================================================================== - SUBWORKFLOW DEFINITION -======================================================================================== -*/ - -workflow UTILS_NFVALIDATION_PLUGIN { - - take: - validate_params // boolean: validate parameters - - main: - - log.debug "Using schema file: ${schema_filename}" - - // Default values for strings - pre_help_text = pre_help_text ?: '' - post_help_text = post_help_text ?: '' - workflow_command = workflow_command ?: '' - - // - // Validate parameters relative to the parameter JSON schema - // - if (validate_params){ - validateParameters() - } - - emit: - dummy_emit = true -} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml deleted file mode 100644 index 3d4a6b04f5..0000000000 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml +++ /dev/null @@ -1,44 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "UTILS_NFVALIDATION_PLUGIN" -description: Use nf-validation to initiate and validate a pipeline -keywords: - - utility - - pipeline - - initialise - - validation -components: [] -input: - - print_help: - type: boolean - description: | - Print help message and exit - - workflow_command: - type: string - description: | - The command to run the workflow e.g. "nextflow run main.nf" - - pre_help_text: - type: string - description: | - Text to print before the help message - - post_help_text: - type: string - description: | - Text to print after the help message - - validate_params: - type: boolean - description: | - Validate the parameters and error if invalid. - - schema_filename: - type: string - description: | - The filename of the schema to validate against. -output: - - dummy_emit: - type: boolean - description: | - Dummy emit to make nf-core subworkflows lint happy -authors: - - "@adamrtalbot" -maintainers: - - "@adamrtalbot" - - "@maxulysse" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config deleted file mode 100644 index 8d047ed59e..0000000000 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config +++ /dev/null @@ -1 +0,0 @@ -validation.monochromeLogs = true \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml deleted file mode 100644 index 60b1cfff49..0000000000 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -subworkflows/utils_nfvalidation_plugin: - - subworkflows/nf-core/utils_nfvalidation_plugin/** From 171640b62dddcb81feddced4f674df88c47654ab Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 16:13:59 +0200 Subject: [PATCH 035/151] some more small updates to the template --- nf_core/pipeline-template/nextflow.config | 2 +- .../subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 346292c980..ef067dcfab 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -255,7 +255,7 @@ manifest { homePage = 'https://github.com/{{ name }}' description = """{{ description }}""" mainScript = 'main.nf' - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.10.0' version = '{{ version }}' doi = '' } diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index c83d2ae0cb..a994cfd584 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -8,7 +8,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' +include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' include { paramsSummaryMap } from 'plugin/nf-schema' include { samplesheetToList } from 'plugin/nf-schema' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' @@ -53,7 +53,7 @@ workflow PIPELINE_INITIALISATION { // // Validate parameters and generate parameter summary to stdout // - UTILS_NFVALIDATION_PLUGIN ( + UTILS_NFSCHEMA_PLUGIN ( validate_params ) From f72a2c325324dfc21d494d4135ae03da416ee709 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 20 Aug 2024 16:19:19 +0200 Subject: [PATCH 036/151] update pytest test_make_pipeline_schema --- tests/pipelines/test_launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/test_launch.py b/tests/pipelines/test_launch.py index da7618d486..7c6e3d6196 100644 --- a/tests/pipelines/test_launch.py +++ b/tests/pipelines/test_launch.py @@ -60,7 +60,7 @@ def test_make_pipeline_schema(self, tmp_path): Path(test_pipeline_dir, "nextflow_schema.json").unlink() self.launcher = nf_core.pipelines.launch.Launch(test_pipeline_dir, params_out=self.nf_params_fn) self.launcher.get_pipeline_schema() - assert len(self.launcher.schema_obj.schema["definitions"]["input_output_options"]["properties"]) > 2 + assert len(self.launcher.schema_obj.schema["definitions"]["input_output_options"]["properties"]) >= 2 assert self.launcher.schema_obj.schema["definitions"]["input_output_options"]["properties"]["outdir"] == { "type": "string", "format": "directory-path", From 249b3da383f17ab0beecfeb9e8346619edaf6776 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 16:59:22 +0200 Subject: [PATCH 037/151] update handling of ignored parameters --- .../.github/workflows/ci.yml | 2 +- nf_core/pipeline-template/README.md | 2 +- nf_core/pipelines/lint/nextflow_config.py | 15 +++++-------- nf_core/pipelines/schema.py | 22 ++++++++++++++----- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 6b2547765d..63fa99cec8 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: Check out pipeline code diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 7718d2e5f8..d31dee93f4 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -16,7 +16,7 @@ [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index a5b7cf325f..150a89e102 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -65,14 +65,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: * Should always be set to default value: ``https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}`` - * ``params.validationShowHiddenParams`` - - * Determines whether boilerplate params are showed by schema. Set to ``false`` by default - - * ``params.validationSchemaIgnoreParams`` - - * A comma separated string of inputs the schema validation should ignore. - **The following variables throw warnings if missing:** * ``manifest.mainScript``: The filename of the main pipeline script (should be ``main.nf``) @@ -151,8 +143,11 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["process.time"], ["params.outdir"], ["params.input"], - ["params.validationShowHiddenParams"], - ["params.validationSchemaIgnoreParams"], + ["validation.help.enabled"], + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"] ] # Throw a warning if these are missing config_warn = [ diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 9db9457516..78f4317094 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -50,6 +50,7 @@ def __init__(self): self.validation_plugin = None self.schema_draft = None self.defs_notation = None + self.ignored_params = [] # Update the validation plugin code everytime the schema gets changed def set_schema_filename(self, schema: str) -> None: @@ -69,16 +70,26 @@ def del_schema_filename(self) -> None: def _update_validation_plugin_from_config(self, config: str) -> None: plugin = "nf-schema" with open(Path(config)) as conf: - nf_schema_pattern = re.compile("id\s*[\"']nf-schema", re.MULTILINE) - nf_validation_pattern = re.compile("id\s*[\"']nf-validation", re.MULTILINE) + nf_schema_pattern = re.compile(r"id\s*[\"']nf-schema", re.MULTILINE) + nf_validation_pattern = re.compile(r"id\s*[\"']nf-validation", re.MULTILINE) config_content = conf.read() if re.search(nf_validation_pattern, config_content): plugin = "nf-validation" + self.ignored_params = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") + self.ignored_params.append("validationSchemaIgnoreParams") elif re.search(nf_schema_pattern, config_content): plugin = "nf-schema" + ignored_params_pattern = re.compile(r"defaultIgnoreParams\s*=\s*\[([^\]]*)\]", re.MULTILINE) + ignored_params_match = re.search(ignored_params_pattern, config_content) + ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default + if ignored_params_match and len(ignored_params_match.groups()) == 1: + ignored_params.extend(ignored_params_match.group(1).replace("\"", "").replace("'", '').replace(" ", "").split(",")) + self.ignored_params = ignored_params else: log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") + + self.validation_plugin = plugin # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": @@ -845,13 +856,12 @@ def add_schema_found_configs(self): Update defaults if they have changed """ params_added = [] - params_ignore = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") - params_ignore.append("validationSchemaIgnoreParams") + for p_key, p_val in self.pipeline_params.items(): s_key = self.schema_params.get(p_key) # Check if key is in schema parameters # Key is in pipeline but not in schema or ignored from schema - if p_key not in self.schema_params and p_key not in params_ignore: + if p_key not in self.schema_params and p_key not in self.ignored_params: if ( self.no_prompts or self.schema_from_scratch @@ -884,7 +894,7 @@ def add_schema_found_configs(self): elif ( s_key and (p_key not in self.schema_defaults) - and (p_key not in params_ignore) + and (p_key not in self.ignored_params) and (p_def := self.build_schema_param(p_val).get("default")) ): if self.no_prompts or Confirm.ask( From af34d0718db98544a8da402952aea87ca778a60e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 17:50:32 +0200 Subject: [PATCH 038/151] update validation plugin fetching --- nf_core/pipeline-template/nextflow.config | 2 +- nf_core/pipelines/schema.py | 58 ++++++++++++----------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ef067dcfab..3a98c40f0a 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -61,7 +61,7 @@ params { max_time = '240.h' // Schema validation default options - validate_params = true + validate_params = true } diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 78f4317094..e9c5556dbb 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -69,35 +69,45 @@ def del_schema_filename(self) -> None: def _update_validation_plugin_from_config(self, config: str) -> None: plugin = "nf-schema" - with open(Path(config)) as conf: - nf_schema_pattern = re.compile(r"id\s*[\"']nf-schema", re.MULTILINE) - nf_validation_pattern = re.compile(r"id\s*[\"']nf-validation", re.MULTILINE) - config_content = conf.read() - if re.search(nf_validation_pattern, config_content): - plugin = "nf-validation" - self.ignored_params = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") - self.ignored_params.append("validationSchemaIgnoreParams") - elif re.search(nf_schema_pattern, config_content): + conf = nf_core.utils.fetch_wf_config(Path(self.schema_filename).parent) + + plugins = str(conf.get("plugins", "")).strip("\"").strip("'").strip(" ").split(",") + plugin_found = False + for plugin_instance in plugins: + if "nf-schema" in plugin_instance: plugin = "nf-schema" - ignored_params_pattern = re.compile(r"defaultIgnoreParams\s*=\s*\[([^\]]*)\]", re.MULTILINE) - ignored_params_match = re.search(ignored_params_pattern, config_content) - ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default - if ignored_params_match and len(ignored_params_match.groups()) == 1: - ignored_params.extend(ignored_params_match.group(1).replace("\"", "").replace("'", '').replace(" ", "").split(",")) - self.ignored_params = ignored_params - else: - log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") - + plugin_found = True + break + elif "nf-validation" in plugin_instance: + plugin = "nf-validation" + plugin_found = True + break + + if not plugin_found: + log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") + + if "nf-validation" in plugins: + plugin = "nf-validation" + elif "nf-schema" in plugins: + plugin = "nf-schema" - self.validation_plugin = plugin # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": self.defs_notation = "$defs" + ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default + ignored_params_config = conf.get("validation", {}).get("defaultIgnoreParams", []) + if len(ignored_params_config) > 0: + ignored_params.extend(ignored_params_config) + self.ignored_params = ignored_params self.schema_draft = "https://json-schema.org/draft/2020-12/schema" + else: self.defs_notation = "definitions" self.schema_draft = "https://json-schema.org/draft-07/schema" + self.get_wf_params() + self.ignored_params = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") + self.ignored_params.append("validationSchemaIgnoreParams") def get_schema_path( self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None @@ -325,17 +335,11 @@ def validate_default_params(self): if self.pipeline_params == {}: self.get_wf_params() - # Collect parameters to ignore - if "validationSchemaIgnoreParams" in self.pipeline_params: - params_ignore = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") - else: - params_ignore = [] - # Go over group keys for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): group_properties = group.get("properties") for param in group_properties: - if param in params_ignore: + if param in self.ignored_params: continue if param in self.pipeline_params: self.validate_config_default_parameter(param, group_properties[param], self.pipeline_params[param]) @@ -348,7 +352,7 @@ def validate_default_params(self): ungrouped_properties = self.schema.get("properties") if ungrouped_properties: for param in ungrouped_properties: - if param in params_ignore: + if param in self.ignored_params: continue if param in self.pipeline_params: self.validate_config_default_parameter( From e6381b71c67fcc1ecb18c8b3c6e796dd6df0dec7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 21 Aug 2024 10:59:31 +0200 Subject: [PATCH 039/151] update plugin linting --- nf_core/pipelines/lint/nextflow_config.py | 56 ++++++++--------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 150a89e102..bd2f4f768b 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -1,3 +1,4 @@ +import ast import logging import re from pathlib import Path @@ -330,6 +331,25 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) ) + # Lint for plugins + config_plugins = ast.literal_eval(self.nf_config.get("plugins", "").strip("\"")) + found_plugins = [] + if len(config_plugins) == 0: + failed.append("nextflow.config contains an empty plugins scope") + for plugin in config_plugins: + if "@" not in plugin: + failed.append(f"Plugin '{plugin}' does not have a pinned version") + found_plugins.append(plugin.split("@")[0]) + + if "nf-validation" in found_plugins and "nf-schema" in found_plugins: + failed.append("nextflow.config contains both nf-validation and nf-schema") + if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: + failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + + if "nf-validation" in found_plugins: + warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") + + # Check for the availability of the "test" configuration profile by parsing nextflow.config # Also check for the presence of nf-validation/nf-schema and check if they have pinned versions with open(Path(self.wf_path, "nextflow.config")) as f: @@ -359,42 +379,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: else: failed.append("nextflow.config does not contain configuration profile `test`") - # Lint for nf-validation and nf-schema - match_plugins = re.search(r"\bplugins\s*\{([^}]+)}", cleaned_content, re.MULTILINE) - if not match_plugins: - failed.append( - "nextflow.config does not contain `plugins` scope, but `nf-validation` or `nf-schema` plugins are required" - ) - else: - found_plugins = {} - for line in match_plugins.group(1).split("\n"): - cleaned_line = line.split("//")[0].strip().replace("\"", "'") - if "id" not in line: continue - match_line = re.search(r"\bid\s'([^']+)'", cleaned_line) - if not match_line: - failed.append(f"nextflow.config contains an invalid plugins identifier: {cleaned_line}") - continue - plugin = match_line.group(1) - name = plugin.split("@")[0] - version = "" - if "@" in plugin: - version = plugin.split("@")[1] - found_plugins[name] = version - - if len(found_plugins) == 0: - failed.append("nextflow.config contains an empty plugins scope") - elif "nf-validation" in found_plugins and "nf-schema" in found_plugins: - failed.append("nextflow.config contains both nf-validation and nf-schema") - elif "nf-validation" in found_plugins and found_plugins["nf-validation"] == "": - failed.append("nextflow.config contains an unpinned version of nf-validation") - elif "nf-schema" in found_plugins and found_plugins["nf-schema"] == "": - failed.append("nextflow.config contains an unpinned version of nf-schema") - elif "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: - failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") - - if "nf-validation" in found_plugins: - warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") - # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json ignore_defaults = [] for item in ignore_configs: From e66a732e7db6452fb12b0f311762b19bbe7f3418 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 21 Aug 2024 11:01:16 +0200 Subject: [PATCH 040/151] ignore genomes for now --- nf_core/pipeline-template/nextflow.config | 2 +- nf_core/pipeline-template/nextflow_schema.json | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3a98c40f0a..bb55678001 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -266,7 +266,7 @@ plugins { } validation { - parametersSchema = "${projectDir}/nextflow_schema.json" + defaultIgnoreParams = ["genomes"] help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 14120bbabc..908d310171 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -81,12 +81,6 @@ "fa_icon": "fas fa-ban", "hidden": true, "default": "s3://ngi-igenomes/igenomes/" - }, - "genomes": { - "type": "object", - "description": "An object containing all reference data availabe in igenomes", - "fa_icon": "fas fa-ban", - "hidden": true } } }, From b6ae8d01dd440558cb3a2af821a04ea8fc89591a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 21 Aug 2024 12:23:52 +0200 Subject: [PATCH 041/151] improve config linting for validation even more --- nf_core/pipelines/lint/nextflow_config.py | 59 ++++++++++++++--------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index bd2f4f768b..6e292deb09 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -144,11 +144,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["process.time"], ["params.outdir"], ["params.input"], - ["validation.help.enabled"], - ["validation.help.beforeText"], - ["validation.help.afterText"], - ["validation.summary.beforeText"], - ["validation.summary.afterText"] + ["validation.help.enabled"] ] # Throw a warning if these are missing config_warn = [ @@ -157,6 +153,11 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["trace.file"], ["report.file"], ["dag.file"], + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.help.command"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"] ] # Old depreciated vars - fail if present config_fail_ifdefined = [ @@ -168,6 +169,35 @@ def nextflow_config(self) -> Dict[str, List[str]]: "params.enable_conda", ] + # Lint for plugins + config_plugins = ast.literal_eval(self.nf_config.get("plugins", "").strip("\"")) + found_plugins = [] + if len(config_plugins) == 0: + failed.append("nextflow.config contains an empty plugins scope") + for plugin in config_plugins: + if "@" not in plugin: + failed.append(f"Plugin '{plugin}' does not have a pinned version") + found_plugins.append(plugin.split("@")[0]) + + if "nf-validation" in found_plugins and "nf-schema" in found_plugins: + failed.append("nextflow.config contains both nf-validation and nf-schema") + if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: + failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + + if "nf-schema" in found_plugins: + if self.nf_config.get("validation.help.enabled", "false") == "false": + failed.append("The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages") + config_fail_ifdefined.extend([ + "params.validationFailUnrecognisedParams", + "params.validationLenientMode", + "params.validationSchemaIgnoreParams", + "params.validationShowHiddenParams" + ]) + + if "nf-validation" in found_plugins: + warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") + + # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get("nextflow_config", []) if self.lint_config is not None else [] @@ -331,25 +361,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) ) - # Lint for plugins - config_plugins = ast.literal_eval(self.nf_config.get("plugins", "").strip("\"")) - found_plugins = [] - if len(config_plugins) == 0: - failed.append("nextflow.config contains an empty plugins scope") - for plugin in config_plugins: - if "@" not in plugin: - failed.append(f"Plugin '{plugin}' does not have a pinned version") - found_plugins.append(plugin.split("@")[0]) - - if "nf-validation" in found_plugins and "nf-schema" in found_plugins: - failed.append("nextflow.config contains both nf-validation and nf-schema") - if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: - failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") - - if "nf-validation" in found_plugins: - warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") - - # Check for the availability of the "test" configuration profile by parsing nextflow.config # Also check for the presence of nf-validation/nf-schema and check if they have pinned versions with open(Path(self.wf_path, "nextflow.config")) as f: From 1ede989be403e25f8b7485503d6a07bb6f20301d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 21 Aug 2024 11:27:40 +0200 Subject: [PATCH 042/151] add option to exclude adaptivecard and slackreport from pipeline template --- nf_core/pipeline-template/.prettierignore | 4 +++ nf_core/pipeline-template/main.nf | 4 +-- .../utils_nfcore_pipeline_pipeline/main.nf | 6 ++++- nf_core/pipelines/create/templatefeatures.yml | 25 +++++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.prettierignore b/nf_core/pipeline-template/.prettierignore index 3b3d5ff08e..c8e8ad9e11 100644 --- a/nf_core/pipeline-template/.prettierignore +++ b/nf_core/pipeline-template/.prettierignore @@ -1,8 +1,12 @@ {%- if email %} email_template.html {%- endif %} +{%- if adaptivecard %} adaptivecard.json +{%- endif %} +{%- if slackreport %} slackreport.json +{%- endif %} .nextflow* work/ data/ diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 0cf688a7e9..7002a9c2d5 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -102,8 +102,8 @@ workflow { {%- endif %} params.outdir, params.monochrome_logs, - params.hook_url, - {%- if multiqc %}{{ prefix_nodash|upper }}_{{ short_name|upper }}.out.multiqc_report{% endif %} + {% if adaptivecard or slackreport %}params.hook_url,{% endif %} + {% if multiqc %}{{ prefix_nodash|upper }}_{{ short_name|upper }}.out.multiqc_report{% endif %} ) } diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 62ef73d65d..3db8f66cb6 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -18,7 +18,9 @@ include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' +{%- if adaptivecard or slackreport %} include { imNotification } from '../../nf-core/utils_nfcore_pipeline' +{%- endif %} include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' @@ -126,7 +128,7 @@ workflow PIPELINE_COMPLETION { {% endif %} outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output - hook_url // string: hook URL for notifications + {% if adaptivecard or slackreport %}hook_url // string: hook URL for notifications{% endif %} {% if multiqc %}multiqc_report // string: Path to MultiQC report{% endif %} main: @@ -149,9 +151,11 @@ workflow PIPELINE_COMPLETION { completionSummary(monochrome_logs) + {%- if adaptivecard or slackreport %} if (hook_url) { imNotification(summary_params, hook_url) } + {%- endif %} } workflow.onError { diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 5dafd7e312..ff56ee190e 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -275,3 +275,28 @@ email: - ".prettierignore" nfcore_pipelines: False custom_pipelines: True +adaptivecard: + skippable_paths: + - "assets/adaptivecard.json" + short_description: "Add template for status messages" + description: "Enable pipeline status update messages through Microsoft Teams" + help_text: | + This adds an Adaptive Card. A snippets of user interface. + This Adaptive Card is used as a template for pipeline update messages and it is compatible with Microsoft Teams. + linting: + files_unchanged: + - ".prettierignore" + nfcore_pipelines: False + custom_pipelines: True +slackreport: + skippable_paths: + - "assets/slackreport.json" + short_description: "Add template for slack status messages" + description: "Enable pipeline status update messages through Slack" + help_text: | + This adds an JSON template used as a template for pipeline update messages in Slack. + linting: + files_unchanged: + - ".prettierignore" + nfcore_pipelines: False + custom_pipelines: True From c9adbd5bff432f4370a6901e00a576a687cdc416 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 21 Aug 2024 13:58:07 +0200 Subject: [PATCH 043/151] update textual snapshot --- .../__snapshots__/test_create_app.ambr | 512 +++++++++--------- 1 file changed, 256 insertions(+), 256 deletions(-) diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index f2a0578474..ee470c915c 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-2220422576-matrix { + .terminal-3293903238-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2220422576-title { + .terminal-3293903238-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2220422576-r1 { fill: #c5c8c6 } - .terminal-2220422576-r2 { fill: #e3e3e3 } - .terminal-2220422576-r3 { fill: #989898 } - .terminal-2220422576-r4 { fill: #e1e1e1 } - .terminal-2220422576-r5 { fill: #4ebf71;font-weight: bold } - .terminal-2220422576-r6 { fill: #1e1e1e } - .terminal-2220422576-r7 { fill: #507bb3 } - .terminal-2220422576-r8 { fill: #e2e2e2 } - .terminal-2220422576-r9 { fill: #808080 } - .terminal-2220422576-r10 { fill: #dde6ed;font-weight: bold } - .terminal-2220422576-r11 { fill: #001541 } - .terminal-2220422576-r12 { fill: #0178d4 } - .terminal-2220422576-r13 { fill: #454a50 } - .terminal-2220422576-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-2220422576-r15 { fill: #000000 } - .terminal-2220422576-r16 { fill: #e4e4e4 } - .terminal-2220422576-r17 { fill: #14191f } - .terminal-2220422576-r18 { fill: #7ae998 } - .terminal-2220422576-r19 { fill: #0a180e;font-weight: bold } - .terminal-2220422576-r20 { fill: #008139 } - .terminal-2220422576-r21 { fill: #fea62b;font-weight: bold } - .terminal-2220422576-r22 { fill: #a7a9ab } - .terminal-2220422576-r23 { fill: #e2e3e3 } + .terminal-3293903238-r1 { fill: #c5c8c6 } + .terminal-3293903238-r2 { fill: #e3e3e3 } + .terminal-3293903238-r3 { fill: #989898 } + .terminal-3293903238-r4 { fill: #e1e1e1 } + .terminal-3293903238-r5 { fill: #4ebf71;font-weight: bold } + .terminal-3293903238-r6 { fill: #1e1e1e } + .terminal-3293903238-r7 { fill: #507bb3 } + .terminal-3293903238-r8 { fill: #e2e2e2 } + .terminal-3293903238-r9 { fill: #808080 } + .terminal-3293903238-r10 { fill: #dde6ed;font-weight: bold } + .terminal-3293903238-r11 { fill: #001541 } + .terminal-3293903238-r12 { fill: #0178d4 } + .terminal-3293903238-r13 { fill: #454a50 } + .terminal-3293903238-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-3293903238-r15 { fill: #000000 } + .terminal-3293903238-r16 { fill: #14191f } + .terminal-3293903238-r17 { fill: #e4e4e4 } + .terminal-3293903238-r18 { fill: #7ae998 } + .terminal-3293903238-r19 { fill: #0a180e;font-weight: bold } + .terminal-3293903238-r20 { fill: #008139 } + .terminal-3293903238-r21 { fill: #fea62b;font-weight: bold } + .terminal-3293903238-r22 { fill: #a7a9ab } + .terminal-3293903238-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Hide help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - - Nf-core pipelines are configured to use a copy of the most common reference ▇▇ - genome files. - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Hide help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + ▅▅ + + Nf-core pipelines are configured to use a copy of the most common reference  + genome files. + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + ▅▅ + For more information about reference genomes in nf-core pipelines, see the  + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-1370375189-matrix { + .terminal-2443134963-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1370375189-title { + .terminal-2443134963-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1370375189-r1 { fill: #c5c8c6 } - .terminal-1370375189-r2 { fill: #e3e3e3 } - .terminal-1370375189-r3 { fill: #989898 } - .terminal-1370375189-r4 { fill: #e1e1e1 } - .terminal-1370375189-r5 { fill: #4ebf71;font-weight: bold } - .terminal-1370375189-r6 { fill: #1e1e1e } - .terminal-1370375189-r7 { fill: #507bb3 } - .terminal-1370375189-r8 { fill: #e2e2e2 } - .terminal-1370375189-r9 { fill: #808080 } - .terminal-1370375189-r10 { fill: #dde6ed;font-weight: bold } - .terminal-1370375189-r11 { fill: #001541 } - .terminal-1370375189-r12 { fill: #14191f } - .terminal-1370375189-r13 { fill: #454a50 } - .terminal-1370375189-r14 { fill: #7ae998 } - .terminal-1370375189-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-1370375189-r16 { fill: #0a180e;font-weight: bold } - .terminal-1370375189-r17 { fill: #000000 } - .terminal-1370375189-r18 { fill: #008139 } - .terminal-1370375189-r19 { fill: #fea62b;font-weight: bold } - .terminal-1370375189-r20 { fill: #a7a9ab } - .terminal-1370375189-r21 { fill: #e2e3e3 } + .terminal-2443134963-r1 { fill: #c5c8c6 } + .terminal-2443134963-r2 { fill: #e3e3e3 } + .terminal-2443134963-r3 { fill: #989898 } + .terminal-2443134963-r4 { fill: #e1e1e1 } + .terminal-2443134963-r5 { fill: #4ebf71;font-weight: bold } + .terminal-2443134963-r6 { fill: #1e1e1e } + .terminal-2443134963-r7 { fill: #507bb3 } + .terminal-2443134963-r8 { fill: #e2e2e2 } + .terminal-2443134963-r9 { fill: #808080 } + .terminal-2443134963-r10 { fill: #dde6ed;font-weight: bold } + .terminal-2443134963-r11 { fill: #001541 } + .terminal-2443134963-r12 { fill: #14191f } + .terminal-2443134963-r13 { fill: #454a50 } + .terminal-2443134963-r14 { fill: #7ae998 } + .terminal-2443134963-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-2443134963-r16 { fill: #0a180e;font-weight: bold } + .terminal-2443134963-r17 { fill: #000000 } + .terminal-2443134963-r18 { fill: #008139 } + .terminal-2443134963-r19 { fill: #fea62b;font-weight: bold } + .terminal-2443134963-r20 { fill: #a7a9ab } + .terminal-2443134963-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges▁▁ - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use code lintersThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - prettier. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Include citationsInclude pipeline tools Show help  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Show help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help ▃▃ + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use code lintersThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + editor-config and  + prettier. + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Include citationsInclude pipeline tools Show help  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  From 1be17370b30fe34adccb5b698d26145e05522e99 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 21 Aug 2024 14:56:37 +0200 Subject: [PATCH 044/151] check plugin includes in pipeline --- nf_core/pipelines/lint/__init__.py | 3 ++ nf_core/pipelines/lint/nextflow_config.py | 4 ++- nf_core/pipelines/lint/plugin_includes.py | 38 +++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 nf_core/pipelines/lint/plugin_includes.py diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index ed833d3219..219cdd8f5c 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -45,6 +45,7 @@ from .nfcore_yml import nfcore_yml from .pipeline_name_conventions import pipeline_name_conventions from .pipeline_todos import pipeline_todos +from .plugin_includes import plugin_includes from .readme import readme from .schema_description import schema_description from .schema_lint import schema_lint @@ -92,6 +93,7 @@ class PipelineLint(nf_core.utils.Pipeline): nfcore_yml = nfcore_yml pipeline_name_conventions = pipeline_name_conventions pipeline_todos = pipeline_todos + plugin_includes = plugin_includes readme = readme schema_description = schema_description schema_lint = schema_lint @@ -135,6 +137,7 @@ def _get_all_lint_tests(release_mode): "actions_awsfulltest", "readme", "pipeline_todos", + "plugin_includes", "pipeline_name_conventions", "template_strings", "schema_lint", diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 6e292deb09..a735c5042b 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -170,7 +170,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ] # Lint for plugins - config_plugins = ast.literal_eval(self.nf_config.get("plugins", "").strip("\"")) + config_plugins = ast.literal_eval(self.nf_config.get("plugins", "")) found_plugins = [] if len(config_plugins) == 0: failed.append("nextflow.config contains an empty plugins scope") @@ -185,6 +185,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") if "nf-schema" in found_plugins: + passed.append("Found nf-schema plugin") if self.nf_config.get("validation.help.enabled", "false") == "false": failed.append("The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages") config_fail_ifdefined.extend([ @@ -195,6 +196,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ]) if "nf-validation" in found_plugins: + passed.append("Found nf-validation plugin") warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") diff --git a/nf_core/pipelines/lint/plugin_includes.py b/nf_core/pipelines/lint/plugin_includes.py new file mode 100644 index 0000000000..247c4e493b --- /dev/null +++ b/nf_core/pipelines/lint/plugin_includes.py @@ -0,0 +1,38 @@ +import ast +import glob +import logging +import re +from typing import Dict, List + +log = logging.getLogger(__name__) + + +def plugin_includes(self) -> Dict[str, List[str]]: + """Checks the include statements in the all *.nf files for plugin includes + + When nf-schema is used in an nf-core pipeline, the include statements of the plugin + functions have to use nf-schema instead of nf-validation and vice versa + """ + config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", ""))] + validation_plugin = "nf-validation" if "nf-validation" in config_plugins else "nf-schema" + + passed = [] + warned = [] + failed = [] + ignored = [] + + plugin_include_pattern = re.compile(r"^include\s*{[^}]+}\s*from\s*[\"']plugin/([^\"']+)[\"']\s*$", re.MULTILINE) + workflow_files = [file for file in glob.glob(f"{self.wf_path}/**/*.nf", recursive=True) if not file.startswith("./modules/")] + test_passed = True + for file in workflow_files: + with open(file, "r") as of: + plugin_includes = re.findall(plugin_include_pattern, of.read()) + for include in plugin_includes: + if include not in ["nf-validation", "nf-schema"]: continue + if include != validation_plugin: + test_passed = False + failed.append(f"Found a `{include}` plugin import in `{file[2:]}`, but `{validation_plugin}` was used in `nextflow.config`") + + if test_passed: passed.append("No wrong validation plugin imports have been found") + + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} From f7137ca3c6e0240033cb56e690125cfbed54301b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 21 Aug 2024 16:44:58 +0200 Subject: [PATCH 045/151] Apply suggestions from code review --- nf_core/pipelines/create/templatefeatures.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index ff56ee190e..6a5fac358d 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -278,7 +278,7 @@ email: adaptivecard: skippable_paths: - "assets/adaptivecard.json" - short_description: "Add template for status messages" + short_description: "Support Microsoft Teams notifications" description: "Enable pipeline status update messages through Microsoft Teams" help_text: | This adds an Adaptive Card. A snippets of user interface. @@ -291,7 +291,7 @@ adaptivecard: slackreport: skippable_paths: - "assets/slackreport.json" - short_description: "Add template for slack status messages" + short_description: "Support Slack notifications" description: "Enable pipeline status update messages through Slack" help_text: | This adds an JSON template used as a template for pipeline update messages in Slack. From 2f1a4428dc73e15aea2fc001bec4df323700035e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 14:51:07 +0200 Subject: [PATCH 046/151] make nf-schema optional in the template --- .../assets/schema_input.json | 2 +- nf_core/pipeline-template/main.nf | 1 - nf_core/pipeline-template/modules.json | 4 +- nf_core/pipeline-template/nextflow.config | 9 ++-- .../pipeline-template/nextflow_schema.json | 3 +- .../utils_nfcore_pipeline_pipeline/main.nf | 18 +++++--- .../nf-core/utils_nfcore_pipeline/main.nf | 46 +++++++++++++++++++ .../pipeline-template/workflows/pipeline.nf | 2 +- nf_core/pipelines/create/templatefeatures.yml | 10 ++++ nf_core/pipelines/lint/nextflow_config.py | 20 ++++---- 10 files changed, 90 insertions(+), 25 deletions(-) diff --git a/nf_core/pipeline-template/assets/schema_input.json b/nf_core/pipeline-template/assets/schema_input.json index e76b95fa99..28a468adaf 100644 --- a/nf_core/pipeline-template/assets/schema_input.json +++ b/nf_core/pipeline-template/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/{{ name }}/master/assets/schema_input.json", "title": "{{ name }} pipeline - params.input schema", "description": "Schema for the file provided with params.input", diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index fddfc5489e..f644f52afc 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -76,7 +76,6 @@ workflow { // PIPELINE_INITIALISATION ( params.version, - params.help, params.validate_params, params.monochrome_logs, args, diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 1a3b96490d..0a883b60b7 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -29,12 +29,12 @@ "branch": "master", "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", "installed_by": ["subworkflows"] - }, + }{% if nf_schema %}, "utils_nfschema_plugin": { "branch": "master", "git_sha": "a3e87febb28bd0461c22a917c5b2c1492053ef85", "installed_by": ["subworkflows"] - } + }{% endif %} } } } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index bb55678001..4acc34e409 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -37,9 +37,9 @@ params { plaintext_email = false monochrome_logs = false hook_url = null - help = false + {% if nf_schema %}help = false helpFull = false - showHidden = false + showHidden = false{% endif %} version = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' @@ -260,13 +260,14 @@ manifest { doi = '' } +{% if nf_schema -%} // Nextflow plugins plugins { id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { - defaultIgnoreParams = ["genomes"] + defaultIgnoreParams = ["genomes", "helpFull", "showHidden", "help-full", "show-hidden"] // The last 4 parameters are here because of a bug in nf-schema. This will be fixed in a later version help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " @@ -293,7 +294,7 @@ validation { afterText = validation.help.afterText } } - +{% endif -%} // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 908d310171..c729c370bf 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -40,8 +40,7 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" - } - {% endif %} + }{% endif %} } }, {%- if igenomes %} diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index a994cfd584..1fef6912b3 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -8,17 +8,14 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' +{% if nf_schema %}include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' include { paramsSummaryMap } from 'plugin/nf-schema' -include { samplesheetToList } from 'plugin/nf-schema' -include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +include { samplesheetToList } from 'plugin/nf-schema'{% endif %} include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' -include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' -include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' include { imNotification } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' -include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' /* ======================================================================================== @@ -50,12 +47,14 @@ workflow PIPELINE_INITIALISATION { workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 ) + {% if nf_schema %} // // Validate parameters and generate parameter summary to stdout // UTILS_NFSCHEMA_PLUGIN ( validate_params ) + {% endif %} // // Check config provided to the pipeline @@ -74,6 +73,7 @@ workflow PIPELINE_INITIALISATION { // // Create channel from input file provided through params.input // + {% if nf_schema %} Channel .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")) .map { @@ -93,6 +93,12 @@ workflow PIPELINE_INITIALISATION { return [ meta, fastqs.flatten() ] } .set { ch_samplesheet } + {% else %} + Channel + .fromPath(params.input) + .splitCsv(header: true, strip: true) + .set { ch_samplesheet } + {% endif %} emit: samplesheet = ch_samplesheet diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a5dcd5f83d..14558c3927 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -61,6 +61,25 @@ def checkProfileProvided(nextflow_cli_args) { } } +// +// Citation string for pipeline +// +def workflowCitation() { + def temp_doi_ref = "" + String[] manifest_doi = workflow.manifest.doi.tokenize(",") + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" + return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + + "* The pipeline\n" + + temp_doi_ref + "\n" + + "* The nf-core framework\n" + + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + + "* Software dependencies\n" + + " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" +} + // // Generate workflow version string // @@ -138,6 +157,33 @@ def paramsSummaryMultiqc(summary_params) { return yaml_file_text } +// +// nf-core logo +// +def nfCoreLogo(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + String.format( + """\n + ${dashedLine(monochrome_logs)} + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} + ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} + ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} + ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} + ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} + ${dashedLine(monochrome_logs)} + """.stripIndent() + ) +} + +// +// Return dashed line +// +def dashedLine(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" +} + // // ANSII colours used for terminal logging // diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 4bb0913211..8d2773820d 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -6,7 +6,7 @@ include { FASTQC } from '../modules/nf-core/fastqc/main' {% if multiqc %}include { MULTIQC } from '../modules/nf-core/multiqc/main'{% endif %} -include { paramsSummaryMap } from 'plugin/nf-schema' +{% if nf_schema %}include { paramsSummaryMap } from 'plugin/nf-schema'{% endif %} {% if multiqc %}include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline'{% endif %} include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' {% if citations or multiqc %}include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'{% endif %} diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index b97bf347ab..7a24b3d91a 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -242,3 +242,13 @@ changelog: - "CHANGELOG.md" nfcore_pipelines: False custom_pipelines: True +nf_schema: + skippable_paths: + - "subworkflows/nf-core/utils_nfschema_plugin" + short_description: "Use nf-schema for this pipeline." + help_text: | + nf-schema is used to validate input parameters based on a JSON schema. + It also provides helper functionality to create help messages, get a summary + of changed parameters and validate and convert a samplesheet to a channel. + nfcore_pipelines: False + custom_pipelines: False diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index a735c5042b..1dfdabaf10 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -143,8 +143,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["process.memory"], ["process.time"], ["params.outdir"], - ["params.input"], - ["validation.help.enabled"] + ["params.input"] ] # Throw a warning if these are missing config_warn = [ @@ -152,12 +151,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["timeline.file"], ["trace.file"], ["report.file"], - ["dag.file"], - ["validation.help.beforeText"], - ["validation.help.afterText"], - ["validation.help.command"], - ["validation.summary.beforeText"], - ["validation.summary.afterText"] + ["dag.file"] ] # Old depreciated vars - fail if present config_fail_ifdefined = [ @@ -188,6 +182,16 @@ def nextflow_config(self) -> Dict[str, List[str]]: passed.append("Found nf-schema plugin") if self.nf_config.get("validation.help.enabled", "false") == "false": failed.append("The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages") + config_fail.extend([ + ["validation.help.enabled"] + ]) + config_warn.extend([ + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.help.command"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"] + ]) config_fail_ifdefined.extend([ "params.validationFailUnrecognisedParams", "params.validationLenientMode", From 364b54bde2994927d3124bfdf53d97bd1ecc6350 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 15:57:29 +0200 Subject: [PATCH 047/151] small fixes for template without nf-schema --- .../utils_nfcore_pipeline_pipeline/main.nf | 33 ++++++++++--------- .../pipeline-template/workflows/pipeline.nf | 6 ++-- nf_core/pipelines/lint/nextflow_config.py | 6 ++-- nf_core/pipelines/lint/plugin_includes.py | 2 +- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index d96f203544..0867a0a4c9 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -77,9 +77,14 @@ workflow PIPELINE_INITIALISATION { // // Create channel from input file provided through params.input // - {% if nf_schema %} - Channel - .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")) + + Channel{% if nf_schema %} + .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")){% else %} + .fromPath(params.input) + .splitCsv(header: true, strip: true) + .map { row -> + [[id:row.sample], row.fastq_1, row.fastq_2] + }{% endif %} .map { meta, fastq_1, fastq_2 -> if (!fastq_2) { @@ -97,12 +102,6 @@ workflow PIPELINE_INITIALISATION { return [ meta, fastqs.flatten() ] } .set { ch_samplesheet } - {% else %} - Channel - .fromPath(params.input) - .splitCsv(header: true, strip: true) - .set { ch_samplesheet } - {% endif %} emit: samplesheet = ch_samplesheet @@ -130,19 +129,21 @@ workflow PIPELINE_COMPLETION { main: - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - // // Completion email and summary // workflow.onComplete { {%- if email %} if (email || email_on_fail) { - {%- if multiqc %} - completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, multiqc_report.toList()) - {%- else %} - completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, []) - {%- endif %} + completionEmail( + {% if nf_schema %}paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json"){% else %}{}{% endif %}, + email, + email_on_fail, + plaintext_email, + outdir, + monochrome_logs, + {% if multiqc %}multiqc_report.toList(){% else %}[]{% endif %} + ) } {%- endif %} diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 8d2773820d..ca5de36a61 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -60,9 +60,13 @@ workflow {{ short_name|upper }} { Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + {% if nf_schema %} summary_params = paramsSummaryMap( workflow, parameters_schema: "nextflow_schema.json") ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + ch_multiqc_files = ch_multiqc_files.mix( + ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + {% endif %} {%- if citations %} ch_multiqc_custom_methods_description = params.multiqc_methods_description ? @@ -72,8 +76,6 @@ workflow {{ short_name|upper }} { methodsDescriptionText(ch_multiqc_custom_methods_description)) {%- endif %} - ch_multiqc_files = ch_multiqc_files.mix( - ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) {%- if citations %} ch_multiqc_files = ch_multiqc_files.mix( diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 1dfdabaf10..f5689813a4 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -164,10 +164,10 @@ def nextflow_config(self) -> Dict[str, List[str]]: ] # Lint for plugins - config_plugins = ast.literal_eval(self.nf_config.get("plugins", "")) + config_plugins = ast.literal_eval(self.nf_config.get("plugins", "[]")) found_plugins = [] if len(config_plugins) == 0: - failed.append("nextflow.config contains an empty plugins scope") + warned.append("nextflow.config contains an empty plugins scope") for plugin in config_plugins: if "@" not in plugin: failed.append(f"Plugin '{plugin}' does not have a pinned version") @@ -176,7 +176,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: if "nf-validation" in found_plugins and "nf-schema" in found_plugins: failed.append("nextflow.config contains both nf-validation and nf-schema") if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: - failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + warned.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") if "nf-schema" in found_plugins: passed.append("Found nf-schema plugin") diff --git a/nf_core/pipelines/lint/plugin_includes.py b/nf_core/pipelines/lint/plugin_includes.py index 247c4e493b..740c001a11 100644 --- a/nf_core/pipelines/lint/plugin_includes.py +++ b/nf_core/pipelines/lint/plugin_includes.py @@ -13,7 +13,7 @@ def plugin_includes(self) -> Dict[str, List[str]]: When nf-schema is used in an nf-core pipeline, the include statements of the plugin functions have to use nf-schema instead of nf-validation and vice versa """ - config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", ""))] + config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", "[]"))] validation_plugin = "nf-validation" if "nf-validation" in config_plugins else "nf-schema" passed = [] From 4e631384906d2c02ab537276f22ea47c3ceb438d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 16:02:58 +0200 Subject: [PATCH 048/151] bump nextflow version --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .../nf-core/utils_nextflow_pipeline/tests/nextflow.config | 2 +- .../nf-core/utils_nfcore_pipeline/tests/nextflow.config | 2 +- nf_core/pipelines/lint/readme.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 7e90febb74..f07b31e9de 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -38,7 +38,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: go to subdirectory and change nextflow workdir diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index a95a477459..56c6c822a9 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -39,7 +39,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: go to working directory diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config index d0a926bf6d..0fa4abaf43 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.10.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config index d0a926bf6d..0fa4abaf43 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.10.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/nf_core/pipelines/lint/readme.py b/nf_core/pipelines/lint/readme.py index 4c16243690..1c09104258 100644 --- a/nf_core/pipelines/lint/readme.py +++ b/nf_core/pipelines/lint/readme.py @@ -36,7 +36,7 @@ def readme(self): if "nextflow_badge" not in ignore_configs: # Check that there is a readme badge showing the minimum required version of Nextflow - # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) + # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) # and that it has the correct version nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" match = re.search(nf_badge_re, content) From 11efc5547be76f1a6412d049dbfcd10cd337c660 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 16:08:41 +0200 Subject: [PATCH 049/151] update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98fb925338..b4fcfe8646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ - add templatefeatures.yml to python package ([#3112](https://github.com/nf-core/tools/pull/3112)) - add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125)) - add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126)) +- Use nf-schema instead of nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116)) +- add option to exclude nf-schema from the template ([#3116](https://github.com/nf-core/tools/pull/3116)) ### Linting @@ -26,6 +28,10 @@ - Conda module linting: Include package name in log file ([#3014](https://github.com/nf-core/tools/pull/3014)) - Restructure pipeline tests and move pipeline linting into subfolder ([#3070](https://github.com/nf-core/tools/pull/3070)) - Fix module linting warning for process_high_memory ([#3086](https://github.com/nf-core/tools/issues/3086)) +- Linting will now fail when an unpinned plugin is used ([#3116](https://github.com/nf-core/tools/pull/3116)) +- Linting will now check if the schema is correct for the used validation plugin ([#3116])(https://github.com/nf-core/tools/pull/3116) +- Linting will now check the use of the right validation plugin include statements in the workflow scripts ([#3116])(https://github.com/nf-core/tools/pull/3116) +- Full linting for correct use of nf-schema and nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116)) ### Download From 47c3fe324280a54d4cae4227ab7679ac488b1edb Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 16:13:04 +0200 Subject: [PATCH 050/151] fix ruff linting --- nf_core/pipelines/lint/nextflow_config.py | 53 +++++++++++------------ nf_core/pipelines/lint/plugin_includes.py | 24 ++++++---- nf_core/pipelines/schema.py | 17 +++++--- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index f5689813a4..255042ce5a 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -143,16 +143,10 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["process.memory"], ["process.time"], ["params.outdir"], - ["params.input"] + ["params.input"], ] # Throw a warning if these are missing - config_warn = [ - ["manifest.mainScript"], - ["timeline.file"], - ["trace.file"], - ["report.file"], - ["dag.file"] - ] + config_warn = [["manifest.mainScript"], ["timeline.file"], ["trace.file"], ["report.file"], ["dag.file"]] # Old depreciated vars - fail if present config_fail_ifdefined = [ "params.nf_required_version", @@ -181,28 +175,33 @@ def nextflow_config(self) -> Dict[str, List[str]]: if "nf-schema" in found_plugins: passed.append("Found nf-schema plugin") if self.nf_config.get("validation.help.enabled", "false") == "false": - failed.append("The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages") - config_fail.extend([ - ["validation.help.enabled"] - ]) - config_warn.extend([ - ["validation.help.beforeText"], - ["validation.help.afterText"], - ["validation.help.command"], - ["validation.summary.beforeText"], - ["validation.summary.afterText"] - ]) - config_fail_ifdefined.extend([ - "params.validationFailUnrecognisedParams", - "params.validationLenientMode", - "params.validationSchemaIgnoreParams", - "params.validationShowHiddenParams" - ]) + failed.append( + "The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages" + ) + config_fail.extend([["validation.help.enabled"]]) + config_warn.extend( + [ + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.help.command"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"], + ] + ) + config_fail_ifdefined.extend( + [ + "params.validationFailUnrecognisedParams", + "params.validationLenientMode", + "params.validationSchemaIgnoreParams", + "params.validationShowHiddenParams", + ] + ) if "nf-validation" in found_plugins: passed.append("Found nf-validation plugin") - warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") - + warned.append( + "nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/" + ) # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get("nextflow_config", []) if self.lint_config is not None else [] diff --git a/nf_core/pipelines/lint/plugin_includes.py b/nf_core/pipelines/lint/plugin_includes.py index 740c001a11..6a57b1d280 100644 --- a/nf_core/pipelines/lint/plugin_includes.py +++ b/nf_core/pipelines/lint/plugin_includes.py @@ -16,23 +16,29 @@ def plugin_includes(self) -> Dict[str, List[str]]: config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", "[]"))] validation_plugin = "nf-validation" if "nf-validation" in config_plugins else "nf-schema" - passed = [] - warned = [] - failed = [] - ignored = [] + passed: list[str] = [] + warned: list[str] = [] + failed: list[str] = [] + ignored: list[str] = [] plugin_include_pattern = re.compile(r"^include\s*{[^}]+}\s*from\s*[\"']plugin/([^\"']+)[\"']\s*$", re.MULTILINE) - workflow_files = [file for file in glob.glob(f"{self.wf_path}/**/*.nf", recursive=True) if not file.startswith("./modules/")] + workflow_files = [ + file for file in glob.glob(f"{self.wf_path}/**/*.nf", recursive=True) if not file.startswith("./modules/") + ] test_passed = True for file in workflow_files: - with open(file, "r") as of: + with open(file) as of: plugin_includes = re.findall(plugin_include_pattern, of.read()) for include in plugin_includes: - if include not in ["nf-validation", "nf-schema"]: continue + if include not in ["nf-validation", "nf-schema"]: + continue if include != validation_plugin: test_passed = False - failed.append(f"Found a `{include}` plugin import in `{file[2:]}`, but `{validation_plugin}` was used in `nextflow.config`") + failed.append( + f"Found a `{include}` plugin import in `{file[2:]}`, but `{validation_plugin}` was used in `nextflow.config`" + ) - if test_passed: passed.append("No wrong validation plugin imports have been found") + if test_passed: + passed.append("No wrong validation plugin imports have been found") return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index e9c5556dbb..ccbc86645e 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -5,7 +5,6 @@ import logging import tempfile import webbrowser -import re from pathlib import Path from typing import Union @@ -58,10 +57,10 @@ def set_schema_filename(self, schema: str) -> None: basepath = "/".join(str(schema).split("/")[:-1]) config = f"{basepath}/nextflow.config" if basepath != "" else "nextflow.config" self._update_validation_plugin_from_config(config) - + def get_schema_filename(self) -> str: return self._schema_filename - + def del_schema_filename(self) -> None: del self._schema_filename @@ -71,7 +70,7 @@ def _update_validation_plugin_from_config(self, config: str) -> None: plugin = "nf-schema" conf = nf_core.utils.fetch_wf_config(Path(self.schema_filename).parent) - plugins = str(conf.get("plugins", "")).strip("\"").strip("'").strip(" ").split(",") + plugins = str(conf.get("plugins", "")).strip('"').strip("'").strip(" ").split(",") plugin_found = False for plugin_instance in plugins: if "nf-schema" in plugin_instance: @@ -95,7 +94,7 @@ def _update_validation_plugin_from_config(self, config: str) -> None: # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": self.defs_notation = "$defs" - ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default + ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default ignored_params_config = conf.get("validation", {}).get("defaultIgnoreParams", []) if len(ignored_params_config) > 0: ignored_params.extend(ignored_params_config) @@ -447,7 +446,9 @@ def validate_schema(self, schema=None): # Add a small check for older nf-schema JSON schemas if "defs" in schema: - raise AssertionError(f'Using "defs" for schema definitions is not supported. Please use {self.defs_notation} instead') + raise AssertionError( + f'Using "defs" for schema definitions is not supported. Please use {self.defs_notation} instead' + ) for d_key, d_schema in schema.get(self.defs_notation, {}).items(): # Check that this definition is mentioned in allOf @@ -458,7 +459,9 @@ def validate_schema(self, schema=None): if allOf["$ref"] == f"#/{self.defs_notation}/{d_key}": in_allOf = True if not in_allOf: - raise AssertionError(f"Definition subschema `#/{self.defs_notation}/{d_key}` not included in schema `allOf`") + raise AssertionError( + f"Definition subschema `#/{self.defs_notation}/{d_key}` not included in schema `allOf`" + ) # TODO add support for nested parameters for d_param_id in d_schema.get("properties", {}): From 09bfca2663275c37173ce8018c858164974479f8 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 16:29:39 +0200 Subject: [PATCH 051/151] add a plugin includes test --- tests/pipelines/lint/test_plugin_includes.py | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/pipelines/lint/test_plugin_includes.py diff --git a/tests/pipelines/lint/test_plugin_includes.py b/tests/pipelines/lint/test_plugin_includes.py new file mode 100644 index 0000000000..21e6107c14 --- /dev/null +++ b/tests/pipelines/lint/test_plugin_includes.py @@ -0,0 +1,23 @@ +import nf_core.pipelines.lint + +from ..test_lint import TestLint + +class TestLintNextflowConfig(TestLint): + def setUp(self) -> None: + super().setUp() + self.new_pipeline = self._make_pipeline_copy() + + def test_default_values_match(self): + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + result = lint_obj.plugin_includes() + assert len(result["failed"]) == 0 + assert len(result["warned"]) == 0 + + def test_wrong_include(self): + test_path = self.new_pipeline / "test.nf" + with open(test_path, "w") as of: + of.write("include { paramsSummary } from 'plugin/nf-validation'\n") + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + result = lint_obj.plugin_includes() + assert len(result["failed"]) == 1 + assert len(result["warned"]) == 0 \ No newline at end of file From 44dbbf7a2b095144cb954bb1ed09578776979c92 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 23 Aug 2024 09:50:51 +0200 Subject: [PATCH 052/151] update review comments --- nf_core/pipeline-template/nextflow.config | 7 ++++--- nf_core/pipelines/create/templatefeatures.yml | 7 ++++--- nf_core/pipelines/lint/nextflow_config.py | 1 - nf_core/pipelines/schema.py | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ad800cc55b..3b3183ece1 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -273,6 +273,7 @@ validation { help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " + {% if is_nfcore %} beforeText = """ -\033[2m----------------------------------------------------\033[0m- \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m @@ -289,12 +290,12 @@ validation { * Software dependencies https://github.com/${manifest.name}/blob/master/CITATIONS.md -""" - } +"""{% endif %} + }{% if is_nfcore %} summary { beforeText = validation.help.beforeText afterText = validation.help.afterText - } + }[% endif %] } {% endif -%} diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 29b53eb0a7..224bc38266 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -245,13 +245,14 @@ changelog: nf_schema: skippable_paths: - "subworkflows/nf-core/utils_nfschema_plugin" - short_description: "Use nf-schema for this pipeline." + short_description: "Use nf-schema" + description: "Use nf-schema for this pipeline." help_text: | nf-schema is used to validate input parameters based on a JSON schema. It also provides helper functionality to create help messages, get a summary of changed parameters and validate and convert a samplesheet to a channel. - nfcore_pipelines: False - custom_pipelines: False + nfcore_pipelines: True + custom_pipelines: True license: skippable_paths: - "LICENSE" diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 255042ce5a..71eb6f118f 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -367,7 +367,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) # Check for the availability of the "test" configuration profile by parsing nextflow.config - # Also check for the presence of nf-validation/nf-schema and check if they have pinned versions with open(Path(self.wf_path, "nextflow.config")) as f: content = f.read() diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index ccbc86645e..04f5ca2ace 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -447,7 +447,7 @@ def validate_schema(self, schema=None): # Add a small check for older nf-schema JSON schemas if "defs" in schema: raise AssertionError( - f'Using "defs" for schema definitions is not supported. Please use {self.defs_notation} instead' + f'Using "defs" for schema definitions is not supported. Please use "{self.defs_notation}" instead' ) for d_key, d_schema in schema.get(self.defs_notation, {}).items(): From ea51814bc2b1549c14605ba66591e7043bb5e4b3 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 23 Aug 2024 10:17:15 +0200 Subject: [PATCH 053/151] remove duplicate code --- nf_core/pipelines/schema.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 04f5ca2ace..dbe851415f 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -85,11 +85,6 @@ def _update_validation_plugin_from_config(self, config: str) -> None: if not plugin_found: log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") - if "nf-validation" in plugins: - plugin = "nf-validation" - elif "nf-schema" in plugins: - plugin = "nf-schema" - self.validation_plugin = plugin # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": From 113fa4226cbdb471607615cf9c45b61bcfbc55c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 23 Aug 2024 10:06:12 +0000 Subject: [PATCH 054/151] update utils_nextflow_pipeline --- nf_core/pipeline-template/modules.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index eb9391b29a..8b156083f3 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -22,7 +22,7 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "20c03aede5a80ff520a905cea1f8ca121b5bb661", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { From 195fc5168c9810d36f994996d588c219b72b9eef Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 23 Aug 2024 11:35:43 +0000 Subject: [PATCH 055/151] Uncomment nextflow extension and align comments --- .gitpod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index b2fbb73133..b93095c780 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -11,12 +11,12 @@ tasks: unset JAVA_TOOL_OPTIONS vscode: - extensions: # based on nf-core.nf-core-extensionpack + extensions: - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code - charliermarsh.ruff # Code linter Ruff From 65f26b1455b5592ec301fb618464372b3a8280da Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 23 Aug 2024 11:44:48 +0000 Subject: [PATCH 056/151] Add Digest comment and remove conda defaults channel --- nf_core/gitpod/gitpod.Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index fd0e8bb799..944222668b 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,6 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . +# See https://docs.renovatebot.com/docker/#digest-pinning for why a digest is used. FROM gitpod/workspace-base@sha256:f189a4195c3861365356f9c1b438ab26fd88e1ff46ce2843afc62861fc982e0c USER root @@ -40,9 +41,8 @@ RUN chown -R gitpod:gitpod /opt/conda /usr/src/nf_core # Change user to gitpod USER gitpod -# Install nextflow, nf-core, Mamba, and pytest-workflow -RUN conda config --add channels defaults && \ - conda config --add channels bioconda && \ +# Install nextflow, nf-core, nf-test, and other useful tools +RUN conda config --add channels bioconda && \ conda config --add channels conda-forge && \ conda config --set channel_priority strict && \ conda install --quiet --yes --name base \ From 506491f66e3dd612e726b508ab72e125fd8e31b8 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 23 Aug 2024 12:04:32 +0000 Subject: [PATCH 057/151] Blank JAVA_TOOL_OPTIONS --- nf_core/gitpod/gitpod.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 944222668b..849729966b 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -63,3 +63,4 @@ RUN nextflow self-update && \ # Setup pdiff for nf-test diffs ENV NFT_DIFF="pdiff" ENV NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" +ENV JAVA_TOOL_OPTIONS= From 411cd22da23380799e7535b7340a5a0e16510bd7 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 23 Aug 2024 12:07:25 +0000 Subject: [PATCH 058/151] Remove unset JAVA_TOOL_OPTIONS --- .gitpod.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index b93095c780..efe193f35f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -6,9 +6,6 @@ tasks: python -m pip install -r requirements-dev.txt pre-commit install --install-hooks nextflow self-update - - name: unset JAVA_TOOL_OPTIONS - command: | - unset JAVA_TOOL_OPTIONS vscode: extensions: From bc6ecda3aec81e24b858b6e98e62eb30ad13dd0a Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 23 Aug 2024 12:11:13 +0000 Subject: [PATCH 059/151] Remove unset JAVA_TOOL_OPTIONS from pipeline template --- nf_core/pipeline-template/.gitpod.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index 30e85ed977..5907fb59c9 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -4,9 +4,6 @@ tasks: command: | pre-commit install --install-hooks nextflow self-update - - name: unset JAVA_TOOL_OPTIONS - command: | - unset JAVA_TOOL_OPTIONS vscode: extensions: # based on nf-core.nf-core-extensionpack @@ -15,7 +12,7 @@ vscode: - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files{% endif %} - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code - charliermarsh.ruff # Code linter Ruff From d324760487b25de1f12027699099bc5d649eaf60 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 23 Aug 2024 12:14:18 +0000 Subject: [PATCH 060/151] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98fb925338..2e13e82f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ - Components: allow spaces at the betinning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 ([#3122](https://github.com/nf-core/tools/pull/3122)) - Update python:3.12-slim Docker digest to 59c7332 ([#3124](https://github.com/nf-core/tools/pull/3124)) +- Update gitpod ([#3136](https://github.com/nf-core/tools/pull/3136)) ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09] From 1bcb70fbfc3b17b3a73dec382a96a20d02108617 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 23 Aug 2024 14:26:26 +0200 Subject: [PATCH 061/151] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e13e82f5e..aca2e6f200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,7 +73,7 @@ - Components: allow spaces at the betinning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 ([#3122](https://github.com/nf-core/tools/pull/3122)) - Update python:3.12-slim Docker digest to 59c7332 ([#3124](https://github.com/nf-core/tools/pull/3124)) -- Update gitpod ([#3136](https://github.com/nf-core/tools/pull/3136)) +- Update gitpod setup ([#3136](https://github.com/nf-core/tools/pull/3136)) ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09] From 6e018ce6981548fb11ae5f34e3c476305695ad8f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 23 Aug 2024 15:41:07 +0200 Subject: [PATCH 062/151] organise changelog and bump version to 3.0.0dev --- CHANGELOG.md | 55 +++++++++++++++++++++++++++------------------------- setup.py | 2 +- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aca2e6f200..42cb2a62aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # nf-core/tools: Changelog -## v2.14.2dev +## v3.0.0dev + +**Highlights** + +- Pipeline commands are renamed from nf-core to nf-core pipelines +- More customisation for pipeline templates +- A Text User Interface app when running nf-core pipelines create +- The pipeline template will come with the nf-schema plugin +- New command nf-core pipelines rocrate to create a Research Object (RO) crate for a pipeline +- `nf-core licences` command is deprecated. +- Pipeline linting will run with the used version of the template +- The structure of nf-core/tools pytests has been updated +- The structure of the API docs has been updated ### Template @@ -9,14 +21,13 @@ - Remove deprecated syntax ([#3046](https://github.com/nf-core/tools/pull/3046)) - Use filename in code block for `params.yml` ([#3055](https://github.com/nf-core/tools/pull/3055)) - Remove release announcement for non nf-core pipelines ([#3072](https://github.com/nf-core/tools/pull/3072)) +- handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108), [#3112](https://github.com/nf-core/tools/pull/3112)) - add option to exclude code linters for custom pipeline template ([#3084](https://github.com/nf-core/tools/pull/3084)) - add option to exclude citations for custom pipeline template ([#3101](https://github.com/nf-core/tools/pull/3101)) - add option to exclude gitpod for custom pipeline template ([#3100](https://github.com/nf-core/tools/pull/3100)) - add option to exclude codespaces from pipeline template ([#3105](https://github.com/nf-core/tools/pull/3105)) - add option to exclude multiqc from pipeline template ([#3103](https://github.com/nf-core/tools/pull/3103)) - add option to exclude changelog from custom pipeline template ([#3104](https://github.com/nf-core/tools/pull/3104)) -- handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108)) -- add templatefeatures.yml to python package ([#3112](https://github.com/nf-core/tools/pull/3112)) - add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125)) - add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126)) @@ -27,53 +38,45 @@ - Restructure pipeline tests and move pipeline linting into subfolder ([#3070](https://github.com/nf-core/tools/pull/3070)) - Fix module linting warning for process_high_memory ([#3086](https://github.com/nf-core/tools/issues/3086)) -### Download +### Pipeline create command + +- Create: allow more special characters on the pipeline name for non-nf-core pipelines ([#3008](https://github.com/nf-core/tools/pull/3008)) +- Create: Mock git cretentials to generate stable textual snapshots ([#3007](https://github.com/nf-core/tools/pull/3007)) +- Create app: display input textbox with equally spaced grid ([#3038](https://github.com/nf-core/tools/pull/3038)) +- Pipelines: allow numbers in custom pipeline name ([#3094](https://github.com/nf-core/tools/pull/3094)) ### Components - The `modules_nfcore` tag in the `main.nf.test` file of modules/subworkflows now displays the organization name in custom modules repositories ([#3005](https://github.com/nf-core/tools/pull/3005)) +- Add `--migrate_pytest` option to `nf-core test` command ([#3085](https://github.com/nf-core/tools/pull/3085)) +- Components: allow spaces at the beginning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115)) ### General -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.4 ([#2974](https://github.com/nf-core/tools/pull/2974)) -- Update gitpod/workspace-base Docker digest to 92dd1bc ([#2982](https://github.com/nf-core/tools/pull/2982)) - Update output of generation script for API docs to new structure ([#2988](https://github.com/nf-core/tools/pull/2988)) - Add no clobber and put bash options on their own line ([#2991](https://github.com/nf-core/tools/pull/2991)) - update minimal textual version and snapshots ([#2998](https://github.com/nf-core/tools/pull/2998)) - move pipeline subcommands for v3.0 ([#2983](https://github.com/nf-core/tools/pull/2983)) - return directory if base_dir is the root directory ([#3003](https://github.com/nf-core/tools/pull/3003)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.6 ([#3006](https://github.com/nf-core/tools/pull/3006)) -- Create: allow more special characters on the pipeline name for non-nf-core pipelines ([#3008](https://github.com/nf-core/tools/pull/3008)) - Remove nf-core licences command ([#3012](https://github.com/nf-core/tools/pull/3012)) - README - absolute image paths ([#3013](https://github.com/nf-core/tools/pull/3013)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.7 ([#3015](https://github.com/nf-core/tools/pull/3015)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.8 ([#3017](https://github.com/nf-core/tools/pull/3017)) -- Update python:3.12-slim Docker digest to e3ae8cf ([#3020](https://github.com/nf-core/tools/pull/3020)) -- Update python:3.12-slim Docker digest to 2fba8e7 ([#3023](https://github.com/nf-core/tools/pull/3023)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.10 ([#3031](https://github.com/nf-core/tools/pull/3031)) - Add warning deprecation message to top-level commands ([#3036](https://github.com/nf-core/tools/pull/3036)) -- Create: Mock git cretentials to generate stable textual snapshots ([#3007](https://github.com/nf-core/tools/pull/3007)) - move pipeline commands to functions to avoid duplication ([#3039](https://github.com/nf-core/tools/pull/3039)) -- Create app: display input textbox with equally spaced grid ([#3038](https://github.com/nf-core/tools/pull/3038)) -- Update python:3.12-slim Docker digest to da2d7af ([#3041](https://github.com/nf-core/tools/pull/3041)) -- Update gitpod/workspace-base Docker digest to 0f38224 ([#3048](https://github.com/nf-core/tools/pull/3048)) - update output_dir for api docs to new website structure ([#3051](https://github.com/nf-core/tools/pull/3051)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.5.1 ([#3052](https://github.com/nf-core/tools/pull/3052)) - Add `--limit-output` argument for modules/subworkflow update ([#3047](https://github.com/nf-core/tools/pull/3047)) - update api docs to new structure ([#3054](https://github.com/nf-core/tools/pull/3054)) -- Update to pytest v8 and move it to dev dependencies ([#3058](https://github.com/nf-core/tools/pull/3058)) - handle new jsonschema error type ([#3061](https://github.com/nf-core/tools/pull/3061)) -- Update python:3.12-slim Docker digest to f11725a ([#3071](https://github.com/nf-core/tools/pull/3071)) - Fix number of arguments for pipelines_create within the command_create function ([#3074](https://github.com/nf-core/tools/pull/3074)) -- Update python:3.12-slim Docker digest to 740d94a ([#3079](https://github.com/nf-core/tools/pull/3079)) -- Add `--migrate_pytest` option to `nf-core test` command ([#3085](https://github.com/nf-core/tools/pull/3085)) -- Update pre-commit hook pre-commit/mirrors-mypy to v1.11.1 ([#3091](https://github.com/nf-core/tools/pull/3091)) -- Pipelines: allow numbers in custom pipeline name ([#3094](https://github.com/nf-core/tools/pull/3094)) - Add bot action to update textual snapshots and write bot documentation ([#3102](https://github.com/nf-core/tools/pull/3102)) -- Components: allow spaces at the betinning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115)) +- Update gitpod setup ([#3136](https://github.com/nf-core/tools/pull/3136)) + +## Version updates + - Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 ([#3122](https://github.com/nf-core/tools/pull/3122)) +- Update gitpod/workspace-base Docker digest to 92dd1bc ([#2982](https://github.com/nf-core/tools/pull/2982)) - Update python:3.12-slim Docker digest to 59c7332 ([#3124](https://github.com/nf-core/tools/pull/3124)) -- Update gitpod setup ([#3136](https://github.com/nf-core/tools/pull/3136)) +- Update pre-commit hook pre-commit/mirrors-mypy to v1.11.1 ([#3091](https://github.com/nf-core/tools/pull/3091)) +- Update to pytest v8 and move it to dev dependencies ([#3058](https://github.com/nf-core/tools/pull/3058)) ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09] diff --git a/setup.py b/setup.py index 8f32daa904..45df29b8bc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.14.2dev" +version = "3.0.0dev" with open("README.md") as f: readme = f.read() From 41b2676c8746e92960907c3263f296ea7f606d96 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 23 Aug 2024 15:52:40 +0200 Subject: [PATCH 063/151] more details! :D --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42cb2a62aa..2a3e98f63b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,13 @@ **Highlights** -- Pipeline commands are renamed from nf-core to nf-core pipelines -- More customisation for pipeline templates -- A Text User Interface app when running nf-core pipelines create -- The pipeline template will come with the nf-schema plugin -- New command nf-core pipelines rocrate to create a Research Object (RO) crate for a pipeline +- Pipeline commands are renamed from `nf-core ` to `nf-core pipelines ` to match the structure of modules and subworkflows commands. +- More customisation for pipeline templates. The template has been divided into features which can be skipped, making it easier to avoid annoying merge conflicts, for example, for pipeline which don't have a FastQC module. +- A new Text User Interface app when running nf-core pipelines create +- The pipeline template will come with the `nf-schema` plugin +- Pipeline linting will run with the used version of the template to minimise errors in opened PRs with every new tools release. +- New command `nf-core pipelines rocrate` to create a [Research Object (RO) crate](https://www.researchobject.org/ro-crate/about_ro_crate) for a pipeline - `nf-core licences` command is deprecated. -- Pipeline linting will run with the used version of the template - The structure of nf-core/tools pytests has been updated - The structure of the API docs has been updated From 8000e1f5c31e7903ff72bb2453b67ca8db262f67 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 23 Aug 2024 16:04:19 +0200 Subject: [PATCH 064/151] add suggestions from @mashehu --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a3e98f63b..5590abc04a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,12 @@ **Highlights** -- Pipeline commands are renamed from `nf-core ` to `nf-core pipelines ` to match the structure of modules and subworkflows commands. -- More customisation for pipeline templates. The template has been divided into features which can be skipped, making it easier to avoid annoying merge conflicts, for example, for pipeline which don't have a FastQC module. -- A new Text User Interface app when running nf-core pipelines create -- The pipeline template will come with the `nf-schema` plugin -- Pipeline linting will run with the used version of the template to minimise errors in opened PRs with every new tools release. -- New command `nf-core pipelines rocrate` to create a [Research Object (RO) crate](https://www.researchobject.org/ro-crate/about_ro_crate) for a pipeline +- Pipeline commands are renamed from `nf-core ` to `nf-core pipelines ` to follow the same command structure as modules and subworkflows commands. +- More customisation for pipeline templates. The template has been divided into features which can be skipped, e.g. you can create a new pipeline without any traces of FastQC in it. +- A new Text User Interface app when running `nf-core pipelines create` to help us guide you through the process better (no worries, you can still use the cli if you give all values as parameters) +- We replaced nf-validation with nf-schema in the pipeline template +- CI tests now lint with the nf-core tools version matching the template version of the pipeline, to minimise errors in opened PRs with every new tools release. +- New command `nf-core pipelines ro-crate` to create a [Research Object (RO) crate](https://www.researchobject.org/ro-crate/) for a pipeline - `nf-core licences` command is deprecated. - The structure of nf-core/tools pytests has been updated - The structure of the API docs has been updated From f2b282c696ebfff78ac4e328dca3d9a25b258ec4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 21 Aug 2024 16:12:58 +0200 Subject: [PATCH 065/151] add option to exclude fastqc from pipeline template --- nf_core/pipeline-template/CITATIONS.md | 7 +++---- nf_core/pipeline-template/README.md | 4 ++-- nf_core/pipeline-template/conf/modules.config | 2 ++ nf_core/pipeline-template/docs/output.md | 9 ++++++--- nf_core/pipeline-template/modules.json | 3 ++- .../local/utils_nfcore_pipeline_pipeline/main.nf | 4 ++-- nf_core/pipeline-template/workflows/pipeline.nf | 4 +++- nf_core/pipelines/create/templatefeatures.yml | 10 ++++++++++ 8 files changed, 30 insertions(+), 13 deletions(-) diff --git a/nf_core/pipeline-template/CITATIONS.md b/nf_core/pipeline-template/CITATIONS.md index 7cf37e5018..2373f1de7f 100644 --- a/nf_core/pipeline-template/CITATIONS.md +++ b/nf_core/pipeline-template/CITATIONS.md @@ -12,11 +12,10 @@ ## Pipeline tools -- [FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/) +{% if fastqc %}- [FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/) - > Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]. - -{% if multiqc %}- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) +> Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]. +> {% endif %} > {% if multiqc %}- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. > {%- endif %} diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 7718d2e5f8..a618f87bf7 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -44,8 +44,8 @@ workflows use the "tube map" design for that. See https://nf-co.re/docs/contributing/design_guidelines#examples for examples. --> -1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) - {% if multiqc %}2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)){% endif %} +{% if fastqc %}1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)){% endif %} +{% if multiqc %}2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)){% endif %} ## Usage diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index 84972d8e24..078142fad0 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -18,9 +18,11 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] + {%- if fastqc %} withName: FASTQC { ext.args = '--quiet' } + {%- endif %} {%- if multiqc %} withName: 'MULTIQC' { diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index 76195a6825..5e42d50cc5 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -12,10 +12,13 @@ The directories listed below will be created in the results directory after the The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data using the following steps: -- [FastQC](#fastqc) - Raw read QC - {% if multiqc %}- [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline{% endif %} +{% if fastqc %}- [FastQC](#fastqc) - Raw read QC{% endif %} +{% if multiqc %}- [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline{% endif %} + - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution +{%- if fastqc %} + ### FastQC
@@ -28,7 +31,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d
[FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/). - +{%- endif %} {%- if multiqc %} ### MultiQC diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 8b156083f3..6ab68d0f2a 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -5,11 +5,12 @@ "https://github.com/nf-core/modules.git": { "modules": { "nf-core": { + {%- if fastqc %} "fastqc": { "branch": "master", "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", "installed_by": ["modules"] - }{%- if multiqc %}, + }{% endif %}{%- if multiqc %}{% if fastqc %},{% endif %} "multiqc": { "branch": "master", "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 3db8f66cb6..fe9e558cfe 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -230,7 +230,7 @@ def toolCitationText() { // Uncomment function in methodsDescriptionText to render in MultiQC report def citation_text = [ "Tools used in the workflow included:", - "FastQC (Andrews 2010),", + {% if fastqc %}"FastQC (Andrews 2010),",{% endif %} "MultiQC (Ewels et al. 2016)", "." ].join(' ').trim() @@ -243,7 +243,7 @@ def toolBibliographyText() { // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "
  • Author (2023) Pub name, Journal, DOI
    • " : "", // Uncomment function in methodsDescriptionText to render in MultiQC report def reference_text = [ - "
  • Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
    • ", + {% if fastqc %}"
  • Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
    • ",{% endif %} "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • " ].join(' ').trim() diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index d98c392f0c..e15d32f977 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { FASTQC } from '../modules/nf-core/fastqc/main' +{% if fastqc %}include { FASTQC } from '../modules/nf-core/fastqc/main'{% endif %} {% if multiqc %}include { MULTIQC } from '../modules/nf-core/multiqc/main'{% endif %} include { paramsSummaryMap } from 'plugin/nf-validation' {% if multiqc %}include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline'{% endif %} @@ -27,6 +27,7 @@ workflow {{ short_name|upper }} { ch_versions = Channel.empty() {% if multiqc %}ch_multiqc_files = Channel.empty(){% endif %} + {%- if fastqc %} // // MODULE: Run FastQC // @@ -35,6 +36,7 @@ workflow {{ short_name|upper }} { ) {% if multiqc %}ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}){% endif %} ch_versions = ch_versions.mix(FASTQC.out.versions.first()) + {%- endif %} // // Collate and save software versions diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 6a5fac358d..9fb56d610f 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -228,6 +228,16 @@ multiqc: multiqc_config: False nfcore_pipelines: True custom_pipelines: True +fastqc: + skippable_paths: + - "modules/nf-core/fastqc/" + short_description: "Use fastqc" + description: "The pipeline will include the FastQC module which performs quality control analysis of input FASTQ files." + help_text: | + FastQC is a tool which provides quality control checks on raw sequencing data. + The pipeline will include the FastQC module. + nfcore_pipelines: True + custom_pipelines: True changelog: skippable_paths: - "CHANGELOG.md" From e4f685d90e48c9ba0d01312eeae4fb2238afb000 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 21 Aug 2024 14:34:56 +0000 Subject: [PATCH 066/151] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1527f5ba49..6415612bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ - add option to exclude changelog from custom pipeline template ([#3104](https://github.com/nf-core/tools/pull/3104)) - add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125)) - add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126)) +- add option to exclude fastqc from pipeline template ([#3129](https://github.com/nf-core/tools/pull/3129)) ### Linting From 0e4a7d1393fbe944f39e6c38219c43d02831999a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Aug 2024 11:31:14 +0200 Subject: [PATCH 067/151] don't fail when no nf-core modules installed --- nf_core/components/lint/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index d0f6685365..0373170cfb 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -114,9 +114,7 @@ def __init__( ) ) if not self.all_remote_components: - raise LookupError( - f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline." - ) + log.debug(f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline.") local_component_dir = Path(self.directory, self.component_type, "local") if local_component_dir.exists(): @@ -146,7 +144,7 @@ def __init__( ] self.all_local_components = [] if not self.all_remote_components: - raise LookupError(f"No {self.component_type} in '{self.component_type}' directory") + log.debug(f"No {self.component_type} in '{self.component_type}' directory") # This could be better, perhaps glob for all nextflow.config files in? self.config = nf_core.utils.fetch_wf_config(self.directory / "tests" / "config", cache_config=True) From 92618ad8be8362ba5bece5eb82ab44fc08012c21 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Aug 2024 11:46:39 +0200 Subject: [PATCH 068/151] fix versions yaml file name --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index e15d32f977..8c797ede38 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -44,7 +44,7 @@ workflow {{ short_name|upper }} { softwareVersionsToYAML(ch_versions) .collectFile( storeDir: "${params.outdir}/pipeline_info", - name: 'nf_core_pipeline_software_mqc_versions.yml', + name: {% if is_nfcore %}'nf_core_' {% else %} '' {% endif %} + 'pipeline_software_' + {% if multiqc %} 'mqc_' {% else %} '' {% endif %} + 'versions.yml', sort: true, newLine: true ).set { ch_collated_versions } From d437582e632ad7bed1768a684eafdeba13cd16cf Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Aug 2024 12:46:28 +0200 Subject: [PATCH 069/151] udpate bpipe and fix pytests --- nf_core/components/lint/__init__.py | 4 ++-- nf_core/modules/lint/__init__.py | 2 +- tests/modules/test_lint.py | 11 +++++------ tests/subworkflows/test_lint.py | 10 ++++------ tests/test_modules.py | 4 ++-- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 0373170cfb..c1b1f24cb7 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -114,7 +114,7 @@ def __init__( ) ) if not self.all_remote_components: - log.debug(f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline.") + log.warning(f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline.") local_component_dir = Path(self.directory, self.component_type, "local") if local_component_dir.exists(): @@ -144,7 +144,7 @@ def __init__( ] self.all_local_components = [] if not self.all_remote_components: - log.debug(f"No {self.component_type} in '{self.component_type}' directory") + log.warning(f"No {self.component_type} in '{self.component_type}' directory") # This could be better, perhaps glob for all nextflow.config files in? self.config = nf_core.utils.fetch_wf_config(self.directory / "tests" / "config", cache_config=True) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index b75f7e7570..017b3965b4 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -110,7 +110,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for module or all - if module is None and not all_modules: + if module is None and not all_modules and len(self.all_remote_components) > 0: questions = [ { "type": "list", diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index dfe288a6ee..cc7e0565e0 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Union -import pytest import yaml from git.repo import Repo @@ -191,8 +190,8 @@ def test_modules_lint_empty(self): """Test linting a pipeline with no modules installed""" self.mods_remove.remove("fastqc", force=True) self.mods_remove.remove("multiqc", force=True) - with pytest.raises(LookupError): - nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + assert "No modules from https://github.com/nf-core/modules.git installed in pipeline" in self.caplog.text def test_modules_lint_new_modules(self): """lint a new module""" @@ -206,8 +205,8 @@ def test_modules_lint_no_gitlab(self): """Test linting a pipeline with no modules installed""" self.mods_remove.remove("fastqc", force=True) self.mods_remove.remove("multiqc", force=True) - with pytest.raises(LookupError): - nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir, remote_url=GITLAB_URL) + nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir, remote_url=GITLAB_URL) + assert f"No modules from {GITLAB_URL} installed in pipeline" in self.caplog.text def test_modules_lint_gitlab_modules(self): """Lint modules from a different remote""" @@ -566,7 +565,7 @@ def test_modules_meta_yml_input_mismatch(self): fh.write(main_nf) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 - assert len(module_lint.warned) == 2 + assert len(module_lint.warned) == 2, f"Linting warning with {[x.__dict__ for x in module_lint.warned]}" lint_tests = [x.lint_test for x in module_lint.warned] # check that it is there twice: assert lint_tests.count("meta_input_meta_only") == 1 diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index 2693008707..c8b97fd0b0 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -2,8 +2,6 @@ import shutil from pathlib import Path -import pytest - import nf_core.subworkflows from ..test_subworkflows import TestSubworkflows @@ -25,8 +23,8 @@ def test_subworkflows_lint_empty(self): self.subworkflow_remove.remove("utils_nextflow_pipeline", force=True) self.subworkflow_remove.remove("utils_nfcore_pipeline", force=True) self.subworkflow_remove.remove("utils_nfvalidation_plugin", force=True) - with pytest.raises(LookupError): - nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + assert "No subworkflows from https://github.com/nf-core/modules.git installed in pipeline" in self.caplog.text def test_subworkflows_lint_new_subworkflow(self): """lint a new subworkflow""" @@ -39,8 +37,8 @@ def test_subworkflows_lint_new_subworkflow(self): def test_subworkflows_lint_no_gitlab(self): """Test linting a pipeline with no subworkflows installed""" - with pytest.raises(LookupError): - nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir, remote_url=GITLAB_URL) + nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir, remote_url=GITLAB_URL) + assert f"No subworkflows from {GITLAB_URL} installed in pipeline" in self.caplog.text def test_subworkflows_lint_gitlab_subworkflows(self): """Lint subworkflows from a different remote""" diff --git a/tests/test_modules.py b/tests/test_modules.py index 9ce74fd4e1..0e16497176 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -44,8 +44,8 @@ def create_modules_repo_dummy(tmp_dir): yaml.dump(nf_core_yml.model_dump(), fh) # mock biocontainers and anaconda response with responses.RequestsMock() as rsps: - mock_anaconda_api_calls(rsps, "bpipe", "0.9.12--hdfd78af_0") - mock_biocontainers_api_calls(rsps, "bpipe", "0.9.12--hdfd78af_0") + mock_anaconda_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") + mock_biocontainers_api_calls(rsps, "bpipe", "0.9.13--hdfd78af_0") # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.create.ModuleCreate( root_dir, "bpipe/test", "@author", "process_single", False, False From 9551cdc3467dca4b4cd1bc3d3bf735340b9ee7bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 26 Aug 2024 14:21:37 +0200 Subject: [PATCH 070/151] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/conf/modules.config | 2 +- .../subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index 078142fad0..35e861d9b1 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -18,7 +18,7 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] - {%- if fastqc %} + {% if fastqc -%} withName: FASTQC { ext.args = '--quiet' } diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index fe9e558cfe..b130b7f88a 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -231,7 +231,7 @@ def toolCitationText() { def citation_text = [ "Tools used in the workflow included:", {% if fastqc %}"FastQC (Andrews 2010),",{% endif %} - "MultiQC (Ewels et al. 2016)", + {% if multiqc %}"MultiQC (Ewels et al. 2016)",{% endif %} "." ].join(' ').trim() @@ -244,7 +244,7 @@ def toolBibliographyText() { // Uncomment function in methodsDescriptionText to render in MultiQC report def reference_text = [ {% if fastqc %}"
  • Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
    • ",{% endif %} - "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • " + {% if multiqc %}"
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • "{% endif %} ].join(' ').trim() return reference_text From 9d29e9342b5258b8c08f495d49409761707493c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 26 Aug 2024 12:25:34 +0000 Subject: [PATCH 071/151] update textual snapshots --- .../__snapshots__/test_create_app.ambr | 764 +++++++++--------- .../pytest-0/test_github_details0 | 1 - .../pytest-0/test_github_exit_message0 | 1 - .../pytest-0/test_github_question0 | 1 - .../pytest-1/test_github_details0 | 1 - .../pytest-1/test_github_exit_message0 | 1 - .../pytest-1/test_github_question0 | 1 - 7 files changed, 382 insertions(+), 388 deletions(-) delete mode 160000 tmp/pytest-of-gitpod/pytest-0/test_github_details0 delete mode 160000 tmp/pytest-of-gitpod/pytest-0/test_github_exit_message0 delete mode 160000 tmp/pytest-of-gitpod/pytest-0/test_github_question0 delete mode 160000 tmp/pytest-of-gitpod/pytest-1/test_github_details0 delete mode 160000 tmp/pytest-of-gitpod/pytest-1/test_github_exit_message0 delete mode 160000 tmp/pytest-of-gitpod/pytest-1/test_github_question0 diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index ee470c915c..f5a19837b6 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-3293903238-matrix { + .terminal-1136262003-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3293903238-title { + .terminal-1136262003-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3293903238-r1 { fill: #c5c8c6 } - .terminal-3293903238-r2 { fill: #e3e3e3 } - .terminal-3293903238-r3 { fill: #989898 } - .terminal-3293903238-r4 { fill: #e1e1e1 } - .terminal-3293903238-r5 { fill: #4ebf71;font-weight: bold } - .terminal-3293903238-r6 { fill: #1e1e1e } - .terminal-3293903238-r7 { fill: #507bb3 } - .terminal-3293903238-r8 { fill: #e2e2e2 } - .terminal-3293903238-r9 { fill: #808080 } - .terminal-3293903238-r10 { fill: #dde6ed;font-weight: bold } - .terminal-3293903238-r11 { fill: #001541 } - .terminal-3293903238-r12 { fill: #0178d4 } - .terminal-3293903238-r13 { fill: #454a50 } - .terminal-3293903238-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-3293903238-r15 { fill: #000000 } - .terminal-3293903238-r16 { fill: #14191f } - .terminal-3293903238-r17 { fill: #e4e4e4 } - .terminal-3293903238-r18 { fill: #7ae998 } - .terminal-3293903238-r19 { fill: #0a180e;font-weight: bold } - .terminal-3293903238-r20 { fill: #008139 } - .terminal-3293903238-r21 { fill: #fea62b;font-weight: bold } - .terminal-3293903238-r22 { fill: #a7a9ab } - .terminal-3293903238-r23 { fill: #e2e3e3 } + .terminal-1136262003-r1 { fill: #c5c8c6 } + .terminal-1136262003-r2 { fill: #e3e3e3 } + .terminal-1136262003-r3 { fill: #989898 } + .terminal-1136262003-r4 { fill: #e1e1e1 } + .terminal-1136262003-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1136262003-r6 { fill: #1e1e1e } + .terminal-1136262003-r7 { fill: #507bb3 } + .terminal-1136262003-r8 { fill: #e2e2e2 } + .terminal-1136262003-r9 { fill: #808080 } + .terminal-1136262003-r10 { fill: #dde6ed;font-weight: bold } + .terminal-1136262003-r11 { fill: #001541 } + .terminal-1136262003-r12 { fill: #0178d4 } + .terminal-1136262003-r13 { fill: #454a50 } + .terminal-1136262003-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-1136262003-r15 { fill: #000000 } + .terminal-1136262003-r16 { fill: #14191f } + .terminal-1136262003-r17 { fill: #e4e4e4 } + .terminal-1136262003-r18 { fill: #7ae998 } + .terminal-1136262003-r19 { fill: #0a180e;font-weight: bold } + .terminal-1136262003-r20 { fill: #008139 } + .terminal-1136262003-r21 { fill: #fea62b;font-weight: bold } + .terminal-1136262003-r22 { fill: #a7a9ab } + .terminal-1136262003-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Hide help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - ▅▅ - - Nf-core pipelines are configured to use a copy of the most common reference  - genome files. - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Hide help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes▅▅ + + + Nf-core pipelines are configured to use a copy of the most common reference  + genome files. + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + ▅▅ + For more information about reference genomes in nf-core pipelines, see the  + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-2443134963-matrix { + .terminal-242174438-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2443134963-title { + .terminal-242174438-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2443134963-r1 { fill: #c5c8c6 } - .terminal-2443134963-r2 { fill: #e3e3e3 } - .terminal-2443134963-r3 { fill: #989898 } - .terminal-2443134963-r4 { fill: #e1e1e1 } - .terminal-2443134963-r5 { fill: #4ebf71;font-weight: bold } - .terminal-2443134963-r6 { fill: #1e1e1e } - .terminal-2443134963-r7 { fill: #507bb3 } - .terminal-2443134963-r8 { fill: #e2e2e2 } - .terminal-2443134963-r9 { fill: #808080 } - .terminal-2443134963-r10 { fill: #dde6ed;font-weight: bold } - .terminal-2443134963-r11 { fill: #001541 } - .terminal-2443134963-r12 { fill: #14191f } - .terminal-2443134963-r13 { fill: #454a50 } - .terminal-2443134963-r14 { fill: #7ae998 } - .terminal-2443134963-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-2443134963-r16 { fill: #0a180e;font-weight: bold } - .terminal-2443134963-r17 { fill: #000000 } - .terminal-2443134963-r18 { fill: #008139 } - .terminal-2443134963-r19 { fill: #fea62b;font-weight: bold } - .terminal-2443134963-r20 { fill: #a7a9ab } - .terminal-2443134963-r21 { fill: #e2e3e3 } + .terminal-242174438-r1 { fill: #c5c8c6 } + .terminal-242174438-r2 { fill: #e3e3e3 } + .terminal-242174438-r3 { fill: #989898 } + .terminal-242174438-r4 { fill: #e1e1e1 } + .terminal-242174438-r5 { fill: #4ebf71;font-weight: bold } + .terminal-242174438-r6 { fill: #1e1e1e } + .terminal-242174438-r7 { fill: #507bb3 } + .terminal-242174438-r8 { fill: #e2e2e2 } + .terminal-242174438-r9 { fill: #808080 } + .terminal-242174438-r10 { fill: #dde6ed;font-weight: bold } + .terminal-242174438-r11 { fill: #001541 } + .terminal-242174438-r12 { fill: #14191f } + .terminal-242174438-r13 { fill: #454a50 } + .terminal-242174438-r14 { fill: #7ae998 } + .terminal-242174438-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-242174438-r16 { fill: #0a180e;font-weight: bold } + .terminal-242174438-r17 { fill: #000000 } + .terminal-242174438-r18 { fill: #008139 } + .terminal-242174438-r19 { fill: #fea62b;font-weight: bold } + .terminal-242174438-r20 { fill: #a7a9ab } + .terminal-242174438-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help ▃▃ - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use code lintersThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - prettier. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Include citationsInclude pipeline tools Show help  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Show help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▆▆ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use code lintersThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + editor-config and  + prettier. + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Include citationsInclude pipeline tools Show help  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2511,254 +2511,254 @@ font-weight: 700; } - .terminal-1633351929-matrix { + .terminal-1052877418-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1633351929-title { + .terminal-1052877418-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1633351929-r1 { fill: #c5c8c6 } - .terminal-1633351929-r2 { fill: #e3e3e3 } - .terminal-1633351929-r3 { fill: #989898 } - .terminal-1633351929-r4 { fill: #e1e1e1 } - .terminal-1633351929-r5 { fill: #4ebf71;font-weight: bold } - .terminal-1633351929-r6 { fill: #1e1e1e } - .terminal-1633351929-r7 { fill: #507bb3 } - .terminal-1633351929-r8 { fill: #e2e2e2 } - .terminal-1633351929-r9 { fill: #808080 } - .terminal-1633351929-r10 { fill: #dde6ed;font-weight: bold } - .terminal-1633351929-r11 { fill: #001541 } - .terminal-1633351929-r12 { fill: #454a50 } - .terminal-1633351929-r13 { fill: #7ae998 } - .terminal-1633351929-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-1633351929-r15 { fill: #0a180e;font-weight: bold } - .terminal-1633351929-r16 { fill: #000000 } - .terminal-1633351929-r17 { fill: #008139 } - .terminal-1633351929-r18 { fill: #fea62b;font-weight: bold } - .terminal-1633351929-r19 { fill: #a7a9ab } - .terminal-1633351929-r20 { fill: #e2e3e3 } + .terminal-1052877418-r1 { fill: #c5c8c6 } + .terminal-1052877418-r2 { fill: #e3e3e3 } + .terminal-1052877418-r3 { fill: #989898 } + .terminal-1052877418-r4 { fill: #e1e1e1 } + .terminal-1052877418-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1052877418-r6 { fill: #1e1e1e } + .terminal-1052877418-r7 { fill: #507bb3 } + .terminal-1052877418-r8 { fill: #e2e2e2 } + .terminal-1052877418-r9 { fill: #808080 } + .terminal-1052877418-r10 { fill: #dde6ed;font-weight: bold } + .terminal-1052877418-r11 { fill: #001541 } + .terminal-1052877418-r12 { fill: #454a50 } + .terminal-1052877418-r13 { fill: #7ae998 } + .terminal-1052877418-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-1052877418-r15 { fill: #0a180e;font-weight: bold } + .terminal-1052877418-r16 { fill: #000000 } + .terminal-1052877418-r17 { fill: #008139 } + .terminal-1052877418-r18 { fill: #fea62b;font-weight: bold } + .terminal-1052877418-r19 { fill: #a7a9ab } + .terminal-1052877418-r20 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most common - reference genome files  - from iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use multiqcThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include the MultiQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - module which generates  - an HTML report for  - quality control. - - - - - - - - - - - - - - - - - - - - - - - - - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Show help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most common + reference genome files  + from iGenomes + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use multiqcThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include the MultiQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + module which generates  + an HTML report for  + quality control. + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use fastqcThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include the FastQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + module which performs  + quality control  + analysis of input FASTQ + files. + + + + + + + + + + + + + + + + + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  diff --git a/tmp/pytest-of-gitpod/pytest-0/test_github_details0 b/tmp/pytest-of-gitpod/pytest-0/test_github_details0 deleted file mode 160000 index ee82f320cb..0000000000 --- a/tmp/pytest-of-gitpod/pytest-0/test_github_details0 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ee82f320cb567b302e7328c1cacab94a98dae787 diff --git a/tmp/pytest-of-gitpod/pytest-0/test_github_exit_message0 b/tmp/pytest-of-gitpod/pytest-0/test_github_exit_message0 deleted file mode 160000 index f7fe48cf9d..0000000000 --- a/tmp/pytest-of-gitpod/pytest-0/test_github_exit_message0 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f7fe48cf9d00ab24581686a6d4226d2e9005c607 diff --git a/tmp/pytest-of-gitpod/pytest-0/test_github_question0 b/tmp/pytest-of-gitpod/pytest-0/test_github_question0 deleted file mode 160000 index dce3324acb..0000000000 --- a/tmp/pytest-of-gitpod/pytest-0/test_github_question0 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dce3324acbbe32f905afae8553e042f39404b37e diff --git a/tmp/pytest-of-gitpod/pytest-1/test_github_details0 b/tmp/pytest-of-gitpod/pytest-1/test_github_details0 deleted file mode 160000 index e2638dca91..0000000000 --- a/tmp/pytest-of-gitpod/pytest-1/test_github_details0 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e2638dca91b137a96b491008b1eeef2bfd791bec diff --git a/tmp/pytest-of-gitpod/pytest-1/test_github_exit_message0 b/tmp/pytest-of-gitpod/pytest-1/test_github_exit_message0 deleted file mode 160000 index 74cda800a8..0000000000 --- a/tmp/pytest-of-gitpod/pytest-1/test_github_exit_message0 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 74cda800a86fd07e93bbf8e00e4890516e3d838d diff --git a/tmp/pytest-of-gitpod/pytest-1/test_github_question0 b/tmp/pytest-of-gitpod/pytest-1/test_github_question0 deleted file mode 160000 index 444ebc5370..0000000000 --- a/tmp/pytest-of-gitpod/pytest-1/test_github_question0 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 444ebc5370bf91ea808f5d65c5788064f0480045 From 52da9e369598bc69ad356b25c38d6c09fed940cf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 21 Aug 2024 15:05:24 +0000 Subject: [PATCH 072/151] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6415612bbf..45d4fc18bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125)) - add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126)) - add option to exclude fastqc from pipeline template ([#3129](https://github.com/nf-core/tools/pull/3129)) +- add option to exclude documentation from pipeline template ([#3130](https://github.com/nf-core/tools/pull/3130)) ### Linting From a8cb9408d16ce47eda65baee2f73354b7566b72e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 26 Aug 2024 14:10:30 +0200 Subject: [PATCH 073/151] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipelines/create/templatefeatures.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 9fb56d610f..ab5f262374 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -310,3 +310,23 @@ slackreport: - ".prettierignore" nfcore_pipelines: False custom_pipelines: True +documentation: + skippable_paths: + - "docs" + short_description: "Add documentation" + description: "Add documentation to the pipeline" + help_text: | + This will add documentation markdown files where you can describe your pipeline. + It includes: + - docs/README.md: A README file where you can describe the structure of your documentation. + - docs/output.md: A file where you can explain the output generated by the pipeline + - docs/usage.md: A file where you can explain the usage of the pipeline and its parameters. + + These files come with an exemplary documentation structure written. + linting: + files_exist: + - "docs/output.md" + - "docs/README.md" + - "docs/usage.md" + nfcore_pipelines: False + custom_pipelines: True From bcbed0567441c2e5d62f2525da1a2ebebc7d5a23 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Aug 2024 15:55:44 +0200 Subject: [PATCH 074/151] update textual snapshots --- .../__snapshots__/test_create_app.ambr | 512 +++++++++--------- 1 file changed, 256 insertions(+), 256 deletions(-) diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index f5a19837b6..0015728fda 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-1136262003-matrix { + .terminal-3611359904-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1136262003-title { + .terminal-3611359904-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1136262003-r1 { fill: #c5c8c6 } - .terminal-1136262003-r2 { fill: #e3e3e3 } - .terminal-1136262003-r3 { fill: #989898 } - .terminal-1136262003-r4 { fill: #e1e1e1 } - .terminal-1136262003-r5 { fill: #4ebf71;font-weight: bold } - .terminal-1136262003-r6 { fill: #1e1e1e } - .terminal-1136262003-r7 { fill: #507bb3 } - .terminal-1136262003-r8 { fill: #e2e2e2 } - .terminal-1136262003-r9 { fill: #808080 } - .terminal-1136262003-r10 { fill: #dde6ed;font-weight: bold } - .terminal-1136262003-r11 { fill: #001541 } - .terminal-1136262003-r12 { fill: #0178d4 } - .terminal-1136262003-r13 { fill: #454a50 } - .terminal-1136262003-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-1136262003-r15 { fill: #000000 } - .terminal-1136262003-r16 { fill: #14191f } - .terminal-1136262003-r17 { fill: #e4e4e4 } - .terminal-1136262003-r18 { fill: #7ae998 } - .terminal-1136262003-r19 { fill: #0a180e;font-weight: bold } - .terminal-1136262003-r20 { fill: #008139 } - .terminal-1136262003-r21 { fill: #fea62b;font-weight: bold } - .terminal-1136262003-r22 { fill: #a7a9ab } - .terminal-1136262003-r23 { fill: #e2e3e3 } + .terminal-3611359904-r1 { fill: #c5c8c6 } + .terminal-3611359904-r2 { fill: #e3e3e3 } + .terminal-3611359904-r3 { fill: #989898 } + .terminal-3611359904-r4 { fill: #e1e1e1 } + .terminal-3611359904-r5 { fill: #4ebf71;font-weight: bold } + .terminal-3611359904-r6 { fill: #1e1e1e } + .terminal-3611359904-r7 { fill: #507bb3 } + .terminal-3611359904-r8 { fill: #e2e2e2 } + .terminal-3611359904-r9 { fill: #808080 } + .terminal-3611359904-r10 { fill: #dde6ed;font-weight: bold } + .terminal-3611359904-r11 { fill: #001541 } + .terminal-3611359904-r12 { fill: #0178d4 } + .terminal-3611359904-r13 { fill: #454a50 } + .terminal-3611359904-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-3611359904-r15 { fill: #000000 } + .terminal-3611359904-r16 { fill: #e4e4e4 } + .terminal-3611359904-r17 { fill: #14191f } + .terminal-3611359904-r18 { fill: #7ae998 } + .terminal-3611359904-r19 { fill: #0a180e;font-weight: bold } + .terminal-3611359904-r20 { fill: #008139 } + .terminal-3611359904-r21 { fill: #fea62b;font-weight: bold } + .terminal-3611359904-r22 { fill: #a7a9ab } + .terminal-3611359904-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Hide help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes▅▅ - - - Nf-core pipelines are configured to use a copy of the most common reference  - genome files. - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Hide help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + + + Nf-core pipelines are configured to use a copy of the most common reference  + genome files. + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + ▅▅ + For more information about reference genomes in nf-core pipelines, see the  + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-242174438-matrix { + .terminal-1661160397-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-242174438-title { + .terminal-1661160397-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-242174438-r1 { fill: #c5c8c6 } - .terminal-242174438-r2 { fill: #e3e3e3 } - .terminal-242174438-r3 { fill: #989898 } - .terminal-242174438-r4 { fill: #e1e1e1 } - .terminal-242174438-r5 { fill: #4ebf71;font-weight: bold } - .terminal-242174438-r6 { fill: #1e1e1e } - .terminal-242174438-r7 { fill: #507bb3 } - .terminal-242174438-r8 { fill: #e2e2e2 } - .terminal-242174438-r9 { fill: #808080 } - .terminal-242174438-r10 { fill: #dde6ed;font-weight: bold } - .terminal-242174438-r11 { fill: #001541 } - .terminal-242174438-r12 { fill: #14191f } - .terminal-242174438-r13 { fill: #454a50 } - .terminal-242174438-r14 { fill: #7ae998 } - .terminal-242174438-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-242174438-r16 { fill: #0a180e;font-weight: bold } - .terminal-242174438-r17 { fill: #000000 } - .terminal-242174438-r18 { fill: #008139 } - .terminal-242174438-r19 { fill: #fea62b;font-weight: bold } - .terminal-242174438-r20 { fill: #a7a9ab } - .terminal-242174438-r21 { fill: #e2e3e3 } + .terminal-1661160397-r1 { fill: #c5c8c6 } + .terminal-1661160397-r2 { fill: #e3e3e3 } + .terminal-1661160397-r3 { fill: #989898 } + .terminal-1661160397-r4 { fill: #e1e1e1 } + .terminal-1661160397-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1661160397-r6 { fill: #1e1e1e } + .terminal-1661160397-r7 { fill: #507bb3 } + .terminal-1661160397-r8 { fill: #e2e2e2 } + .terminal-1661160397-r9 { fill: #808080 } + .terminal-1661160397-r10 { fill: #dde6ed;font-weight: bold } + .terminal-1661160397-r11 { fill: #001541 } + .terminal-1661160397-r12 { fill: #14191f } + .terminal-1661160397-r13 { fill: #454a50 } + .terminal-1661160397-r14 { fill: #7ae998 } + .terminal-1661160397-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-1661160397-r16 { fill: #0a180e;font-weight: bold } + .terminal-1661160397-r17 { fill: #000000 } + .terminal-1661160397-r18 { fill: #008139 } + .terminal-1661160397-r19 { fill: #fea62b;font-weight: bold } + .terminal-1661160397-r20 { fill: #a7a9ab } + .terminal-1661160397-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - ⭘nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add Github CI testsThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use reference genomesThe pipeline will be  Show help  - ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▆▆ - ▊▎        Add Github badgesThe README.md file of  Show help  - ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Add configuration The pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Use code lintersThe pipeline will  Show help  - ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - prettier. - - ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ - ▊▎        Include citationsInclude pipeline tools Show help  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + ⭘nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github CI testsThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use reference genomesThe pipeline will be  Show help  + ▊▁▁▁▁▁▁▁▁▎configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from  + iGenomes + ▃▃ + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add Github badgesThe README.md file of  Show help  + ▊▁▁▁▁▁▁▁▁▎the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Add configuration The pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Use code lintersThe pipeline will  Show help  + ▊▁▁▁▁▁▁▁▁▎include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + editor-config and  + prettier. + + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + ▊▎        Include citationsInclude pipeline tools Show help  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  From d3d6345b5bc1519dd9426b66319043d05d0c2a85 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 22 Aug 2024 10:54:18 +0200 Subject: [PATCH 075/151] add option to exclude test configs from pipeline template --- .../create-test-lint-wf-template.yml | 1 + .../pipeline-template/.github/CONTRIBUTING.md | 7 ++++- .../.github/PULL_REQUEST_TEMPLATE.md | 2 ++ .../.github/workflows/download_pipeline.yml | 4 +-- nf_core/pipeline-template/README.md | 2 +- nf_core/pipeline-template/docs/usage.md | 3 ++ nf_core/pipeline-template/nextflow.config | 4 ++- .../pipeline-template/nextflow_schema.json | 4 +-- nf_core/pipelines/create/templatefeatures.yml | 28 +++++++++++++++++++ 9 files changed, 48 insertions(+), 7 deletions(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 70125a10f3..46f55a4b64 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -70,6 +70,7 @@ jobs: - TEMPLATE: is_nfcore - TEMPLATE: nf_core_configs profile: "self_hosted_runner" + - TEMPLATE: test_config fail-fast: false steps: diff --git a/nf_core/pipeline-template/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md index 5a58501bb2..f331d38673 100644 --- a/nf_core/pipeline-template/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -30,16 +30,18 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests +{%- if test_config %} You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command: ```bash nf-test test --profile debug,test,docker --verbose ``` +{% endif %} When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. -There are typically two types of tests that run: +{% if test_config %}There are typically two types of tests that run:{% endif %} ### Lint tests @@ -48,12 +50,15 @@ To enforce these and ensure that all pipelines stay in sync, we have developed a If any failures or warnings are encountered, please follow the listed URL for more documentation. +{%- if test_config %} + ### Pipeline tests Each `nf-core` pipeline should be set up with a minimal set of test-data. `GitHub Actions` then runs the pipeline on this data to ensure that it exits successfully. If there are any failures then the automated tests fail. These tests are run both with the latest available version of `Nextflow` and also the minimum required version that is stated in the pipeline code. +{%- endif %} ## Patch diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index dee23ccab1..c96f2dd4c2 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -20,8 +20,10 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. {%- endif %} - [ ] Make sure your code lints (`nf-core pipelines lint`). + {%- if test_config %} - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). + {%- endif %} - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 99a42d86d2..e7a28e5ac4 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -68,7 +68,7 @@ jobs: --download-configuration - name: Inspect download - run: tree ./${{ env.REPOTITLE_LOWERCASE }} + run: tree ./${{ env.REPOTITLE_LOWERCASE }}{% endraw %}{% if test_config %}{% raw %} - name: Run the downloaded pipeline (stub) id: stub_run_pipeline @@ -83,4 +83,4 @@ jobs: env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results{% endraw %} + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results{% endraw %}{% endif %} diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index a618f87bf7..c8ed828d38 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -50,7 +50,7 @@ ## Usage > [!NOTE] -> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data. +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. {% if test_config %}Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.{% endif %}