diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 7e90febb74..f07b31e9de 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -38,7 +38,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: go to subdirectory and change nextflow workdir diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 70125a10f3..1fb521b4bb 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -44,7 +44,7 @@ jobs: - name: Create Matrix id: create_matrix run: | - echo "matrix=$(yq 'keys | tojson(0)' nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT + echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/template_features.yml)" >> $GITHUB_OUTPUT RunTestWorkflow: runs-on: ${{ matrix.runner }} @@ -66,8 +66,6 @@ jobs: runner: ubuntu-latest profile: "docker" exclude: - - TEMPLATE: github - - TEMPLATE: is_nfcore - TEMPLATE: nf_core_configs profile: "self_hosted_runner" fail-fast: false diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index a95a477459..56c6c822a9 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -39,7 +39,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: go to working directory diff --git a/.gitpod.yml b/.gitpod.yml index b2fbb73133..efe193f35f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -6,17 +6,14 @@ tasks: python -m pip install -r requirements-dev.txt pre-commit install --install-hooks nextflow self-update - - name: unset JAVA_TOOL_OPTIONS - command: | - unset JAVA_TOOL_OPTIONS vscode: - extensions: # based on nf-core.nf-core-extensionpack + extensions: - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code - charliermarsh.ruff # Code linter Ruff diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 815e980270..4f08d8419d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.0 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix diff --git a/CHANGELOG.md b/CHANGELOG.md index fff2a0bbb3..a98377e62c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # nf-core/tools: Changelog -## v2.14.2dev +## v3.0.0dev + +**Highlights** + +- Pipeline commands are renamed from `nf-core ` to `nf-core pipelines ` to follow the same command structure as modules and subworkflows commands. +- More customisation for pipeline templates. The template has been divided into features which can be skipped, e.g. you can create a new pipeline without any traces of FastQC in it. +- A new Text User Interface app when running `nf-core pipelines create` to help us guide you through the process better (no worries, you can still use the cli if you give all values as parameters) +- We replaced nf-validation with nf-schema in the pipeline template +- CI tests now lint with the nf-core tools version matching the template version of the pipeline, to minimise errors in opened PRs with every new tools release. +- New command `nf-core pipelines ro-crate` to create a [Research Object (RO) crate](https://www.researchobject.org/ro-crate/) for a pipeline +- `nf-core licences` command is deprecated. +- The structure of nf-core/tools pytests has been updated +- The structure of the API docs has been updated ### Template @@ -9,66 +21,79 @@ - Remove deprecated syntax ([#3046](https://github.com/nf-core/tools/pull/3046)) - Use filename in code block for `params.yml` ([#3055](https://github.com/nf-core/tools/pull/3055)) - Remove release announcement for non nf-core pipelines ([#3072](https://github.com/nf-core/tools/pull/3072)) +- handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108), [#3112](https://github.com/nf-core/tools/pull/3112)) - add option to exclude code linters for custom pipeline template ([#3084](https://github.com/nf-core/tools/pull/3084)) - add option to exclude citations for custom pipeline template ([#3101](https://github.com/nf-core/tools/pull/3101)) - add option to exclude gitpod for custom pipeline template ([#3100](https://github.com/nf-core/tools/pull/3100)) - add option to exclude codespaces from pipeline template ([#3105](https://github.com/nf-core/tools/pull/3105)) - add option to exclude multiqc from pipeline template ([#3103](https://github.com/nf-core/tools/pull/3103)) - add option to exclude changelog from custom pipeline template ([#3104](https://github.com/nf-core/tools/pull/3104)) -- handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108)) -- add templatefeatures.yml to python package ([#3112](https://github.com/nf-core/tools/pull/3112)) +- add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125)) +- add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126)) +- Use nf-schema instead of nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116)) +- add option to exclude nf-schema from the template ([#3116](https://github.com/nf-core/tools/pull/3116)) +- add option to exclude fastqc from pipeline template ([#3129](https://github.com/nf-core/tools/pull/3129)) +- add option to exclude documentation from pipeline template ([#3130](https://github.com/nf-core/tools/pull/3130)) +- add option to exclude test configs from pipeline template ([#3133](https://github.com/nf-core/tools/pull/3133)) +- add option to exclude tower.yml from pipeline template ([#3134](https://github.com/nf-core/tools/pull/3134)) +- run nf-core lint `--release` on PRs to master ([#3148](https://github.com/nf-core/tools/pull/3148)) +- Add tests to ensure all files are part of a template customisation group and all groups are tested ([#3099](https://github.com/nf-core/tools/pull/3099)) ### Linting - Fix linting fail on nfcore_external_java_deps if nf_schema is used ([#2976](https://github.com/nf-core/tools/pull/2976)) - Conda module linting: Include package name in log file ([#3014](https://github.com/nf-core/tools/pull/3014)) +- Remove defaults from conda `environment.yml` file. ([#3029](https://github.com/nf-core/tools/pull/3029)) - Restructure pipeline tests and move pipeline linting into subfolder ([#3070](https://github.com/nf-core/tools/pull/3070)) - Fix module linting warning for process_high_memory ([#3086](https://github.com/nf-core/tools/issues/3086)) +- Linting will now fail when an unpinned plugin is used ([#3116](https://github.com/nf-core/tools/pull/3116)) +- Linting will now check if the schema is correct for the used validation plugin ([#3116])(https://github.com/nf-core/tools/pull/3116) +- Linting will now check the use of the right validation plugin include statements in the workflow scripts ([#3116])(https://github.com/nf-core/tools/pull/3116) +- Full linting for correct use of nf-schema and nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116)) +- Handle cases where the directory path contains the name of the component ([#3147](https://github.com/nf-core/tools/pull/3147)) +- Don't test conda `environment.yml` `name` attribute (which should no longer be there) ([#3161](https://github.com/nf-core/tools/pull/3161)) -### Download +### Pipeline create command + +- Create: allow more special characters on the pipeline name for non-nf-core pipelines ([#3008](https://github.com/nf-core/tools/pull/3008)) +- Create: Mock git cretentials to generate stable textual snapshots ([#3007](https://github.com/nf-core/tools/pull/3007)) +- Create app: display input textbox with equally spaced grid ([#3038](https://github.com/nf-core/tools/pull/3038)) +- Pipelines: allow numbers in custom pipeline name ([#3094](https://github.com/nf-core/tools/pull/3094)) ### Components - The `modules_nfcore` tag in the `main.nf.test` file of modules/subworkflows now displays the organization name in custom modules repositories ([#3005](https://github.com/nf-core/tools/pull/3005)) +- Add `--migrate_pytest` option to `nf-core test` command ([#3085](https://github.com/nf-core/tools/pull/3085)) +- Components: allow spaces at the beginning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115)) ### General -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.4 ([#2974](https://github.com/nf-core/tools/pull/2974)) -- Update gitpod/workspace-base Docker digest to 92dd1bc ([#2982](https://github.com/nf-core/tools/pull/2982)) - Update output of generation script for API docs to new structure ([#2988](https://github.com/nf-core/tools/pull/2988)) - Add no clobber and put bash options on their own line ([#2991](https://github.com/nf-core/tools/pull/2991)) - update minimal textual version and snapshots ([#2998](https://github.com/nf-core/tools/pull/2998)) - move pipeline subcommands for v3.0 ([#2983](https://github.com/nf-core/tools/pull/2983)) - return directory if base_dir is the root directory ([#3003](https://github.com/nf-core/tools/pull/3003)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.6 ([#3006](https://github.com/nf-core/tools/pull/3006)) -- Create: allow more special characters on the pipeline name for non-nf-core pipelines ([#3008](https://github.com/nf-core/tools/pull/3008)) - Remove nf-core licences command ([#3012](https://github.com/nf-core/tools/pull/3012)) - README - absolute image paths ([#3013](https://github.com/nf-core/tools/pull/3013)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.7 ([#3015](https://github.com/nf-core/tools/pull/3015)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.8 ([#3017](https://github.com/nf-core/tools/pull/3017)) -- Update python:3.12-slim Docker digest to e3ae8cf ([#3020](https://github.com/nf-core/tools/pull/3020)) -- Update python:3.12-slim Docker digest to 2fba8e7 ([#3023](https://github.com/nf-core/tools/pull/3023)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.10 ([#3031](https://github.com/nf-core/tools/pull/3031)) - Add warning deprecation message to top-level commands ([#3036](https://github.com/nf-core/tools/pull/3036)) -- Create: Mock git cretentials to generate stable textual snapshots ([#3007](https://github.com/nf-core/tools/pull/3007)) - move pipeline commands to functions to avoid duplication ([#3039](https://github.com/nf-core/tools/pull/3039)) -- Create app: display input textbox with equally spaced grid ([#3038](https://github.com/nf-core/tools/pull/3038)) -- Update python:3.12-slim Docker digest to da2d7af ([#3041](https://github.com/nf-core/tools/pull/3041)) -- Update gitpod/workspace-base Docker digest to 0f38224 ([#3048](https://github.com/nf-core/tools/pull/3048)) - update output_dir for api docs to new website structure ([#3051](https://github.com/nf-core/tools/pull/3051)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.5.1 ([#3052](https://github.com/nf-core/tools/pull/3052)) - Add `--limit-output` argument for modules/subworkflow update ([#3047](https://github.com/nf-core/tools/pull/3047)) - update api docs to new structure ([#3054](https://github.com/nf-core/tools/pull/3054)) -- Update to pytest v8 and move it to dev dependencies ([#3058](https://github.com/nf-core/tools/pull/3058)) - handle new jsonschema error type ([#3061](https://github.com/nf-core/tools/pull/3061)) -- Update python:3.12-slim Docker digest to f11725a ([#3071](https://github.com/nf-core/tools/pull/3071)) - Fix number of arguments for pipelines_create within the command_create function ([#3074](https://github.com/nf-core/tools/pull/3074)) -- Update python:3.12-slim Docker digest to 740d94a ([#3079](https://github.com/nf-core/tools/pull/3079)) -- Add `--migrate_pytest` option to `nf-core test` command ([#3085](https://github.com/nf-core/tools/pull/3085)) -- Update pre-commit hook pre-commit/mirrors-mypy to v1.11.1 ([#3091](https://github.com/nf-core/tools/pull/3091)) -- Pipelines: allow numbers in custom pipeline name ([#3094](https://github.com/nf-core/tools/pull/3094)) - Add bot action to update textual snapshots and write bot documentation ([#3102](https://github.com/nf-core/tools/pull/3102)) -- Components: allow spaces at the betinning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115)) +- Update gitpod setup ([#3136](https://github.com/nf-core/tools/pull/3136)) +- fix syncing a pipeline from current directory ([#3143](https://github.com/nf-core/tools/pull/3143)) +- Patch gitpod conda setup to not use defaults channel ([#3159](https://github.com/nf-core/tools/pull/3159)) + +## Version updates + +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 ([#3122](https://github.com/nf-core/tools/pull/3122)) +- Update gitpod/workspace-base Docker digest to 92dd1bc ([#2982](https://github.com/nf-core/tools/pull/2982)) +- Update python:3.12-slim Docker digest to 59c7332 ([#3124](https://github.com/nf-core/tools/pull/3124)) +- Update pre-commit hook pre-commit/mirrors-mypy to v1.11.1 ([#3091](https://github.com/nf-core/tools/pull/3091)) +- Update to pytest v8 and move it to dev dependencies ([#3058](https://github.com/nf-core/tools/pull/3058)) - Update dependency prompt_toolkit to <=3.0.47 ([#3121](https://github.com/nf-core/tools/pull/3121)) ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09] diff --git a/Dockerfile b/Dockerfile index c88abcb1ce..fb1a867937 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:740d94a19218c8dd584b92f804b1158f85b0d241e5215ea26ed2dcade2b9d138 +FROM python:3.12-slim@sha256:59c7332a4a24373861c4a5f0eec2c92b87e3efeb8ddef011744ef9a751b1d11c LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" diff --git a/MANIFEST.in b/MANIFEST.in index 2bec403804..ce2e08c090 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,4 +10,4 @@ include nf_core/assets/logo/nf-core-repo-logo-base-darkbg.png include nf_core/assets/logo/placeholder_logo.svg include nf_core/assets/logo/MavenPro-Bold.ttf include nf_core/pipelines/create/create.tcss -include nf_core/pipelines/create/templatefeatures.yml +include nf_core/pipelines/create/template_features.yml diff --git a/docs/api/_src/pipeline_lint_tests/plugin_includes.md b/docs/api/_src/pipeline_lint_tests/plugin_includes.md new file mode 100644 index 0000000000..48bddadc80 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/plugin_includes.md @@ -0,0 +1,5 @@ +# plugin_includes + +```{eval-rst} +.. automethod:: nf_core.pipelines.lint.PipelineLint.plugin_includes +``` diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 153b8b7e75..0efea13ec9 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1136,6 +1136,13 @@ def command_modules_create( @modules.command("test") @click.pass_context @click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.option( + "-v", + "--verbose", + is_flag=True, + default=False, + help="Print verbose output to the console. Sets `--debug` inside the nf-test command.", +) @click.option( "-d", "--dir", @@ -1171,10 +1178,12 @@ def command_modules_create( default=False, help="Migrate a module with pytest tests to nf-test", ) -def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest): +def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest, verbose): """ Run nf-test for a module. """ + if verbose: + ctx.obj["verbose"] = verbose modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest) diff --git a/nf_core/commands_pipelines.py b/nf_core/commands_pipelines.py index 4b6fa75f3e..23affb1d27 100644 --- a/nf_core/commands_pipelines.py +++ b/nf_core/commands_pipelines.py @@ -144,9 +144,12 @@ def pipelines_lint( ctx.obj["hide_progress"], ) swf_failed = 0 + module_failed = 0 if subworkflow_lint_obj is not None: swf_failed = len(subworkflow_lint_obj.failed) - if len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed > 0: + if module_lint_obj is not None: + module_failed = len(module_lint_obj.failed) + if len(lint_obj.failed) + module_failed + swf_failed > 0: sys.exit(1) except AssertionError as e: log.critical(e) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index d0f6685365..c1b1f24cb7 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -114,9 +114,7 @@ def __init__( ) ) if not self.all_remote_components: - raise LookupError( - f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline." - ) + log.warning(f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline.") local_component_dir = Path(self.directory, self.component_type, "local") if local_component_dir.exists(): @@ -146,7 +144,7 @@ def __init__( ] self.all_local_components = [] if not self.all_remote_components: - raise LookupError(f"No {self.component_type} in '{self.component_type}' directory") + log.warning(f"No {self.component_type} in '{self.component_type}' directory") # This could be better, perhaps glob for all nextflow.config files in? self.config = nf_core.utils.fetch_wf_config(self.directory / "tests" / "config", cache_config=True) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index db3196be91..0f3cdcdfbd 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -64,7 +64,15 @@ def __init__( self.process_name = "" self.environment_yml: Optional[Path] = Path(self.component_dir, "environment.yml") - repo_dir = self.component_dir.parts[: self.component_dir.parts.index(self.component_name.split("/")[0])][-1] + component_list = self.component_name.split("/") + + name_index = len(self.component_dir.parts) - 1 - self.component_dir.parts[::-1].index(component_list[0]) + if len(component_list) != 1 and component_list[0] == component_list[1]: + # Handle cases where the subtool has the same name as the tool + name_index -= 1 + + repo_dir = self.component_dir.parts[:name_index][-1] + self.org = repo_dir self.nftest_testdir = Path(self.component_dir, "tests") self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test") diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index fd0e8bb799..c462c6a47e 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,6 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . +# See https://docs.renovatebot.com/docker/#digest-pinning for why a digest is used. FROM gitpod/workspace-base@sha256:f189a4195c3861365356f9c1b438ab26fd88e1ff46ce2843afc62861fc982e0c USER root @@ -40,12 +41,11 @@ RUN chown -R gitpod:gitpod /opt/conda /usr/src/nf_core # Change user to gitpod USER gitpod -# Install nextflow, nf-core, Mamba, and pytest-workflow -RUN conda config --add channels defaults && \ - conda config --add channels bioconda && \ +# Install nextflow, nf-core, nf-test, and other useful tools +RUN conda config --add channels bioconda && \ conda config --add channels conda-forge && \ conda config --set channel_priority strict && \ - conda install --quiet --yes --name base \ + conda install --quiet --yes --update-all --name base \ nextflow \ nf-test \ prettier \ @@ -63,3 +63,4 @@ RUN nextflow self-update && \ # Setup pdiff for nf-test diffs ENV NFT_DIFF="pdiff" ENV NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" +ENV JAVA_TOOL_OPTIONS= diff --git a/nf_core/module-template/environment.yml b/nf_core/module-template/environment.yml index dcf510affb..a8a40a8e03 100644 --- a/nf_core/module-template/environment.yml +++ b/nf_core/module-template/environment.yml @@ -1,9 +1,7 @@ --- # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json -name: "{{ component_name_underscore }}" channels: - conda-forge - bioconda - - defaults dependencies: - "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index b75f7e7570..017b3965b4 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -110,7 +110,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for module or all - if module is None and not all_modules: + if module is None and not all_modules and len(self.all_remote_components) > 0: questions = [ { "type": "list", diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 341b9cd730..4488b0befa 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -90,42 +90,3 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) env_yml["dependencies"].sort() with open(Path(module.component_dir, "environment.yml"), "w") as fh: yaml.dump(env_yml, fh, Dumper=custom_yaml_dumper()) - - # Check that the name in the environment.yml file matches the name in the meta.yml file - with open(Path(module.component_dir, "meta.yml")) as fh: - meta_yml = yaml.safe_load(fh) - - if env_yml["name"] == meta_yml["name"]: - module.passed.append( - ( - "environment_yml_name", - "The module's `environment.yml` name matches module name", - module.environment_yml, - ) - ) - else: - module.failed.append( - ( - "environment_yml_name", - f"Conflicting process name between environment.yml (`{env_yml['name']}`) and meta.yml (`{module.component_name}`)", - module.environment_yml, - ) - ) - - # Check that the name is lowercase - if env_yml["name"] == env_yml["name"].lower(): - module.passed.append( - ( - "environment_yml_name_lowercase", - "The module's `environment.yml` name is lowercase", - module.environment_yml, - ) - ) - else: - module.failed.append( - ( - "environment_yml_name_lowercase", - "The module's `environment.yml` name is not lowercase", - module.environment_yml, - ) - ) diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index 72dda289a0..5145366e50 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -11,6 +11,7 @@ indent_style = space [*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 +{%- if modules %} # These files are edited and tested upstream in nf-core/modules [/modules/nf-core/**] charset = unset @@ -24,9 +25,12 @@ end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset +{%- endif %} +{%- if email %} [/assets/email*] indent_size = unset +{%- endif %} # ignore python and markdown [*.{py,md}] diff --git a/nf_core/pipeline-template/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md index 5a58501bb2..f331d38673 100644 --- a/nf_core/pipeline-template/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -30,16 +30,18 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests +{%- if test_config %} You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command: ```bash nf-test test --profile debug,test,docker --verbose ``` +{% endif %} When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. -There are typically two types of tests that run: +{% if test_config %}There are typically two types of tests that run:{% endif %} ### Lint tests @@ -48,12 +50,15 @@ To enforce these and ensure that all pipelines stay in sync, we have developed a If any failures or warnings are encountered, please follow the listed URL for more documentation. +{%- if test_config %} + ### Pipeline tests Each `nf-core` pipeline should be set up with a minimal set of test-data. `GitHub Actions` then runs the pipeline on this data to ensure that it exits successfully. If there are any failures then the automated tests fail. These tests are run both with the latest available version of `Nextflow` and also the minimum required version that is stated in the pipeline code. +{%- endif %} ## Patch diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index dee23ccab1..c96f2dd4c2 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -20,8 +20,10 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. {%- endif %} - [ ] Make sure your code lints (`nf-core pipelines lint`). + {%- if test_config %} - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). + {%- endif %} - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 6b2547765d..63fa99cec8 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: Check out pipeline code diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 99a42d86d2..e7a28e5ac4 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -68,7 +68,7 @@ jobs: --download-configuration - name: Inspect download - run: tree ./${{ env.REPOTITLE_LOWERCASE }} + run: tree ./${{ env.REPOTITLE_LOWERCASE }}{% endraw %}{% if test_config %}{% raw %} - name: Run the downloaded pipeline (stub) id: stub_run_pipeline @@ -83,4 +83,4 @@ jobs: env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results{% endraw %} + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results{% endraw %}{% endif %} diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index ecd5e32f6c..dbba830ec1 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -53,12 +53,21 @@ jobs: pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} - name: Run nf-core pipelines lint + if: ${{ github.base_ref != 'master' }} env: GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} run: nf-core -l lint_log.txt pipelines lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md + - name: Run nf-core pipelines lint --release + if: ${{ github.base_ref == 'master' }} + env: + GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} + run: nf-core -l lint_log.txt pipelines lint --release --dir ${GITHUB_WORKSPACE} --markdown lint_results.md + - name: Save PR number if: ${{ always() }} run: echo ${{ github.event.pull_request.number }} > PR_number.txt diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index 30e85ed977..5907fb59c9 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -4,9 +4,6 @@ tasks: command: | pre-commit install --install-hooks nextflow self-update - - name: unset JAVA_TOOL_OPTIONS - command: | - unset JAVA_TOOL_OPTIONS vscode: extensions: # based on nf-core.nf-core-extensionpack @@ -15,7 +12,7 @@ vscode: - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files{% endif %} - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code - charliermarsh.ruff # Code linter Ruff diff --git a/nf_core/pipeline-template/.prettierignore b/nf_core/pipeline-template/.prettierignore index 437d763d0c..c8e8ad9e11 100644 --- a/nf_core/pipeline-template/.prettierignore +++ b/nf_core/pipeline-template/.prettierignore @@ -1,6 +1,12 @@ +{%- if email %} email_template.html +{%- endif %} +{%- if adaptivecard %} adaptivecard.json +{%- endif %} +{%- if slackreport %} slackreport.json +{%- endif %} .nextflow* work/ data/ diff --git a/nf_core/pipeline-template/CITATIONS.md b/nf_core/pipeline-template/CITATIONS.md index 7cf37e5018..2373f1de7f 100644 --- a/nf_core/pipeline-template/CITATIONS.md +++ b/nf_core/pipeline-template/CITATIONS.md @@ -12,11 +12,10 @@ ## Pipeline tools -- [FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/) +{% if fastqc %}- [FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/) - > Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]. - -{% if multiqc %}- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) +> Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]. +> {% endif %} > {% if multiqc %}- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/) > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. > {%- endif %} diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 7718d2e5f8..beb45ed511 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -7,6 +7,10 @@ +{%- else %} + +# {{ name }} + {% endif -%} {% if github_badges -%} [![GitHub Actions CI Status](https://github.com/{{ name }}/actions/workflows/ci.yml/badge.svg)](https://github.com/{{ name }}/actions/workflows/ci.yml) @@ -16,7 +20,7 @@ [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) @@ -44,13 +48,13 @@ workflows use the "tube map" design for that. See https://nf-co.re/docs/contributing/design_guidelines#examples for examples. --> -1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) - {% if multiqc %}2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)){% endif %} +{% if fastqc %}1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)){% endif %} +{% if multiqc %}2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)){% endif %} ## Usage > [!NOTE] -> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data. +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. {% if test_config %}Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.{% endif %}