Add new command nf-core rocrate to create a Research Object (RO) crate for a pipeline

.github/actions/create-lint-wf/action.yml

@@ -59,6 +59,12 @@ runs:
       run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
       working-directory: create-lint-wf
 
+    # Add empty ro-crate file
+    - name: add empty ro-crate file
+      shell: bash
+      run: touch nf-core-testpipeline/ro-crate-metadata.json
+      working-directory: create-lint-wf
+
     # Run nf-core pipelines linting
     - name: nf-core pipelines lint
       shell: bash

.github/workflows/create-test-lint-wf-template.yml

@@ -144,6 +144,11 @@ jobs:
         run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
         working-directory: create-test-lint-wf
 
+      # Add empty ro-crate file
+      - name: add empty ro-crate file
+        run: touch my-prefix-testpipeline/ro-crate-metadata.json
+        working-directory: create-test-lint-wf
+
       # Run nf-core linting
       - name: nf-core pipelines lint
         run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned

CHANGELOG.md

@@ -33,6 +33,7 @@
 
 ### General
 
+- Add new command `nf-core rocrate` to create a Research Object (RO) crate for a pipeline ([#2680](https://github.com/nf-core/tools/pull/2680))
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.4 ([#2974](https://github.com/nf-core/tools/pull/2974))
 - Update gitpod/workspace-base Docker digest to 92dd1bc ([#2982](https://github.com/nf-core/tools/pull/2982))
 - Update output of generation script for API docs to new structure ([#2988](https://github.com/nf-core/tools/pull/2988))

nf_core/__main__.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import sys
+from pathlib import Path
 
 import rich
 import rich.console
@@ -35,6 +36,7 @@
     pipelines_launch,
     pipelines_lint,
     pipelines_list,
+    pipelines_ro_crate,
     pipelines_schema_build,
     pipelines_schema_docs,
     pipelines_schema_lint,
@@ -85,7 +87,7 @@
         },
         {
             "name": "For developers",
-            "commands": ["create", "lint", "bump-version", "sync", "schema", "create-logo"],
+            "commands": ["create", "lint", "bump-version", "sync", "schema", "ro-crate", "create-logo"],
         },
     ],
     "nf-core modules": [
@@ -579,6 +581,44 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived):
     pipelines_list(ctx, keywords, sort, json, show_archived)
 
 
+# nf-core pipelines ro-crate
+@pipelines.command("ro-crate")
+@click.argument(
+    "pipeline_dir",
+    type=click.Path(exists=True),
+    default=Path.cwd(),
+    required=True,
+    metavar="<pipeline directory>",
+)
+@click.option(
+    "-j",
+    "--json_path",
+    default=Path.cwd(),
+    type=str,
+    help="Path to save RO Crate metadata json file to",
+)
+@click.option("-z", "--zip_path", type=str, help="Path to save RO Crate zip file to")
+@click.option(
+    "-pv",
+    "--pipeline_version",
+    type=str,
+    help="Version of pipeline to use for RO Crate",
+    default="",
+)
+@click.pass_context
+def ro_crate(
+    ctx,
+    pipeline_dir: str,
+    json_path: str,
+    zip_path: str,
+    pipeline_version: str,
+):
+    """
+    Make an Research Object Crate
+    """
+    pipelines_ro_crate(ctx, pipeline_dir, json_path, zip_path, pipeline_version)
+
+
 # nf-core pipelines sync
 @pipelines.command("sync")
 @click.pass_context
@@ -1687,7 +1727,7 @@ def command_schema_validate(pipeline, params):
 @click.option(
     "--url",
     type=str,
-    default="https://nf-co.re/pipeline_schema_builder",
+    default="https://oldsite.nf-co.re/pipeline_schema_builder",
     help="Customise the builder URL (for development work)",
 )
 def command_schema_build(directory, no_prompts, web_only, url):

nf_core/commands_pipelines.py

@@ -2,6 +2,7 @@
 import os
 import sys
 from pathlib import Path
+from typing import Optional, Union
 
 import rich
 
@@ -278,6 +279,33 @@ def pipelines_list(ctx, keywords, sort, json, show_archived):
     stdout.print(list_workflows(keywords, sort, json, show_archived))
 
 
+# nf-core pipelines ro-crate
+def pipelines_ro_crate(
+    ctx,
+    pipeline_dir: Union[str, Path],
+    json_path: Optional[Union[str, Path]],
+    zip_path: Optional[Union[str, Path]],
+    pipeline_version: str,
+) -> None:
+    from nf_core.pipelines.rocrate import ROCrate
+
+    if json_path is None and zip_path is None:
+        log.error("Either `--json_path` or `--zip_path` must be specified.")
+        sys.exit(1)
+    else:
+        pipeline_dir = Path(pipeline_dir)
+        if json_path is not None:
+            json_path = Path(json_path)
+        if zip_path is not None:
+            zip_path = Path(zip_path)
+        try:
+            rocrate_obj = ROCrate(pipeline_dir, pipeline_version)
+            rocrate_obj.create_ro_crate(pipeline_dir, metadata_path=json_path, zip_path=zip_path)
+        except (UserWarning, LookupError, FileNotFoundError) as e:
+            log.error(e)
+            sys.exit(1)
+
+
 # nf-core pipelines sync
 def pipelines_sync(ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr):
     """

nf_core/components/info.py

@@ -211,9 +211,9 @@ def get_local_yaml(self) -> Optional[Dict]:
                         return yaml.safe_load(fh)
             log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally")
 
-        return None
+        return {}
 
-    def get_remote_yaml(self) -> Optional[dict]:
+    def get_remote_yaml(self) -> Optional[Dict]:
         """Attempt to get the meta.yml file from a remote repo.
 
         Returns:

nf_core/modules/modules_json.py

@@ -432,7 +432,7 @@ def move_component_to_local(self, component_type: str, component: str, repo_name
             to_name += f"-{datetime.datetime.now().strftime('%y%m%d%H%M%S')}"
         shutil.move(str(current_path), local_dir / to_name)
 
-    def unsynced_components(self) -> Tuple[List[str], List[str], dict]:
+    def unsynced_components(self) -> Tuple[List[str], List[str], Dict]:
         """
         Compute the difference between the modules/subworkflows in the directory and the
         modules/subworkflows in the 'modules.json' file. This is done by looking at all
@@ -1119,8 +1119,10 @@ def dump(self, run_prettier: bool = False) -> None:
         """
         Sort the modules.json, and write it to file
         """
+        # Sort the modules.json
+        if self.modules_json is None:
+            self.load()
         if self.modules_json is not None:
-            # Sort the modules.json
             self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"])
             if run_prettier:
                 dump_json_with_prettier(self.modules_json_path, self.modules_json)

nf_core/pipelines/lint/files_exist.py

@@ -66,6 +66,7 @@ def files_exist(self) -> Dict[str, List[str]]:
         conf/igenomes.config
         .github/workflows/awstest.yml
         .github/workflows/awsfulltest.yml
+        ro-crate-metadata.json
 
     Files that *must not* be present, due to being renamed or removed in the template:
 
@@ -175,6 +176,7 @@ def files_exist(self) -> Dict[str, List[str]]:
         [Path(".github", "workflows", "awstest.yml")],
         [Path(".github", "workflows", "awsfulltest.yml")],
         [Path("modules.json")],
+        [Path("ro-crate-metadata.json")],
     ]
 
     # List of strings. Fails / warns if any of the strings exist.
@@ -204,6 +206,12 @@ def files_exist(self) -> Dict[str, List[str]]:
         (Path("lib", "nfcore_external_java_deps.jar"), [{"plugins": "nf-validation"}, {"plugins": "nf-schema"}]),
     ]
 
+    files_hint = [
+        [
+            ["ro-crate-metadata.json"],
+            ". Run `nf-core rocrate` to generate this file. Read more about RO-Crates in the [nf-core/tools docs](https://nf-co.re/tools#create-a-ro-crate-metadata-file).",
+        ],
+    ]
     # Remove files that should be ignored according to the linting config
     ignore_files = self.lint_config.get("files_exist", []) if self.lint_config is not None else []
 
@@ -231,7 +239,11 @@ def pf(file_path: Union[str, Path]) -> Path:
         if any([pf(f).is_file() for f in files]):
             passed.append(f"File found: {self._wrap_quotes(files)}")
         else:
-            warned.append(f"File not found: {self._wrap_quotes(files)}")
+            hint = ""
+            for file_hint in files_hint:
+                if file_hint[0] == files:
+                    hint = str(file_hint[1])
+            warned.append(f"File not found: {self._wrap_quotes(files)}{hint}")
 
     # Files that cause an error if they exist
     for file in files_fail_ifexists:

nf_core/pipelines/lint_utils.py

@@ -5,6 +5,7 @@
 from typing import List
 
 import rich
+import rich.box
 from rich.console import Console
 from rich.table import Table