From dd00cde99f6ea218237e6f12971271d5899b1561 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 19 Mar 2021 14:07:29 +0100 Subject: [PATCH 01/41] Bump to 1.14dev --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e27d8ca0a..7c56c0cb0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # nf-core/tools: Changelog +## v1.14dev + +_..nothing yet.._ + ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19] * Fixed bug in pipeline linting markdown output that gets posted to PR comments [[#914]](https://github.com/nf-core/tools/issues/914) diff --git a/setup.py b/setup.py index 280d21559a..f2fa6aa93d 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = "1.13.1" +version = "1.14dev" with open("README.md") as f: readme = f.read() From 00307a3c96a3e4da1a332e75313879478b9425a7 Mon Sep 17 00:00:00 2001 From: matthiasho Date: Fri, 19 Mar 2021 19:23:23 +0100 Subject: [PATCH 02/41] Add nf-core colors to rtd sphinx theme add sphinx to requirements file --- docs/api/_src/_static/css/custom.css | 200 +++++++++++++++++++++++++++ docs/api/_src/conf.py | 7 +- requirements-dev.txt | 2 + 3 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 docs/api/_src/_static/css/custom.css diff --git a/docs/api/_src/_static/css/custom.css b/docs/api/_src/_static/css/custom.css new file mode 100644 index 0000000000..2b7166ae22 --- /dev/null +++ b/docs/api/_src/_static/css/custom.css @@ -0,0 +1,200 @@ +@media (prefers-color-scheme: light) { + .wy-nav-side{ + background-color: #ededed; + } + .wy-nav-top, .wy-side-nav-search, .wy-menu-vertical a:active { + background-color: #32AD65; + } + .wy-menu-vertical a{ + color: #343434; + } + .wy-menu-vertical a:hover{ + background-color: #abacab85; + } + .wy-menu-vertical header, .wy-menu-vertical p.caption { + color:#32AD65; + } + + .wy-side-nav-search input[type=text]{ + border: none; + } + + code, .rst-content code.literal{ + background-color: rgba(220,220,220,0.4); + color: #db9444; + border: none; + } + .rst-content .note .admonition-title{ + background-color: #72757bfc + } + + html.writer-html4 .rst-content dl:not(.docutils)>dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple)>dt { + background-color: #32AD65; + border:none; + } + .rst-content .method>dt>code { + color: #f5f6f7; + } + .sig-paren{ + color: #db9444; + } + .rst-content div[class^=highlight], .rst-content pre.literal-block{ + border:none; + } + .highlight{ + background-color: #242424; + color: #e5e6e7; + } + .highlight .hll{ + color: #e5e6e7; + background-color: #585b60fc; + } + .highlight .k, .highlight .nt, .highlight .no{ + color: #6fb2e8; + } + .highlight .s, .highlight .s1, .highlight .s2{ + color: #32AD65; + } + .highlight .nb, .highlight .o{ + color: #db9444; + } + .highlight .c1{ + color: #88898afc; + } + .highlight .nv{ + color: #f9d977; + } + .btn.btn-neutral { + background-color: #e5e6e7d4 !important; + } + .rst-content .hint .admonition-title, .rst-content .hint .wy-alert-title, .rst-content .important .admonition-title, .rst-content .important .wy-alert-title, .rst-content .tip .admonition-title, .rst-content .tip .wy-alert-title, .rst-content .wy-alert-success.admonition-todo .admonition-title, .rst-content .wy-alert-success.admonition-todo .wy-alert-title, .rst-content .wy-alert-success.admonition .admonition-title, .rst-content .wy-alert-success.admonition .wy-alert-title, .rst-content .wy-alert-success.attention .admonition-title, .rst-content .wy-alert-success.attention .wy-alert-title, .rst-content .wy-alert-success.caution .admonition-title, .rst-content .wy-alert-success.caution .wy-alert-title, .rst-content .wy-alert-success.danger .admonition-title, .rst-content .wy-alert-success.danger .wy-alert-title, .rst-content .wy-alert-success.error .admonition-title, .rst-content .wy-alert-success.error .wy-alert-title, .rst-content .wy-alert-success.note .admonition-title, .rst-content .wy-alert-success.note .wy-alert-title, .rst-content .wy-alert-success.seealso .admonition-title, .rst-content .wy-alert-success.seealso .wy-alert-title, .rst-content .wy-alert-success.warning .admonition-title, .rst-content .wy-alert-success.warning .wy-alert-title, .rst-content .wy-alert.wy-alert-success .admonition-title, .wy-alert.wy-alert-success .rst-content .admonition-title, .wy-alert.wy-alert-success .wy-alert-title{ + background-color: #32AD65; + color: #e5e6e7; + } + + .rst-content .note, .rst-content .seealso, .rst-content .wy-alert-info.admonition, .rst-content .wy-alert-info.admonition-todo, .rst-content .wy-alert-info.attention, .rst-content .wy-alert-info.caution, .rst-content .wy-alert-info.danger, .rst-content .wy-alert-info.error, .rst-content .wy-alert-info.hint, .rst-content .wy-alert-info.important, .rst-content .wy-alert-info.tip, .rst-content .wy-alert-info.warning, .wy-alert.wy-alert-info, + .rst-content .hint, .rst-content .important, .rst-content .tip, .rst-content .wy-alert-success.admonition, .rst-content .wy-alert-success.admonition-todo, .rst-content .wy-alert-success.attention, .rst-content .wy-alert-success.caution, .rst-content .wy-alert-success.danger, .rst-content .wy-alert-success.error, .rst-content .wy-alert-success.note, .rst-content .wy-alert-success.seealso, .rst-content .wy-alert-success.warning, .wy-alert.wy-alert-success{ + color: #343434; + background-color: #e3e3e3; + border: none; + } + + +.rst-content .admonition-todo .admonition-title, .rst-content .admonition-todo .wy-alert-title, .rst-content .attention .admonition-title, .rst-content .attention .wy-alert-title, .rst-content .caution .admonition-title, .rst-content .caution .wy-alert-title, .rst-content .warning .admonition-title, .rst-content .warning .wy-alert-title, .rst-content .wy-alert-warning.admonition .admonition-title, .rst-content .wy-alert-warning.admonition .wy-alert-title, .rst-content .wy-alert-warning.danger .admonition-title, .rst-content .wy-alert-warning.danger .wy-alert-title, .rst-content .wy-alert-warning.error .admonition-title, .rst-content .wy-alert-warning.error .wy-alert-title, .rst-content .wy-alert-warning.hint .admonition-title, .rst-content .wy-alert-warning.hint .wy-alert-title, .rst-content .wy-alert-warning.important .admonition-title, .rst-content .wy-alert-warning.important .wy-alert-title, .rst-content .wy-alert-warning.note .admonition-title, .rst-content .wy-alert-warning.note .wy-alert-title, .rst-content .wy-alert-warning.seealso .admonition-title, .rst-content .wy-alert-warning.seealso .wy-alert-title, .rst-content .wy-alert-warning.tip .admonition-title, .rst-content .wy-alert-warning.tip .wy-alert-title, .rst-content .wy-alert.wy-alert-warning .admonition-title, .wy-alert.wy-alert-warning .rst-content .admonition-title, .wy-alert.wy-alert-warning .wy-alert-title { + background-color: #f4a25b; + } +.rst-content .admonition-todo, .rst-content .attention, .rst-content .caution, .rst-content .warning, .rst-content .wy-alert-warning.admonition, .rst-content .wy-alert-warning.danger, .rst-content .wy-alert-warning.error, .rst-content .wy-alert-warning.hint, .rst-content .wy-alert-warning.important, .rst-content .wy-alert-warning.note, .rst-content .wy-alert-warning.seealso, .rst-content .wy-alert-warning.tip, .wy-alert.wy-alert-warning { + color: #343434; + background-color: #e3e3e3; + } + + +} +@media (prefers-color-scheme: dark) { + .wy-nav-content-wrap { + background-color: #181a1b + } + .wy-nav-top, .wy-side-nav-search, .wy-menu-vertical a:active { + background-color: #32AD65; + } + .wy-menu-vertical header, .wy-menu-vertical p.caption { + color:#32AD65; + } + .wy-nav-side{ + background-color: #2E2E2E; + } + .wy-nav-content{ + background-color: #343434; + color: #e5e6e7; + } + a, a:hover, a:focus, a:active { + color: #6fb2e8; + } + code, .rst-content code.literal{ + background-color: rgba(220,220,220,0.1); + color: #db9444; + border: none; + } + .wy-side-nav-search input[type=text]{ + border: none; + background-color: #e5e6e7; + } + .wy-side-nav-search>div.version{ + color: #e5e6e7c9; + } + .wy-side-nav-search .wy-dropdown>a, .wy-side-nav-search>a { + color: #e5e6e7; + } + + html.writer-html4 .rst-content dl:not(.docutils)>dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple)>dt { + background-color: #32AD65; + border:none; + } + html.writer-html4 .rst-content dl:not(.docutils) dl:not(.field-list)>dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple) dl:not(.field-list)>dt{ + background-color:#777; + color:#e5e6e7a6; + } + .rst-content code { + color: #e5e6e7d4; + } + .rst-content .method>dt>code { + color: #e5e6e7; + } + .sig-paren{ + color: #db9444; + } + .rst-content .note .admonition-title{ + background-color: #72757bfc + } + .rst-content div[class^=highlight], .rst-content pre.literal-block{ + border:none; + } + + .highlight{ + background-color: #242424; + color: #e5e6e7; + } + .highlight .hll{ + color: #e5e6e7; + background-color: #585b60fc; + } + .highlight .k, .highlight .nt, .highlight .no{ + color: #6fb2e8; + } + .highlight .s, .highlight .s1, .highlight .s2{ + color: #32AD65; + } + .highlight .nb, .highlight .o{ + color: #db9444; + } + .highlight .c1{ + color: #88898afc; + } + .highlight .nv{ + color: #f9d977; + } + .btn.btn-neutral { + background-color: #e5e6e7d4 !important; + } + + .rst-content .hint .admonition-title, .rst-content .hint .wy-alert-title, .rst-content .important .admonition-title, .rst-content .important .wy-alert-title, .rst-content .tip .admonition-title, .rst-content .tip .wy-alert-title, .rst-content .wy-alert-success.admonition-todo .admonition-title, .rst-content .wy-alert-success.admonition-todo .wy-alert-title, .rst-content .wy-alert-success.admonition .admonition-title, .rst-content .wy-alert-success.admonition .wy-alert-title, .rst-content .wy-alert-success.attention .admonition-title, .rst-content .wy-alert-success.attention .wy-alert-title, .rst-content .wy-alert-success.caution .admonition-title, .rst-content .wy-alert-success.caution .wy-alert-title, .rst-content .wy-alert-success.danger .admonition-title, .rst-content .wy-alert-success.danger .wy-alert-title, .rst-content .wy-alert-success.error .admonition-title, .rst-content .wy-alert-success.error .wy-alert-title, .rst-content .wy-alert-success.note .admonition-title, .rst-content .wy-alert-success.note .wy-alert-title, .rst-content .wy-alert-success.seealso .admonition-title, .rst-content .wy-alert-success.seealso .wy-alert-title, .rst-content .wy-alert-success.warning .admonition-title, .rst-content .wy-alert-success.warning .wy-alert-title, .rst-content .wy-alert.wy-alert-success .admonition-title, .wy-alert.wy-alert-success .rst-content .admonition-title, .wy-alert.wy-alert-success .wy-alert-title{ + background-color: #32AD65; + color: #e5e6e7; + } + + .rst-content .note, .rst-content .seealso, .rst-content .wy-alert-info.admonition, .rst-content .wy-alert-info.admonition-todo, .rst-content .wy-alert-info.attention, .rst-content .wy-alert-info.caution, .rst-content .wy-alert-info.danger, .rst-content .wy-alert-info.error, .rst-content .wy-alert-info.hint, .rst-content .wy-alert-info.important, .rst-content .wy-alert-info.tip, .rst-content .wy-alert-info.warning, .wy-alert.wy-alert-info, + .rst-content .hint, .rst-content .important, .rst-content .tip, .rst-content .wy-alert-success.admonition, .rst-content .wy-alert-success.admonition-todo, .rst-content .wy-alert-success.attention, .rst-content .wy-alert-success.caution, .rst-content .wy-alert-success.danger, .rst-content .wy-alert-success.error, .rst-content .wy-alert-success.note, .rst-content .wy-alert-success.seealso, .rst-content .wy-alert-success.warning, .wy-alert.wy-alert-success{ + color: #343434; + background-color: #e3e3e3; + border: none; + } + + .rst-content .admonition-todo .admonition-title, .rst-content .admonition-todo .wy-alert-title, .rst-content .attention .admonition-title, .rst-content .attention .wy-alert-title, .rst-content .caution .admonition-title, .rst-content .caution .wy-alert-title, .rst-content .warning .admonition-title, .rst-content .warning .wy-alert-title, .rst-content .wy-alert-warning.admonition .admonition-title, .rst-content .wy-alert-warning.admonition .wy-alert-title, .rst-content .wy-alert-warning.danger .admonition-title, .rst-content .wy-alert-warning.danger .wy-alert-title, .rst-content .wy-alert-warning.error .admonition-title, .rst-content .wy-alert-warning.error .wy-alert-title, .rst-content .wy-alert-warning.hint .admonition-title, .rst-content .wy-alert-warning.hint .wy-alert-title, .rst-content .wy-alert-warning.important .admonition-title, .rst-content .wy-alert-warning.important .wy-alert-title, .rst-content .wy-alert-warning.note .admonition-title, .rst-content .wy-alert-warning.note .wy-alert-title, .rst-content .wy-alert-warning.seealso .admonition-title, .rst-content .wy-alert-warning.seealso .wy-alert-title, .rst-content .wy-alert-warning.tip .admonition-title, .rst-content .wy-alert-warning.tip .wy-alert-title, .rst-content .wy-alert.wy-alert-warning .admonition-title, .wy-alert.wy-alert-warning .rst-content .admonition-title, .wy-alert.wy-alert-warning .wy-alert-title { + background-color: #f4a25b; + } + .rst-content .admonition-todo, .rst-content .attention, .rst-content .caution, .rst-content .warning, .rst-content .wy-alert-warning.admonition, .rst-content .wy-alert-warning.danger, .rst-content .wy-alert-warning.error, .rst-content .wy-alert-warning.hint, .rst-content .wy-alert-warning.important, .rst-content .wy-alert-warning.note, .rst-content .wy-alert-warning.seealso, .rst-content .wy-alert-warning.tip, .wy-alert.wy-alert-warning { + color: #343434; + background-color: #e3e3e3; + } + +} \ No newline at end of file diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index d863a80d28..3ade174945 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -14,10 +14,9 @@ # import os import sys +sys.path.insert(0, os.path.abspath('../../../nf_core')) import nf_core -sys.path.insert(0, os.path.abspath("../../../nf_core")) - # -- Project information ----------------------------------------------------- project = "nf-core tools API" @@ -90,6 +89,10 @@ # Custom sidebar templates, must be a dictionary that maps document names # to template names. +html_css_files = [ + 'css/custom.css', +] + # # The default sidebars (for documents that don't match any pattern) are # defined by theme itself. Builtin themes are using these templates by diff --git a/requirements-dev.txt b/requirements-dev.txt index 10e124429e..1ad518c05c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,5 @@ pytest-datafiles pytest-cov mock black +Sphinx +sphinx_rtd_theme \ No newline at end of file From e7dddec6e4823bbc986cd4cbafd47b6a3feda4a5 Mon Sep 17 00:00:00 2001 From: matthiasho Date: Fri, 19 Mar 2021 19:27:17 +0100 Subject: [PATCH 03/41] fix for linting --- docs/api/_src/conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index 3ade174945..68023db2f3 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -14,7 +14,8 @@ # import os import sys -sys.path.insert(0, os.path.abspath('../../../nf_core')) + +sys.path.insert(0, os.path.abspath("../../../nf_core")) import nf_core # -- Project information ----------------------------------------------------- @@ -90,7 +91,7 @@ # Custom sidebar templates, must be a dictionary that maps document names # to template names. html_css_files = [ - 'css/custom.css', + "css/custom.css", ] # From d058b1a34e41b20c531113e24b8dad5143f0718a Mon Sep 17 00:00:00 2001 From: matthiasho Date: Fri, 19 Mar 2021 19:32:33 +0100 Subject: [PATCH 04/41] =?UTF-8?q?dummy=20commit=20to=20run=20the=20tests?= =?UTF-8?q?=20one=20more=20time=20=F0=9F=99=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api/_src/_static/css/custom.css | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/api/_src/_static/css/custom.css b/docs/api/_src/_static/css/custom.css index 2b7166ae22..9bd4341651 100644 --- a/docs/api/_src/_static/css/custom.css +++ b/docs/api/_src/_static/css/custom.css @@ -196,5 +196,4 @@ color: #343434; background-color: #e3e3e3; } - -} \ No newline at end of file +} From a59f497a20487ecc7bc15a0cc4c09ffef855f365 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 09:09:19 +0100 Subject: [PATCH 05/41] fix bug for updated messages --- nf_core/modules/lint.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/lint.py b/nf_core/modules/lint.py index 909dd91170..285fb2798e 100644 --- a/nf_core/modules/lint.py +++ b/nf_core/modules/lint.py @@ -375,7 +375,6 @@ def check_module_changes(self, nfcore_modules): Downloads the 'main.nf', 'functions.nf' and 'meta.yml' files for every module and compare them to the local copies """ - all_modules_up_to_date = True files_to_check = ["main.nf", "functions.nf", "meta.yml"] progress_bar = rich.progress.Progress( @@ -420,13 +419,21 @@ def check_module_changes(self, nfcore_modules): remote_copy = r.content.decode("utf-8") if local_copy != remote_copy: - all_modules_up_to_date = False self.warned.append( LintResult( mod, "check_local_copy", "Local copy of module outdated", - f"{os.path.join(mod.module_dir, f)}", + f"{os.path.join(mod.module_dir, f)}" + ) + ) + else: + self.passed.append( + LintResult( + mod, + "check_local_copy", + "Local copy of module up to date", + f"{os.path.join(mod.module_dir, f)}" ) ) except UnicodeDecodeError as e: @@ -435,12 +442,10 @@ def check_module_changes(self, nfcore_modules): mod, "check_local_copy", f"Could not decode file from {url}. Skipping comparison ({e})", - f"{os.path.join(mod.module_dir, f)}", + f"{os.path.join(mod.module_dir, f)}" ) ) - if all_modules_up_to_date: - self.passed.append("All modules are up to date!") class NFCoreModule(object): From d1a9fb7e7a59df5dcd5b739f0a4fdafbbfa331e5 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 09:12:11 +0100 Subject: [PATCH 06/41] update pytests --- tests/test_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 5296270092..b4809dd270 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -83,7 +83,7 @@ def test_modules_lint_fastqc(self): self.mods.install("fastqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) module_lint.lint(print_results=False, all_modules=True) - assert len(module_lint.passed) == 16 + assert len(module_lint.passed) == 18 assert len(module_lint.warned) == 0 assert len(module_lint.failed) == 0 From 153e5fa943a9b6f0bb61411b24fc770a79818668 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 09:15:06 +0100 Subject: [PATCH 07/41] black --- nf_core/modules/lint.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint.py b/nf_core/modules/lint.py index 285fb2798e..c5dc501a5b 100644 --- a/nf_core/modules/lint.py +++ b/nf_core/modules/lint.py @@ -424,7 +424,7 @@ def check_module_changes(self, nfcore_modules): mod, "check_local_copy", "Local copy of module outdated", - f"{os.path.join(mod.module_dir, f)}" + f"{os.path.join(mod.module_dir, f)}", ) ) else: @@ -433,7 +433,7 @@ def check_module_changes(self, nfcore_modules): mod, "check_local_copy", "Local copy of module up to date", - f"{os.path.join(mod.module_dir, f)}" + f"{os.path.join(mod.module_dir, f)}", ) ) except UnicodeDecodeError as e: @@ -442,12 +442,11 @@ def check_module_changes(self, nfcore_modules): mod, "check_local_copy", f"Could not decode file from {url}. Skipping comparison ({e})", - f"{os.path.join(mod.module_dir, f)}" + f"{os.path.join(mod.module_dir, f)}", ) ) - class NFCoreModule(object): """ A class to hold the information a bout a nf-core module From 6536ed7fce9aa97af3bc260ecee51c4b42d69a91 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 16:57:40 +0100 Subject: [PATCH 08/41] started with more robust jinja2 templating --- nf_core/create.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 6f23c99478..e29af0db39 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -83,7 +83,7 @@ def render_template(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template") - binary_ftypes = ["image", "application/java-archive"] + binary_ftypes = ["image", "application/java-archive", "application/x-java-archive"] object_attrs = vars(self) object_attrs["nf_core_version"] = nf_core.__version__ @@ -107,16 +107,20 @@ def render_template(self): os.makedirs(os.path.dirname(output_path), exist_ok=True) # Just copy binary files - (ftype, encoding) = mimetypes.guess_type(template_fn_path) + (ftype, encoding) = mimetypes.guess_type(template_fn_path, strict=False) if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): log.debug(f"Copying binary file: '{output_path}'") shutil.copy(template_fn_path, output_path) continue # Render the template - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - rendered_output = j_template.render(object_attrs) + try: + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + rendered_output = j_template.render(object_attrs) + except UnicodeDecodeError as e: + log.error(f"Could not decode file {template_fn_path} {e}") + sys.exit(1) # Write to the pipeline output file with open(output_path, "w") as fh: From 0771195aa26532a9da60937cbbfcdd8d8ca7d42f Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 17:35:01 +0100 Subject: [PATCH 09/41] add pytest-workflow dep --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f2fa6aa93d..05dfc2de9c 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ "packaging", "prompt_toolkit>=3.0.3", "pyyaml", + "pytest-workflow", "questionary>=1.8.0", "requests_cache", "requests", From 676cf8fd20ad001718dec25f4b67e22f1f076f7d Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 18:20:57 +0100 Subject: [PATCH 10/41] added check for binary extensions --- nf_core/create.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/create.py b/nf_core/create.py index e29af0db39..40d3b366aa 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -3,6 +3,7 @@ organization's specification based on a template. """ from genericpath import exists +from pathlib import Path import git import jinja2 import logging @@ -84,6 +85,7 @@ def render_template(self): ) template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template") binary_ftypes = ["image", "application/java-archive", "application/x-java-archive"] + binary_extensions = [".jpeg", ".jpg", ".png", ".zip", ".gz", ".jar", ".tar"] object_attrs = vars(self) object_attrs["nf_core_version"] = nf_core.__version__ @@ -113,6 +115,11 @@ def render_template(self): shutil.copy(template_fn_path, output_path) continue + if Path(template_fn_path).suffix in binary_extensions: + log.debug(f"Copying binary file: '{output_path}'") + shutil.copy(template_fn_path, output_path) + continue + # Render the template try: log.debug(f"Rendering template file: '{template_fn}'") From 54b0eb82ab18a656841ebff44af3b3fa97a06e27 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 18:55:41 +0100 Subject: [PATCH 11/41] remove containsKey() --- nf_core/pipeline-template/lib/NfcoreSchema.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 78e8c65d7f..54935ec818 100644 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -219,7 +219,7 @@ class NfcoreSchema { } } } - if(rawSchema.keySet().contains('properties') && rawSchema.get('properties').containsKey(ignore_param)) { + if(rawSchema.keySet().contains('properties') && rawSchema.get('properties').keySet().contains(ignore_param)) { rawSchema.get("properties").remove(ignore_param) } if(rawSchema.keySet().contains('required') && rawSchema.required.contains(ignore_param)) { From f15824efa4d30df3d44db83b370299b60892fdbc Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 19:02:59 +0100 Subject: [PATCH 12/41] add 'params.name' to failing params in nextflow.config lint check --- nf_core/lint/nextflow_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index b458c257c0..19f10f045c 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -116,6 +116,7 @@ def nextflow_config(self): "params.container", "params.singleEnd", "params.igenomesIgnore", + "params.name", ] # Remove field that should be ignored according to the linting config From 45f0009168f57d002f0726d2602617e104f36f01 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 20:38:02 +0100 Subject: [PATCH 13/41] Split create-lint-wf commands into separate steps --- .github/workflows/create-lint-wf.yml | 39 ++++++++++++++++++---------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index eec5e9840c..569cac6599 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -4,6 +4,8 @@ on: [push, pull_request] jobs: MakeTestWorkflow: runs-on: ubuntu-latest + env: + NXF_VER: 21.03.0-edge steps: - uses: actions/checkout@v2 name: Check out source-code repository @@ -22,21 +24,32 @@ jobs: env: CAPSULE_LOG: none run: | - mkdir /tmp/nextflow - cd /tmp/nextflow wget -qO- get.nextflow.io | bash - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + sudo mv nextflow /usr/local/bin/ - - name: Run nf-core/tools - run: | - nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core --log-file log.txt lint nf-core-testpipeline --fail-ignored - nf-core --log-file log.txt list - nf-core --log-file log.txt licences nf-core-testpipeline - nf-core --log-file log.txt sync nf-core-testpipeline/ - nf-core --log-file log.txt schema build nf-core-testpipeline/ --no-prompts - nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1 - nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc + - name: nf-core create + run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" + + - name: nf-core lint + run: nf-core --log-file log.txt lint nf-core-testpipeline --fail-ignored + + - name: nf-core list + run: nf-core --log-file log.txt list + + - name: nf-core licences + run: nf-core --log-file log.txt licences nf-core-testpipeline + + - name: nf-core sync + run: nf-core --log-file log.txt sync nf-core-testpipeline/ + + - name: nf-core schema + run: nf-core --log-file log.txt schema build nf-core-testpipeline/ --no-prompts + + - name: nf-core bump-version + run: nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1 + + - name: nf-core modules install + run: nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc - name: Upload log file artifact if: ${{ always() }} From 879c12de34ba9539799f7479a8d7a8751bb66802 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 20:38:19 +0100 Subject: [PATCH 14/41] Automated PR comments for Python and Markdownlint CI test failures --- .github/workflows/markdown-lint.yml | 25 +++++++++++++++++++++++++ .github/workflows/python-lint.yml | 22 ++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 5cdf07ec9a..5a590aff8e 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -16,3 +16,28 @@ jobs: - name: Run Markdownlint run: markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Markdown linting is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install `markdownlint-cli` + * On Mac: `brew install markdownlint-cli` + * Everything else: [Install `npm`](https://www.npmjs.com/get-npm) then [install `markdownlint-cli`](https://www.npmjs.com/package/markdownlint-cli): `npm install -g markdownlint-cli` + * Try to automatically fix errors in your pipeline: `markdownlint . --config .github/markdownlint.yml --fix` + * Check that there are no remaining problems: `markdownlint . --config .github/markdownlint.yml` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 4c8b5b00f9..cf41eb67da 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -15,3 +15,25 @@ jobs: - name: Check code lints with Black uses: jpetrucciani/black-check@master + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Python linting (`black`) is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` + * Fix formatting errors in your pipeline: `black .` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false From c9a3b1154947475fc5bd8331ec66bfa7bb32545a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 20:40:15 +0100 Subject: [PATCH 15/41] Changelog, bump v1.13.2dev --- CHANGELOG.md | 7 +++++-- setup.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c56c0cb0e..3eb87806e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,11 @@ # nf-core/tools: Changelog -## v1.14dev +## v1.13.2dev -_..nothing yet.._ +### Tools code + +* Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read +* Added automated PR comments to the Markdownlint and Python Black lint CI tests to explain failures ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19] diff --git a/setup.py b/setup.py index 05dfc2de9c..d5dc60e0d9 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = "1.14dev" +version = "1.13.2dev" with open("README.md") as f: readme = f.read() From a316a3a2312da6d2288e2b04a8f189378897b5f7 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 20:45:14 +0100 Subject: [PATCH 16/41] better logging for directory --- nf_core/modules/create.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index ea35b54c29..89340ecf94 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -251,7 +251,10 @@ def get_repo_type(self, directory): elif os.path.exists(os.path.join(directory, "software")): return "modules" else: - raise UserWarning(f"Could not determine repository type: '{directory}'") + raise UserWarning( + f"This directory does not look like a clone of nf-core/modules or an nf-core pipeline: '{directory}'" + " Please point to a valid directory." + ) def get_module_dirs(self): """Given a directory and a tool/subtool, set the file paths and check if they already exist From b3c1d93d126eacda102efb5518c6f7a0b7b5796f Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 22 Mar 2021 20:51:29 +0100 Subject: [PATCH 17/41] changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c56c0cb0e..326eb9c891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # nf-core/tools: Changelog -## v1.14dev +## v1.13.2 -_..nothing yet.._ +* Added better logging message if a user doesn't specificy the directory correctly with `nf-core modules` commands [[#942](https://github.com/nf-core/tools/pull/942)] +* Fixed parameter validation bug caused by JSONObject [[#937](https://github.com/nf-core/tools/issues/937)] ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19] From 3261493f89234eb8da1f15a233d54682089f47f0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 22 Mar 2021 22:01:03 +0100 Subject: [PATCH 18/41] Refactor code for tidyness points --- nf_core/create.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 40d3b366aa..2492169486 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -3,7 +3,6 @@ organization's specification based on a template. """ from genericpath import exists -from pathlib import Path import git import jinja2 import logging @@ -108,31 +107,32 @@ def render_template(self): output_path = os.path.join(self.outdir, template_fn) os.makedirs(os.path.dirname(output_path), exist_ok=True) - # Just copy binary files - (ftype, encoding) = mimetypes.guess_type(template_fn_path, strict=False) - if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): - log.debug(f"Copying binary file: '{output_path}'") - shutil.copy(template_fn_path, output_path) - continue + try: + # Just copy certain file extensions + filename, file_extension = os.path.splitext(template_fn_path) + if file_extension in binary_extensions: + raise AttributeError(f"File extension: {file_extension}") - if Path(template_fn_path).suffix in binary_extensions: - log.debug(f"Copying binary file: '{output_path}'") - shutil.copy(template_fn_path, output_path) - continue + # Try to detect binary files + (ftype, encoding) = mimetypes.guess_type(template_fn_path, strict=False) + if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): + raise AttributeError(f"Encoding: {encoding}") - # Render the template - try: + # Got this far - render the template log.debug(f"Rendering template file: '{template_fn}'") j_template = env.get_template(template_fn) rendered_output = j_template.render(object_attrs) - except UnicodeDecodeError as e: - log.error(f"Could not decode file {template_fn_path} {e}") - sys.exit(1) - # Write to the pipeline output file - with open(output_path, "w") as fh: - log.debug(f"Writing to output file: '{output_path}'") - fh.write(rendered_output) + # Write to the pipeline output file + with open(output_path, "w") as fh: + log.debug(f"Writing to output file: '{output_path}'") + fh.write(rendered_output) + + # Copy the file directly instead of using Jinja + except (AttributeError, UnicodeDecodeError) as e: + log.debug(f"Copying file without Jinja: '{output_path}' - {e}") + shutil.copy(template_fn_path, output_path) + continue # Make a logo and save it self.make_pipeline_logo() From e6ccc10baa4e9160435f8c3b40ffbc95945e2624 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 10:43:35 +0100 Subject: [PATCH 19/41] Pipeline create - apply file permissions to new files Fixes nf-core/tools#923 --- nf_core/create.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/create.py b/nf_core/create.py index 2492169486..bf4271b6d3 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -134,6 +134,10 @@ def render_template(self): shutil.copy(template_fn_path, output_path) continue + # Mirror file permissions + template_stat = os.stat(template_fn_path) + os.chmod(output_path, template_stat.st_mode) + # Make a logo and save it self.make_pipeline_logo() From 2013dd1c1e59c50497d93820cfc9b16fcad6205c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 10:46:45 +0100 Subject: [PATCH 20/41] File perms on nf-core modules create too --- nf_core/modules/create.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 89340ecf94..d01369ca3c 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -236,6 +236,10 @@ def render_template(self): log.debug(f"Writing output to: '{dest_fn}'") fh.write(rendered_output) + # Mirror file permissions + template_stat = os.stat(template_fn) + os.chmod(dest_fn, template_stat.st_mode) + def get_repo_type(self, directory): """ Determine whether this is a pipeline repository or a clone of From dcec8299cb29174c81cee33c63289037f7365a28 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 10:47:23 +0100 Subject: [PATCH 21/41] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f267e9a92..da937bd1d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Added better logging message if a user doesn't specificy the directory correctly with `nf-core modules` commands [[#942](https://github.com/nf-core/tools/pull/942)] * Fixed parameter validation bug caused by JSONObject [[#937](https://github.com/nf-core/tools/issues/937)] +* Fixed template creation error regarding file permissions [[#923](https://github.com/nf-core/tools/issues/923)] * Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read * Added automated PR comments to the Markdownlint and Python Black lint CI tests to explain failures From b28dcaf94b0b5bbebbcee794994f0fa62be653d6 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 11:00:51 +0100 Subject: [PATCH 22/41] =?UTF-8?q?Sometimes=20I=20=E2=9D=A4=EF=B8=8F=20CI?= =?UTF-8?q?=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nf_core/modules/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index d01369ca3c..c788e01f7b 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -237,7 +237,7 @@ def render_template(self): fh.write(rendered_output) # Mirror file permissions - template_stat = os.stat(template_fn) + template_stat = os.stat(os.path.join("nf_core", "module-template", template_fn)) os.chmod(dest_fn, template_stat.st_mode) def get_repo_type(self, directory): From da108c65548983f08943aea576d6aaecbde0c505 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Tue, 23 Mar 2021 11:23:55 +0100 Subject: [PATCH 23/41] added comment in release checklist --- .github/RELEASE_CHECKLIST.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index 00b6f94730..1f9a8b2eb4 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -2,13 +2,14 @@ 1. Check issue milestones to see outstanding issues to resolve if possible or transfer to the milestones for the next release e.g. [`v1.9`](https://github.com/nf-core/tools/issues?q=is%3Aopen+is%3Aissue+milestone%3A1.9) 2. Most importantly, pick an undeniably outstanding [name](http://www.codenamegenerator.com/) for the release where *Prefix* = *Metal* and *Dictionary* = *Animal*. -3. Create a PR to `dev` to bump the version in `CHANGELOG.md` and `setup.py`. -4. Make sure all CI tests are passing! -5. Create a PR from `dev` to `master` -6. Make sure all CI tests are passing again (additional tests are run on PRs to `master`) -7. Request review (2 approvals required) -8. Merge the PR into `master` -9. Once CI tests on commit have passed, create a new release copying the `CHANGELOG` for that release into the description section. +3. Check whether the GitHub Actions workflow scripts need updating of the Nextflow versions +4. Create a PR to `dev` to bump the version in `CHANGELOG.md` and `setup.py`. +5. Make sure all CI tests are passing! +6. Create a PR from `dev` to `master` +7. Make sure all CI tests are passing again (additional tests are run on PRs to `master`) +8. Request review (2 approvals required) +9. Merge the PR into `master` +10. Once CI tests on commit have passed, create a new release copying the `CHANGELOG` for that release into the description section. ## After release From f16f281b52362f96b7b492b0c63542459d6692d1 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Tue, 23 Mar 2021 11:47:50 +0100 Subject: [PATCH 24/41] bumped nextflow versions to edge release --- .github/workflows/create-lint-wf.yml | 2 + .github/workflows/pytest.yml | 59 ++++++++++--------- .github/workflows/sync.yml | 2 + .../.github/workflows/ci.yml | 2 +- 4 files changed, 35 insertions(+), 30 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index eec5e9840c..7535eeb551 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -4,6 +4,8 @@ on: [push, pull_request] jobs: MakeTestWorkflow: runs-on: ubuntu-latest + env: + NXF_VER: 21.03.0-edge steps: - uses: actions/checkout@v2 name: Check out source-code repository diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 493fb53c20..6493838668 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -4,47 +4,48 @@ name: Python tests on: push: paths: - - '**.py' + - "**.py" pull_request: paths: - - '**.py' + - "**.py" jobs: pytest: - runs-on: ubuntu-latest + env: + NXF_VER: 21.03.0-edge strategy: matrix: python-version: [3.6, 3.7, 3.8] steps: - - uses: actions/checkout@v2 - name: Check out source-code repository + - uses: actions/checkout@v2 + name: Check out source-code repository - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - python -m pip install --upgrade pip -r requirements-dev.txt - pip install -e . + - name: Install python dependencies + run: | + python -m pip install --upgrade pip -r requirements-dev.txt + pip install -e . - - name: Install Nextflow - env: + - name: Install Nextflow + env: CAPSULE_LOG: none - run: | - mkdir /tmp/nextflow - cd /tmp/nextflow - wget -qO- get.nextflow.io | bash - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow - - - name: Test with pytest - run: python3 -m pytest --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core - - - uses: codecov/codecov-action@v1 - name: Upload code coverage report - with: - if: success() - token: ${{ secrets.CODECOV_TOKEN }} + run: | + mkdir /tmp/nextflow + cd /tmp/nextflow + wget -qO- get.nextflow.io | bash + sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + + - name: Test with pytest + run: python3 -m pytest --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core + + - uses: codecov/codecov-action@v1 + name: Upload code coverage report + with: + if: success() + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 6d5bd24c6b..957714de52 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -7,6 +7,8 @@ on: jobs: get-pipelines: runs-on: ubuntu-latest + env: + NXF_VER: 21.03.0-edge outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 0228228801..2f387022ee 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: # Nextflow versions: check pipeline minimum and current latest - nxf_ver: ['20.04.0', ''] + nxf_ver: ['20.04.0', '21.03.0-edge'] steps: - name: Check out pipeline code uses: actions/checkout@v2 From 618179c9bb6a6781be09e71d5ccd84a6f6435198 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 11:58:17 +0100 Subject: [PATCH 25/41] Update CHANGELOG.md Co-authored-by: Kevin Menden --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da937bd1d8..64b50dc04d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * Added better logging message if a user doesn't specificy the directory correctly with `nf-core modules` commands [[#942](https://github.com/nf-core/tools/pull/942)] * Fixed parameter validation bug caused by JSONObject [[#937](https://github.com/nf-core/tools/issues/937)] -* Fixed template creation error regarding file permissions [[#923](https://github.com/nf-core/tools/issues/923)] +* Fixed template creation error regarding file permissions [[#932](https://github.com/nf-core/tools/issues/932)] * Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read * Added automated PR comments to the Markdownlint and Python Black lint CI tests to explain failures From 6c9a8d3ae74889ad24895796f6c1bdb2ff0eb146 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 13:05:58 +0100 Subject: [PATCH 26/41] Add code lint CI PR comments to pipeline template --- .github/workflows/markdown-lint.yml | 9 +-- .../.github/workflows/linting.yml | 55 +++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 5a590aff8e..3e2158c9f4 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -30,13 +30,14 @@ jobs: * Install `markdownlint-cli` * On Mac: `brew install markdownlint-cli` - * Everything else: [Install `npm`](https://www.npmjs.com/get-npm) then [install `markdownlint-cli`](https://www.npmjs.com/package/markdownlint-cli): `npm install -g markdownlint-cli` - * Try to automatically fix errors in your pipeline: `markdownlint . --config .github/markdownlint.yml --fix` - * Check that there are no remaining problems: `markdownlint . --config .github/markdownlint.yml` + * Everything else: [Install `npm`](https://www.npmjs.com/get-npm) then [install `markdownlint-cli`](https://www.npmjs.com/package/markdownlint-cli) (`npm install -g markdownlint-cli`) + * Fix the markdown errors + * Automatically: `markdownlint . --config .github/markdownlint.yml --fix` + * Manually resolve anything left from `markdownlint . --config .github/markdownlint.yml` Once you push these changes the test should pass, and you can hide this comment :+1: - We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + We highly recommend setting up markdownlint in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! Thanks again for your contribution! repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 900067a0da..50918d9776 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -19,6 +19,34 @@ jobs: run: npm install -g markdownlint-cli - name: Run Markdownlint run: markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Markdown linting is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install `markdownlint-cli` + * On Mac: `brew install markdownlint-cli` + * Everything else: [Install `npm`](https://www.npmjs.com/get-npm) then [install `markdownlint-cli`](https://www.npmjs.com/package/markdownlint-cli) (`npm install -g markdownlint-cli`) + * Fix the markdown errors + * Automatically: `markdownlint . --config .github/markdownlint.yml --fix` + * Manually resolve anything left from `markdownlint . --config .github/markdownlint.yml` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up markdownlint in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false + + YAML: runs-on: ubuntu-latest steps: @@ -30,6 +58,33 @@ jobs: run: npm install -g yaml-lint - name: Run yaml-lint run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml") + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## YAML linting is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install `yaml-lint` + * [Install `npm`](https://www.npmjs.com/get-npm) then [install `yaml-lint`](https://www.npmjs.com/package/yaml-lint) (`npm install -g yaml-lint`) + * Fix the markdown errors + * Run the test locally: `yamllint $(find . -type f -name "*.yml")` + * Fix any reported errors in your YAML files + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up yaml-lint in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false + + nf-core: runs-on: ubuntu-latest steps: From ac62403b54327f530e5acf0717074fb3ef23f69c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 13:06:45 +0100 Subject: [PATCH 27/41] Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b50dc04d..905550a74e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * Fixed parameter validation bug caused by JSONObject [[#937](https://github.com/nf-core/tools/issues/937)] * Fixed template creation error regarding file permissions [[#932](https://github.com/nf-core/tools/issues/932)] * Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read -* Added automated PR comments to the Markdownlint and Python Black lint CI tests to explain failures +* Added automated PR comments to the Markdown, YAML and Python lint CI tests to explain failures (tools and pipeline template) ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19] From 7a1c333a480f7ad9d6184e20933fe3b449d365a4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 13:31:17 +0100 Subject: [PATCH 28/41] Copy file if any exception with jinja in pipeline create --- nf_core/create.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index bf4271b6d3..85c60b7bc4 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -132,7 +132,11 @@ def render_template(self): except (AttributeError, UnicodeDecodeError) as e: log.debug(f"Copying file without Jinja: '{output_path}' - {e}") shutil.copy(template_fn_path, output_path) - continue + + # Something else went wrong + except Exception as e: + log.error(f"Copying raw file as error rendering with Jinja: '{output_path}' - {e}") + shutil.copy(template_fn_path, output_path) # Mirror file permissions template_stat = os.stat(template_fn_path) From 719d22d4666378e2dcaa02b2c0f41994f79d7a07 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 13:31:37 +0100 Subject: [PATCH 29/41] Make lint summary table coloured according to overall status --- nf_core/lint/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 338c908bbb..4d9c5de35d 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -318,14 +318,15 @@ def _s(some_list): console.print(table) # Summary table - table = Table(box=rich.box.ROUNDED) - table.add_column("[bold green]LINT RESULTS SUMMARY".format(len(self.passed)), no_wrap=True) - table.add_row(r"[✔] {:>3} Test{} Passed".format(len(self.passed), _s(self.passed)), style="green") + summary_colour = "red" if len(self.failed) > 0 else "green" + table = Table(box=rich.box.ROUNDED, style=summary_colour) + table.add_column(f"LINT RESULTS SUMMARY".format(len(self.passed)), no_wrap=True) + table.add_row(r"[green][✔] {:>3} Test{} Passed".format(len(self.passed), _s(self.passed))) if len(self.fix): - table.add_row(r"[?] {:>3} Test{} Fixed".format(len(self.fixed), _s(self.fixed)), style="bright_blue") - table.add_row(r"[?] {:>3} Test{} Ignored".format(len(self.ignored), _s(self.ignored)), style="grey58") - table.add_row(r"[!] {:>3} Test Warning{}".format(len(self.warned), _s(self.warned)), style="yellow") - table.add_row(r"[✗] {:>3} Test{} Failed".format(len(self.failed), _s(self.failed)), style="red") + table.add_row(r"[bright blue][?] {:>3} Test{} Fixed".format(len(self.fixed), _s(self.fixed))) + table.add_row(r"[grey58][?] {:>3} Test{} Ignored".format(len(self.ignored), _s(self.ignored))) + table.add_row(r"[yellow][!] {:>3} Test Warning{}".format(len(self.warned), _s(self.warned))) + table.add_row(r"[red][✗] {:>3} Test{} Failed".format(len(self.failed), _s(self.failed))) console.print(table) if len(self.could_fix): From 217eceae6afc6de5dc4d063d8055d9cc82cb9200 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 13:32:03 +0100 Subject: [PATCH 30/41] Proper {{}} escaping in workflow for new comments --- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 50918d9776..5b09090281 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -1,4 +1,4 @@ -name: nf-core linting +{% raw %}name: nf-core linting # This workflow is triggered on pushes and PRs to the repository. # It runs the `nf-core lint` and markdown lint tests to ensure that the code meets the nf-core guidelines on: @@ -108,7 +108,7 @@ jobs: run: | python -m pip install --upgrade pip pip install nf-core -{% raw %} + - name: Run nf-core lint env: GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} From 6084456892e7457d8f5bb971861ef9590a373ed5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 13:33:10 +0100 Subject: [PATCH 31/41] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b50dc04d..ff7a7847af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Fixed template creation error regarding file permissions [[#932](https://github.com/nf-core/tools/issues/932)] * Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read * Added automated PR comments to the Markdownlint and Python Black lint CI tests to explain failures +* Make `nf-core lint` summary table borders coloured according to overall pass / fail status ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19] From 17fbcf8de2104aefc769850dc57e626ebb8bc4e5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 13:44:56 +0100 Subject: [PATCH 32/41] Use '21.03.0-edge' in bump-versions code --- nf_core/bump_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 28e3f9eeaa..b770cb2e65 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -145,8 +145,8 @@ def bump_nextflow_version(pipeline_obj, new_version): pipeline_obj, [ ( - r"nxf_ver: \[[\'\"]?{}[\'\"]?, ''\]".format(current_version.replace(".", r"\.")), - "nxf_ver: ['{}', '']".format(new_version), + r"nxf_ver: \[[\'\"]?{}[\'\"]?, '21.03.0-edge'\]".format(current_version.replace(".", r"\.")), + "nxf_ver: ['{}', '21.03.0-edge']".format(new_version), ) ], ) From a14af2f310c10f21f6673bc63dc44a4ee56d5e66 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:08:16 +0100 Subject: [PATCH 33/41] Find yml and yaml files for yaml linting --- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 5b09090281..4570f43bda 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -57,7 +57,7 @@ jobs: - name: Install yaml-lint run: npm install -g yaml-lint - name: Run yaml-lint - run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml") + run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml" -o -name "*.yaml") # If the above check failed, post a comment on the PR explaining the failure - name: Post PR comment @@ -73,7 +73,7 @@ jobs: * Install `yaml-lint` * [Install `npm`](https://www.npmjs.com/get-npm) then [install `yaml-lint`](https://www.npmjs.com/package/yaml-lint) (`npm install -g yaml-lint`) * Fix the markdown errors - * Run the test locally: `yamllint $(find . -type f -name "*.yml")` + * Run the test locally: `yamllint $(find . -type f -name "*.yml" -o -name "*.yaml")` * Fix any reported errors in your YAML files Once you push these changes the test should pass, and you can hide this comment :+1: From f714baae3df276df21c3908504c2677094724aef Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:11:24 +0100 Subject: [PATCH 34/41] Log API headers on fail - create PR --- nf_core/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 68e9a9c5a4..c7520c131b 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -340,7 +340,9 @@ def make_pull_request(self): # Something went wrong else: - raise PullRequestException(f"GitHub API returned code {r.status_code}: \n{returned_data_prettyprint}") + raise PullRequestException( + f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r.headers}" + ) def close_open_template_merge_prs(self): """Get all template merging branches (starting with 'nf-core-template-merge-') From 01dcdf23408ba7d20789f8eb3f1603f44dcca72a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:18:15 +0100 Subject: [PATCH 35/41] Sync - keep trying if we get 403 errors --- nf_core/sync.py | 75 +++++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index c7520c131b..02ede2ceaa 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -6,9 +6,11 @@ import json import logging import os +import re import requests import requests_cache import shutil +import time import nf_core import nf_core.create @@ -311,38 +313,49 @@ def make_pull_request(self): "please see the `v{tag}` [release page](https://github.com/nf-core/tools/releases/tag/{tag})." ).format(tag=nf_core.__version__) - # Make new pull-request - pr_content = { - "title": pr_title, - "body": pr_body_text, - "maintainer_can_modify": True, - "head": self.merge_branch, - "base": self.from_branch, - } - - r = requests.post( - url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), - data=json.dumps(pr_content), - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), - ) - try: - self.gh_pr_returned_data = json.loads(r.content) - returned_data_prettyprint = json.dumps(self.gh_pr_returned_data, indent=4) - except: - self.gh_pr_returned_data = r.content - returned_data_prettyprint = r.content - - # PR worked - if r.status_code == 201: - self.pr_url = self.gh_pr_returned_data["html_url"] - log.debug("GitHub API PR worked:\n{}".format(returned_data_prettyprint)) - log.info("GitHub PR created: {}".format(self.gh_pr_returned_data["html_url"])) - - # Something went wrong - else: - raise PullRequestException( - f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r.headers}" + while True: + # Make new pull-request + pr_content = { + "title": pr_title, + "body": pr_body_text, + "maintainer_can_modify": True, + "head": self.merge_branch, + "base": self.from_branch, + } + + r = requests.post( + url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), + data=json.dumps(pr_content), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), ) + try: + self.gh_pr_returned_data = json.loads(r.content) + returned_data_prettyprint = json.dumps(self.gh_pr_returned_data, indent=4) + except: + self.gh_pr_returned_data = r.content + returned_data_prettyprint = r.content + + # PR worked + if r.status_code == 201: + self.pr_url = self.gh_pr_returned_data["html_url"] + log.debug(f"GitHub API PR worked:\n{returned_data_prettyprint}") + log.info(f"GitHub PR created: {self.gh_pr_returned_data["html_url"]}") + break + + # Returned 403 error - too many simultaneous requests + # https://github.com/nf-core/tools/issues/911 + if r.status_code == 403: + log.debug(f"GitHub API PR failed with 403 error:\n{returned_data_prettyprint}\n\n{r.headers}") + wait_time = float(re.sub('[^0-9]','', r.headers.get('Retry-After', 30))) + log.warning(f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds..") + time.sleep(wait_time) + + + # Something went wrong + else: + raise PullRequestException( + f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r.headers}" + ) def close_open_template_merge_prs(self): """Get all template merging branches (starting with 'nf-core-template-merge-') From 07c3978590496443de05f1519f7af42b114f681c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:19:00 +0100 Subject: [PATCH 36/41] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d13846bfb2..87f42a3026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Split the `create-lint-wf` tests up into separate steps in GitHub Actions to make the CI results easier to read * Added automated PR comments to the Markdown, YAML and Python lint CI tests to explain failures (tools and pipeline template) * Make `nf-core lint` summary table borders coloured according to overall pass / fail status +* Attempted a fix for the automated sync when we submit too many PRs at once [[#911](https://github.com/nf-core/tools/issues/911)] ## [v1.13.1 - Copper Crocodile Patch :crocodile: :pirate_flag:](https://github.com/nf-core/tools/releases/tag/1.13.1) - [2021-03-19] From d0633b2b4db0af115afc358dd9df1788b76c7db5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:20:52 +0100 Subject: [PATCH 37/41] Fix copypasta syntax error --- nf_core/sync.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 02ede2ceaa..468145ba0b 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -339,18 +339,17 @@ def make_pull_request(self): if r.status_code == 201: self.pr_url = self.gh_pr_returned_data["html_url"] log.debug(f"GitHub API PR worked:\n{returned_data_prettyprint}") - log.info(f"GitHub PR created: {self.gh_pr_returned_data["html_url"]}") + log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}") break # Returned 403 error - too many simultaneous requests # https://github.com/nf-core/tools/issues/911 if r.status_code == 403: log.debug(f"GitHub API PR failed with 403 error:\n{returned_data_prettyprint}\n\n{r.headers}") - wait_time = float(re.sub('[^0-9]','', r.headers.get('Retry-After', 30))) + wait_time = float(re.sub("[^0-9]", "", r.headers.get("Retry-After", 30))) log.warning(f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds..") time.sleep(wait_time) - # Something went wrong else: raise PullRequestException( From 3f15ce010ab28d35b452aeb51f65532ccda69f82 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:22:36 +0100 Subject: [PATCH 38/41] fix tests --- tests/test_sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_sync.py b/tests/test_sync.py index 4a34e68a0e..3900a95d5d 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -192,6 +192,7 @@ class MockResponse: def __init__(self, data, status_code): self.status_code = status_code self.content = json.dumps(data) + self.headers = {"content-encoding": "test", "connection": "fake"} if kwargs["url"] == "https://api.github.com/repos/no_existing_pr/response/pulls": response_data = {"html_url": "great_success"} From fa00cde38cd2dcd9b3e45216698e2ea1f0d435e0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:24:03 +0100 Subject: [PATCH 39/41] Reduce diff - didn't need to indent that bit --- nf_core/sync.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 468145ba0b..ee8893089c 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -313,16 +313,16 @@ def make_pull_request(self): "please see the `v{tag}` [release page](https://github.com/nf-core/tools/releases/tag/{tag})." ).format(tag=nf_core.__version__) - while True: - # Make new pull-request - pr_content = { - "title": pr_title, - "body": pr_body_text, - "maintainer_can_modify": True, - "head": self.merge_branch, - "base": self.from_branch, - } + # Make new pull-request + pr_content = { + "title": pr_title, + "body": pr_body_text, + "maintainer_can_modify": True, + "head": self.merge_branch, + "base": self.from_branch, + } + while True: r = requests.post( url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), data=json.dumps(pr_content), From ba85e6cf34d8d43df3c5587e7a607aa38036cf51 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:32:08 +0100 Subject: [PATCH 40/41] Bump to v1.13.2 --- CHANGELOG.md | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87f42a3026..d354c50ab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v1.13.2dev +## [v1.13.2 - Copper Crocodile CPR :crocodile: :face_with_head_bandage:](https://github.com/nf-core/tools/releases/tag/1.13.2) - [2021-03-23] * Added better logging message if a user doesn't specificy the directory correctly with `nf-core modules` commands [[#942](https://github.com/nf-core/tools/pull/942)] * Fixed parameter validation bug caused by JSONObject [[#937](https://github.com/nf-core/tools/issues/937)] diff --git a/setup.py b/setup.py index d5dc60e0d9..9f27ad7dde 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,8 @@ #!/usr/bin/env python from setuptools import setup, find_packages -import sys -version = "1.13.2dev" +version = "1.13.2" with open("README.md") as f: readme = f.read() From 54f7f18fe96ef32e9c012b2748886c45b99327af Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 23 Mar 2021 14:37:28 +0100 Subject: [PATCH 41/41] Update .github/workflows/pytest.yml --- .github/workflows/pytest.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 6493838668..b31b62e8b5 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -36,10 +36,8 @@ jobs: env: CAPSULE_LOG: none run: | - mkdir /tmp/nextflow - cd /tmp/nextflow wget -qO- get.nextflow.io | bash - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + sudo mv nextflow /usr/local/bin/ - name: Test with pytest run: python3 -m pytest --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core