From 53e5abac7f51ea24e793772a397298f0ca1c0e8a Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 24 Jun 2024 22:12:52 -0400 Subject: [PATCH 01/11] fix: exception handling for rest calls --- modules/local/seqera_runs_dump/functions.nf | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/modules/local/seqera_runs_dump/functions.nf b/modules/local/seqera_runs_dump/functions.nf index 10043a7..6cc78f5 100644 --- a/modules/local/seqera_runs_dump/functions.nf +++ b/modules/local/seqera_runs_dump/functions.nf @@ -1,6 +1,6 @@ - @Grab('com.github.groovy-wslite:groovy-wslite:1.1.2;transitive=false') import wslite.rest.RESTClient +import groovy.json.JsonSlurper Long getWorkspaceId(orgName, workspaceName, client, authHeader) { def orgResponse = client.get(path: '/orgs', headers: authHeader) @@ -14,6 +14,8 @@ Long getWorkspaceId(orgName, workspaceName, client, authHeader) { if (workspaceReponse.statusCode == 200) { def workspaceMap = workspaceReponse.json?.workspaces.collectEntries { ws -> [ws.name, ws.id]} return workspaceMap?.get(workspaceName) + } else { + log.error "Failed to fetch workspaces for orgId: ${orgId}, statusCode: ${workspaceResponse.statusCode}" } } return null @@ -39,11 +41,18 @@ Map getRunMetadata(meta, log, api_endpoint) { return metaMap ?: [:] } } - } catch(Exception ex) { + } catch (wslite.rest.RESTClientException ex) { log.warn """ Could not get workflow details for workflow ${runId} in workspace ${meta.workspace}: - ↳ Status code ${ex.response.statusCode} returned from request to ${ex.request.url} (authentication headers excluded) + ↳ Status code ${ex.response?.statusCode} returned from request to ${ex.request?.url} (authentication headers excluded) + """.stripIndent() + log.error "Exception: ${ex.message}", ex + } catch (Exception ex) { + log.warn """ + An error occurred while getting workflow details for workflow ${runId} in workspace ${meta.workspace}: + ↳ ${ex.message} """.stripIndent() + log.error "Exception: ${ex.message}", ex } return [:] } From b818772795fdd20390a330dca943dbd515e4009a Mon Sep 17 00:00:00 2001 From: ejseqera Date: Tue, 25 Jun 2024 17:26:09 -0400 Subject: [PATCH 02/11] feat: add support for providing custom java truststore to api/cli --- README.md | 2 ++ main.nf | 4 +++- modules/local/seqera_runs_dump/functions.nf | 15 +++++++++++- modules/local/seqera_runs_dump/main.nf | 8 ++++++- .../local/seqera_runs_dump/nextflow.config | 1 + nextflow.config | 3 +++ nextflow_schema.json | 10 ++++++++ workflows/nf_aggregate/main.nf | 24 +++++++++++-------- 8 files changed, 54 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index db33305..16fce74 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ nextflow run seqeralabs/nf-aggregate \ -profile docker ``` +If you are using a Seqera Platform Enterprise instance that is secured with a private CA SSL certificate not recognized by default Java certificate authorities, you can specify a custom `cacerts` store path through the `--java_truststore_path` parameter and optionally, a password with the `--java_truststore_password`. This certificate will be used to achieve connectivity with your Seqera Platform instance through API and CLI. + ## Output The results from the pipeline will be published in the path specified by the `--outdir` and will consist of the following contents: diff --git a/main.nf b/main.nf index 0950f8e..0d800d3 100644 --- a/main.nf +++ b/main.nf @@ -46,7 +46,9 @@ workflow { ch_multiqc_logo, params.seqera_api_endpoint, params.skip_run_gantt, - params.skip_multiqc + params.skip_multiqc, + params.java_truststore_path, + params.java_truststore_password ) } diff --git a/modules/local/seqera_runs_dump/functions.nf b/modules/local/seqera_runs_dump/functions.nf index 6cc78f5..b813ae7 100644 --- a/modules/local/seqera_runs_dump/functions.nf +++ b/modules/local/seqera_runs_dump/functions.nf @@ -2,6 +2,14 @@ import wslite.rest.RESTClient import groovy.json.JsonSlurper +// Set system properties for custom Java trustStore +def setTrustStore(trustStorePath, trustStorePassword) { + System.setProperty("javax.net.ssl.trustStore", trustStorePath) + if (trustStorePassword) { + System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword) + } +} + Long getWorkspaceId(orgName, workspaceName, client, authHeader) { def orgResponse = client.get(path: '/orgs', headers: authHeader) if (orgResponse.statusCode == 200) { @@ -21,9 +29,14 @@ Long getWorkspaceId(orgName, workspaceName, client, authHeader) { return null } -Map getRunMetadata(meta, log, api_endpoint) { +Map getRunMetadata(meta, log, api_endpoint, trustStorePath, trustStorePassword) { def runId = meta.id def (orgName, workspaceName) = meta.workspace.tokenize("/") + + if (trustStorePath) { + log.info "Setting custom truststore: ${trustStorePath}" + setTrustStore(trustStorePath, trustStorePassword) + } def client = new RESTClient(api_endpoint) def token = System.getenv("TOWER_ACCESS_TOKEN") diff --git a/modules/local/seqera_runs_dump/main.nf b/modules/local/seqera_runs_dump/main.nf index 619a13c..43d324f 100644 --- a/modules/local/seqera_runs_dump/main.nf +++ b/modules/local/seqera_runs_dump/main.nf @@ -8,6 +8,8 @@ process SEQERA_RUNS_DUMP { input: val meta val api_endpoint + val java_truststore_path + val java_truststore_password output: tuple val(metaOut), path("${prefix}"), emit: run_dump @@ -17,11 +19,15 @@ process SEQERA_RUNS_DUMP { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' prefix = task.ext.prefix ?: "${meta.id}" - metaOut = meta + getRunMetadata(meta, log, api_endpoint) + metaOut = meta + getRunMetadata(meta, log, api_endpoint, java_truststore_path, java_truststore_password) fusion = metaOut.fusion ? '--add-fusion-logs' : '' + javaTrustStore = java_truststore_path ? "-Djavax.net.ssl.trustStore=${java_truststore_path}" : '' + javaTrustStorePassword = java_truststore_password ? "-Djavax.net.ssl.trustStorePassword=${java_truststore_password}" : '' """ tw \\ $args \\ + $javaTrustStore \\ + $javaTrustStorePassword \\ --url=${api_endpoint} \\ --access-token=$TOWER_ACCESS_TOKEN \\ runs \\ diff --git a/modules/local/seqera_runs_dump/nextflow.config b/modules/local/seqera_runs_dump/nextflow.config index ce78f9b..dbf621d 100644 --- a/modules/local/seqera_runs_dump/nextflow.config +++ b/modules/local/seqera_runs_dump/nextflow.config @@ -2,6 +2,7 @@ process { withName: 'SEQERA_RUNS_DUMP' { ext.args = { params.seqera_cli_extra_args ? params.seqera_cli_extra_args.split("\\s(?=--)") : '' } ext.args2 = { params.skip_run_gantt ? '' : '--add-task-logs' } + containerOptions = { params.java_truststore_path ? "--volume ${params.java_truststore_path}:${params.java_truststore_path}" : '' } publishDir = [ path: { "${params.outdir}/${metaOut?.projectName?.replace("/", "_") ?: ""}/runs_dump" }, mode: params.publish_dir_mode, diff --git a/nextflow.config b/nextflow.config index f7ca780..4bde377 100644 --- a/nextflow.config +++ b/nextflow.config @@ -18,6 +18,9 @@ params { // Seqera CLI options seqera_api_endpoint = "https://api.cloud.seqera.io" seqera_cli_extra_args = null + java_truststore_path = null + java_truststore_password = null + // MultiQC options multiqc_config = null multiqc_title = null diff --git a/nextflow_schema.json b/nextflow_schema.json index c017879..28d4651 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -48,6 +48,16 @@ "description": "Extra arguments to pass to the Seqera Platform CLI command in addition to defaults defined by the pipeline.", "fa_icon": "fas fa-plus" }, + "java_truststore_path": { + "type": "string", + "description": "Path to custom cacerts Java truststore used by Seqera Platform", + "fa_icon": "fas fa-key" + }, + "java_truststore_password": { + "type": "string", + "description": "Password for custom cacerts Java truststore used by Seqera Platform", + "fa_icon": "fas fa-key" + }, "skip_run_gantt": { "type": "boolean", "description": "Skip GANTT chart creation for each run.", diff --git a/workflows/nf_aggregate/main.nf b/workflows/nf_aggregate/main.nf index ecbd16f..710d597 100644 --- a/workflows/nf_aggregate/main.nf +++ b/workflows/nf_aggregate/main.nf @@ -13,13 +13,15 @@ include { paramsSummaryMap } from 'plugin/nf-validation' workflow NF_AGGREGATE { take: - ids // channel: run ids read in from --input - multiqc_custom_config // channel: user specified custom config file used by MultiQC - multiqc_logo // channel: logo rendered in MultiQC report - seqera_api_endpoint // val: Seqera Platform API endpoint URL - skip_run_gantt // val: Skip GANTT chart creation for each run - skip_multiqc // val: Skip MultiQC - + ids // channel: run ids read in from --input + multiqc_custom_config // channel: user specified custom config file used by MultiQC + multiqc_logo // channel: logo rendered in MultiQC report + seqera_api_endpoint // val: Seqera Platform API endpoint URL + skip_run_gantt // val: Skip GANTT chart creation for each run + skip_multiqc // val: Skip MultiQC + java_truststore_path // val: Path to java truststore if using private certs + java_truststore_password // val: Password for java truststore if using private certs + main: ch_versions = Channel.empty() @@ -30,7 +32,9 @@ workflow NF_AGGREGATE { // SEQERA_RUNS_DUMP ( ids, - seqera_api_endpoint + seqera_api_endpoint, + java_truststore_path, + java_truststore_password ) ch_versions = ch_versions.mix(SEQERA_RUNS_DUMP.out.versions.first()) @@ -40,9 +44,9 @@ workflow NF_AGGREGATE { SEQERA_RUNS_DUMP .out .run_dump - .filter { + .filter { meta, run_dir -> - meta.fusion && !params.skip_run_gantt + meta.fusion && !params.skip_run_gantt } .set { ch_runs_for_gantt } From bd282975e56286e1caae2471ca507c570d297f99 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 28 Jun 2024 14:29:37 +0100 Subject: [PATCH 03/11] fix: Add fullstops at end of lines in schema --- nextflow_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 28d4651..f8fd597 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -50,12 +50,12 @@ }, "java_truststore_path": { "type": "string", - "description": "Path to custom cacerts Java truststore used by Seqera Platform", + "description": "Path to custom cacerts Java truststore used by Seqera Platform.", "fa_icon": "fas fa-key" }, "java_truststore_password": { "type": "string", - "description": "Password for custom cacerts Java truststore used by Seqera Platform", + "description": "Password for custom cacerts Java truststore used by Seqera Platform.", "fa_icon": "fas fa-key" }, "skip_run_gantt": { From 3d05070a99daf520bf5208a80e078598fb747601 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 28 Jun 2024 14:30:28 +0100 Subject: [PATCH 04/11] fix: evaluate as empty string if java truststore parameters are not provided --- workflows/nf_aggregate/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/nf_aggregate/main.nf b/workflows/nf_aggregate/main.nf index 710d597..c30f3c3 100644 --- a/workflows/nf_aggregate/main.nf +++ b/workflows/nf_aggregate/main.nf @@ -33,8 +33,8 @@ workflow NF_AGGREGATE { SEQERA_RUNS_DUMP ( ids, seqera_api_endpoint, - java_truststore_path, - java_truststore_password + java_truststore_path ?: '', + java_truststore_password ?: '' ) ch_versions = ch_versions.mix(SEQERA_RUNS_DUMP.out.versions.first()) From 59bace11fd225b00d5af9d8b9b45d5baa0576a86 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 28 Jun 2024 14:34:07 +0100 Subject: [PATCH 05/11] feat: bump pipeline version to 0.3.0 --- CHANGELOG.md | 16 ++++++++++++++++ nextflow.config | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d557f12..1a179b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [[0.3.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.3.0)] - 2024-06-28 + +### Credits + +Special thanks to the following for their contributions to the release: + +- [Adam Talbot](https://github.com/adamrtalbot) +- [Esha Joshi](https://github.com/ejseqera) +- [Rob Syme](https://github.com/robsyme) + +Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form. + +### Enhancements & fixes + +[PR #49](https://github.com/seqeralabs/nf-aggregate/pull/49) - Add custom java truststore support and improved exception handling + ## [[0.2.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.2.0)] - 2024-05-29 ### Credits diff --git a/nextflow.config b/nextflow.config index 4bde377..5178fbb 100644 --- a/nextflow.config +++ b/nextflow.config @@ -222,6 +222,6 @@ manifest { mainScript = 'main.nf' nextflowVersion = '!>=23.10.0' defaultBranch = 'main' - version = '0.2.0' + version = '0.3.0' doi = '' } From f882e7b5be3bbe0044035482a3b7194b4eb7caf2 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 28 Jun 2024 14:35:22 +0100 Subject: [PATCH 06/11] fix: eclint --- modules/local/seqera_runs_dump/functions.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/seqera_runs_dump/functions.nf b/modules/local/seqera_runs_dump/functions.nf index b813ae7..038cb7c 100644 --- a/modules/local/seqera_runs_dump/functions.nf +++ b/modules/local/seqera_runs_dump/functions.nf @@ -32,7 +32,7 @@ Long getWorkspaceId(orgName, workspaceName, client, authHeader) { Map getRunMetadata(meta, log, api_endpoint, trustStorePath, trustStorePassword) { def runId = meta.id def (orgName, workspaceName) = meta.workspace.tokenize("/") - + if (trustStorePath) { log.info "Setting custom truststore: ${trustStorePath}" setTrustStore(trustStorePath, trustStorePassword) From 6152f645fbd75fb4617909e3ebc48fb90517fbae Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 28 Jun 2024 15:38:23 +0100 Subject: [PATCH 07/11] fix: add new input channels to nf-test files --- modules/local/plot_run_gantt/tests/main.nf.test | 2 ++ modules/local/seqera_runs_dump/tests/main.nf.test | 2 ++ 2 files changed, 4 insertions(+) diff --git a/modules/local/plot_run_gantt/tests/main.nf.test b/modules/local/plot_run_gantt/tests/main.nf.test index 712ed7c..ced8d9c 100644 --- a/modules/local/plot_run_gantt/tests/main.nf.test +++ b/modules/local/plot_run_gantt/tests/main.nf.test @@ -17,6 +17,8 @@ nextflow_process { """ input[0] = ['id': '4Bi5xBK6E2Nbhj', 'workspace': 'community/showcase'] input[1] = "https://api.tower.nf" + input[2] = "" + input[3] = "" """ } } diff --git a/modules/local/seqera_runs_dump/tests/main.nf.test b/modules/local/seqera_runs_dump/tests/main.nf.test index 9d90498..41758fb 100644 --- a/modules/local/seqera_runs_dump/tests/main.nf.test +++ b/modules/local/seqera_runs_dump/tests/main.nf.test @@ -15,6 +15,8 @@ nextflow_process { """ input[0] = ['id': '4Bi5xBK6E2Nbhj', 'workspace': 'community/showcase'] input[1] = "https://api.tower.nf" + input[2] = "" + input[3] = "" """ } } From 2cfdad0c8b100aa99d4f38353f77f8f91b04e034 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 28 Jun 2024 16:20:48 +0100 Subject: [PATCH 08/11] fix: initialise java truststore parameter to empty --- main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index 0d800d3..a25a6ce 100644 --- a/main.nf +++ b/main.nf @@ -47,8 +47,8 @@ workflow { params.seqera_api_endpoint, params.skip_run_gantt, params.skip_multiqc, - params.java_truststore_path, - params.java_truststore_password + params.java_truststore_path ?: '', + params.java_truststore_password ?: '' ) } From 5a9bf540b939584c59ad1d8e6bba1d073fb2f32f Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 28 Jun 2024 16:31:46 +0100 Subject: [PATCH 09/11] fix: add inputs for javastore parameters to workflow tests --- workflows/nf_aggregate/tests/main.nf.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflows/nf_aggregate/tests/main.nf.test b/workflows/nf_aggregate/tests/main.nf.test index 1243975..b4008ae 100644 --- a/workflows/nf_aggregate/tests/main.nf.test +++ b/workflows/nf_aggregate/tests/main.nf.test @@ -22,6 +22,8 @@ nextflow_workflow { input[3] = 'https://api.tower.nf' input[4] = false input[5] = false + input[6] = "" + input[7] = "" """ } } From 8821349cf36535319925adc7b05d025904627544 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 28 Jun 2024 16:35:31 +0100 Subject: [PATCH 10/11] revert: initialise java truststore parameter to empty --- main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index a25a6ce..0d800d3 100644 --- a/main.nf +++ b/main.nf @@ -47,8 +47,8 @@ workflow { params.seqera_api_endpoint, params.skip_run_gantt, params.skip_multiqc, - params.java_truststore_path ?: '', - params.java_truststore_password ?: '' + params.java_truststore_path, + params.java_truststore_password ) } From 847293a03b2d831a01e0544f2f037f0c96fae480 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Mon, 1 Jul 2024 13:49:56 +0200 Subject: [PATCH 11/11] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a179b5..d186c1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [[0.3.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.3.0)] - 2024-06-28 +## [[0.3.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.3.0)] - 2024-07-01 ### Credits