diff --git a/main.nf b/main.nf index ced534f..210d26c 100644 --- a/main.nf +++ b/main.nf @@ -15,9 +15,9 @@ include { skyline_reports } from "./workflows/skyline_run_reports" include { generate_dia_qc_report } from "./workflows/generate_qc_report" include { panorama_upload_results } from "./workflows/panorama_upload" include { panorama_upload_mzmls } from "./workflows/panorama_upload" +include { save_run_details } from "./workflows/save_run_details" // modules -include { SAVE_RUN_DETAILS } from "./modules/save_run_details" include { ENCYCLOPEDIA_BLIB_TO_DLIB } from "./modules/encyclopedia" include { ENCYCLOPEDIA_DLIB_TO_TSV } from "./modules/encyclopedia" include { BLIB_BUILD_LIBRARY } from "./modules/diann" @@ -54,6 +54,12 @@ workflow { all_elib_ch = null // hold all elibs generated by encyclopedia all_diann_file_ch = null // all files generated by diann to upload + // version file channles + encyclopedia_version = null + diann_version = null + proteowizard_version = null + dia_qc_version = null + config_file = file(workflow.configFiles[1]) // the config file used // check for old param variable names @@ -80,9 +86,6 @@ workflow { } } - // save details about this run - SAVE_RUN_DETAILS() - run_details_file = SAVE_RUN_DETAILS.out.run_details // if accessing panoramaweb and running on aws, set up an aws secret if(workflow.profile == 'aws' && is_panorama_used) { @@ -121,6 +124,13 @@ workflow { ) } + + // save details about this run + input_files = all_mzml_ch.map{ it -> ['Spectra File', it.baseName] } + version_files = Channel.empty() + save_run_details(input_files.collect(), version_files.collect()) + run_details_file = save_run_details.out.run_details + return } @@ -149,6 +159,7 @@ workflow { } all_diann_file_ch = Channel.empty() // will be no diann + diann_version = Channel.empty() // convert blib to dlib if necessary if(params.spectral_library.endsWith(".blib")) { @@ -204,8 +215,9 @@ workflow { ) - final_elib = encyclopedia_quant.out.elib + encyclopedia_version = encyclopedia_quant.out.encyclopedia_version + final_elib = encyclopedia_quant.out.elib all_elib_ch = all_elib_ch.concat( encyclopedia_quant.out.individual_elibs, encyclopedia_quant.out.elib, @@ -259,6 +271,7 @@ workflow { all_elib_ch = Channel.empty() // will be no encyclopedia + encyclopedia_version = Channel.empty() all_mzml_ch = wide_mzml_ch diann_search( @@ -267,6 +280,8 @@ workflow { spectral_library_to_use ) + diann_version = diann_search.out.diann_version + // create compatible spectral library for Skyline, if needed if(!params.skyline.skip) { BLIB_BUILD_LIBRARY(diann_search.out.speclib, @@ -306,6 +321,7 @@ workflow { final_elib, wide_mzml_ch ) + proteowizard_version = skyline_import.out.proteowizard_version } // annotate skyline document if replicate_metadata was specified @@ -320,6 +336,9 @@ workflow { // generate QC report if(!params.qc_report.skip) { generate_dia_qc_report(final_skyline_file, get_input_files.out.replicate_metadata) + dia_qc_version = generate_dia_qc_report.out.dia_qc_version + } else { + dia_qc_version = Channel.empty() } // run reports if requested @@ -340,8 +359,21 @@ workflow { skyr_file_ch = Channel.empty() final_skyline_file = Channel.empty() qc_report_files = Channel.empty() + proteowizard_version = Channel.empty() + dia_qc_version = Channel.empty() } + version_files = encyclopedia_version.concat(diann_version, + proteowizard_version, + dia_qc_version).splitText() + + input_files = fasta.map{ it -> ['Fasta file', it.name] }.concat( + spectral_library.map{ it -> ['Spectra library', it.baseName] }, + all_mzml_ch.map{ it -> ['Spectra file', it.baseName] }) + + save_run_details(input_files.collect(), version_files.collect()) + run_details_file = save_run_details.out.run_details + // upload results to Panorama if(params.panorama.upload) { @@ -380,7 +412,7 @@ def is_panorama_used() { (params.quant_spectra_dir && any_entry_is_panorama(params.quant_spectra_dir)) || (params.chromatogram_library_spectra_dir && any_entry_is_panorama(params.chromatogram_library_spectra_dir)) || (params.skyline_skyr_file && any_entry_is_panorama(params.skyline_skyr_file)) - + } // diff --git a/modules/diann.nf b/modules/diann.nf index b29d798..95a7e44 100644 --- a/modules/diann.nf +++ b/modules/diann.nf @@ -15,6 +15,7 @@ process DIANN_SEARCH { path("report.tsv.speclib"), emit: speclib path("report.tsv"), emit: precursor_tsv path("*.quant"), emit: quant_files + path("diann_version.txt"), emit: version script: @@ -34,12 +35,15 @@ process DIANN_SEARCH { ${diann_params} \ > >(tee "diann.stdout") 2> >(tee "diann.stderr" >&2) mv -v lib.tsv.speclib report.tsv.speclib + + head -n 1 diann.stdout | egrep -o '[0-9]+\\.[0-9]+\\.[0-9]+' | xargs printf "diann_version=%s\\n" > diann_version.txt """ stub: """ touch report.tsv.speclib report.tsv stub.quant touch stub.stderr stub.stdout + diann | egrep -o '[0-9]+\\.[0-9]+\\.[0-9]+' | xargs printf "diann_version=%s\\n" > diann_version.txt """ } @@ -60,6 +64,7 @@ process DIANN_SEARCH_LIB_FREE { path("report.tsv"), emit: precursor_tsv path("*.quant"), emit: quant_files path("lib.predicted.speclib"), emit: predicted_speclib + path("diann_version.txt"), emit: version script: @@ -80,12 +85,15 @@ process DIANN_SEARCH_LIB_FREE { ${diann_params} \ > >(tee "diann.stdout") 2> >(tee "diann.stderr" >&2) mv -v lib.tsv.speclib report.tsv.speclib + + head -n 1 diann.stdout | egrep -o '[0-9]+\\.[0-9]+\\.[0-9]+' | xargs printf "diann_version=%s\\n" > diann_version.txt """ stub: """ touch lib.predicted.speclib report.tsv.speclib report.tsv stub.quant touch stub.stderr stub.stdout + diann | egrep -o '[0-9]+\\.[0-9]+\\.[0-9]+' | xargs printf "diann_version=%s\\n" > diann_version.txt """ } diff --git a/modules/encyclopedia.nf b/modules/encyclopedia.nf index 2578b83..b9edaaa 100644 --- a/modules/encyclopedia.nf +++ b/modules/encyclopedia.nf @@ -76,6 +76,7 @@ process ENCYCLOPEDIA_CREATE_ELIB { path("${outputFilePrefix}-combined-results.elib"), emit: elib path("${outputFilePrefix}-combined-results.elib.peptides.txt"), emit: peptide_quant, optional: true path("${outputFilePrefix}-combined-results.elib.proteins.txt"), emit: protein_quant, optional: true + path("encyclopedia_version.txt"), emit: version script: """ @@ -92,6 +93,10 @@ process ENCYCLOPEDIA_CREATE_ELIB { -percolatorVersion /usr/local/bin/percolator \\ ${encyclopedia_params} \\ > >(tee "${outputFilePrefix}.stdout") 2> >(tee "${outputFilePrefix}.stderr" >&2) + + # get EncyclopeDIA version info + ${exec_java_command(task.memory)} --version > version.txt || echo "encyclopedia_version_exit=\$?" + echo "encyclopedia_version=\$(cat version.txt| awk '{print \$4}')" > encyclopedia_version.txt """ stub: @@ -100,6 +105,10 @@ process ENCYCLOPEDIA_CREATE_ELIB { touch "${outputFilePrefix}-combined-results.elib" touch "${outputFilePrefix}-combined-results.elib.peptides.txt" touch "${outputFilePrefix}-combined-results.elib.proteins.txt" + + # get EncyclopeDIA version info + ${exec_java_command(task.memory)} --version > version.txt || echo "encyclopedia_version_exit=\$?" + echo "encyclopedia_version=\$(cat version.txt| awk '{print \$4}')" > encyclopedia_version.txt """ } diff --git a/modules/qc_report.nf b/modules/qc_report.nf index 72d0b36..493e2ff 100644 --- a/modules/qc_report.nf +++ b/modules/qc_report.nf @@ -28,9 +28,7 @@ process MAKE_EMPTY_FILE { } process PARSE_REPORTS { - publishDir "${params.result_dir}/qc_report", pattern: '*.db3', failOnError: true, mode: 'copy' - publishDir "${params.result_dir}/qc_report", pattern: '*.stdout', failOnError: true, mode: 'copy' - publishDir "${params.result_dir}/qc_report", pattern: '*.stderr', failOnError: true, mode: 'copy' + publishDir "${params.result_dir}/qc_report", failOnError: true, mode: 'copy' label 'process_high_memory' container params.images.qc_pipeline @@ -44,6 +42,7 @@ process PARSE_REPORTS { path('*.qmd'), emit: qc_report_qmd path("*.stdout"), emit: stdout path("*.stderr"), emit: stderr + path('dia_qc_version.txt'), emit: version script: def metadata_arg = replicate_metadata.name == 'EMPTY' ? '' : "-m $replicate_metadata" @@ -59,6 +58,10 @@ process PARSE_REPORTS { ${format_flags(params.qc_report.color_vars, '--addColorVar')} \ qc_report_data.db3 \ > >(tee "make_qmd.stdout") 2> >(tee "make_qmd.stderr") + + # get dia_qc version and git info + dia_qc --version|awk '{print \$3}'|xargs -0 printf 'dia_qc_version=%s' > dia_qc_version.txt + echo "dia_qc_git_repo='\$GIT_REPO - \$GIT_BRANCH [\$GIT_SHORT_HASH]'" >> dia_qc_version.txt """ else @@ -75,11 +78,19 @@ process PARSE_REPORTS { ${format_flags(params.qc_report.color_vars, '--addColorVar')} \ qc_report_data.db3 \ > >(tee "make_qmd.stdout") 2> >(tee "make_qmd.stderr") + + # get dia_qc version and git info + dia_qc --version|awk '{print \$3}'|xargs -0 printf 'dia_qc_version=%s' > dia_qc_version.txt + echo "dia_qc_git_repo='\$GIT_REPO - \$GIT_BRANCH [\$GIT_SHORT_HASH]'" >> dia_qc_version.txt """ stub: """ touch stub.stdout stub.stderr stub.db3 stub.qmd + + # get dia_qc version and git info + dia_qc --version|awk '{print \$3}'|xargs -0 printf 'dia_qc_version=%s' > dia_qc_version.txt + echo "dia_qc_git_repo='\$GIT_REPO - \$GIT_BRANCH [\$GIT_SHORT_HASH]'" >> dia_qc_version.txt """ } diff --git a/modules/save_run_details.nf b/modules/save_run_details.nf deleted file mode 100644 index 6566c14..0000000 --- a/modules/save_run_details.nf +++ /dev/null @@ -1,25 +0,0 @@ -// create and return a text file that contains -// details about this nextflow workflow run - -process SAVE_RUN_DETAILS { - label 'process_low_constant' - publishDir "${params.result_dir}", failOnError: true, mode: 'copy' - container params.images.ubuntu - - output: - path("nextflow_run_details.txt"), emit: run_details - - script: - """ - - echo "Nextflow run at: ${workflow.start}" > nextflow_run_details.txt - echo "Nextflow version: ${nextflow.version}" >> nextflow_run_details.txt - echo "Workflow git address: ${workflow.repository}" >> nextflow_run_details.txt - echo "Workflow git revision (branch): ${workflow.revision}" >> nextflow_run_details.txt - echo "Workflow git commit hash: ${workflow.commitId}" >> nextflow_run_details.txt - echo "Run session ID: ${workflow.sessionId}" >> nextflow_run_details.txt - echo "Command line: ${workflow.commandLine}" >> nextflow_run_details.txt - - echo "Done!" # Needed for proper exit - """ -} diff --git a/modules/skyline.nf b/modules/skyline.nf index acc91ad..d27076c 100644 --- a/modules/skyline.nf +++ b/modules/skyline.nf @@ -18,27 +18,70 @@ process SKYLINE_ADD_LIB { output: path("results.sky.zip"), emit: skyline_zipfile path("skyline_add_library.log"), emit: log + path("pwiz_versions.txt"), emit: version - script: - """ - unzip ${skyline_template_zipfile} + shell: + ''' + unzip !{skyline_template_zipfile} wine SkylineCmd \ - --in="${skyline_template_zipfile.baseName}" \ + --in="!{skyline_template_zipfile.baseName}" \ --log-file=skyline_add_library.log \ - --import-fasta="${fasta}" \ - --add-library-path="${elib}" \ + --import-fasta="!{fasta}" \ + --add-library-path="!{elib}" \ --out="results.sky" \ --save \ --share-zip="results.sky.zip" \ --share-type="complete" - """ + + # parse Skyline version info + wine SkylineCmd --version > version.txt + vars=($(cat version.txt | \ + tr -cd '\\11\\12\\15\\40-\\176' | \ + egrep -o 'Skyline.*' | \ + sed -E "s/(Skyline[-a-z]*) \\((.*)\\) ([.0-9]+) \\(([A-Za-z0-9]{7})\\)/\\1 \\3 \\4/")) + skyline_build="${vars[0]}" + skyline_version="${vars[1]}" + skyline_commit="${vars[2]}" + + # parse msconvert info + msconvert_version=$(cat version.txt | \ + tr -cd '\\11\\12\\15\\40-\\176' | \ + egrep -o 'Proteo[a-zA-Z0-9\\. ]+' | \ + egrep -o [0-9].*) + + echo "skyline_build=${skyline_build}" > pwiz_versions.txt + echo "skyline_version=${skyline_version}" >> pwiz_versions.txt + echo "skyline_commit=${skyline_commit}" >> pwiz_versions.txt + echo "msconvert_version=${msconvert_version}" >> pwiz_versions.txt + ''' stub: - """ + ''' touch "results.sky.zip" touch "skyline_add_library.log" - """ + + # parse Skyline version info + wine SkylineCmd --version > version.txt + vars=(\$(cat version.txt | \ + tr -cd '\\11\\12\\15\\40-\\176' | \ + egrep -o 'Skyline.*' | \ + sed -E "s/(Skyline[-a-z]*) \\((.*)\\) ([.0-9]+) \\(([A-Za-z0-9]{7})\\)/\\1 \\3 \\4/")) + skyline_build="\${vars[0]}" + skyline_version="\${vars[1]}" + skyline_commit="\${vars[2]}" + + # parse msconvert info + msconvert_version=\$(cat version.txt | \ + tr -cd '\\11\\12\\15\\40-\\176' | \ + egrep -o 'Proteo[a-zA-Z0-9\\. ]+' | \ + egrep -o [0-9].*) + + echo "skyline_build=\${skyline_build}" > pwiz_versions.txt + echo "skyline_version=\${skyline_version}" >> pwiz_versions.txt + echo "skyline_commit=\${skyline_commit}" >> pwiz_versions.txt + echo "msconvert_version=\${msconvert_version}" >> pwiz_versions.txt + ''' } process SKYLINE_IMPORT_MZML { diff --git a/workflows/diann_search.nf b/workflows/diann_search.nf index 048a75a..2af93ae 100644 --- a/workflows/diann_search.nf +++ b/workflows/diann_search.nf @@ -17,6 +17,7 @@ workflow diann_search { stdout stderr predicted_speclib + diann_version main: @@ -28,6 +29,7 @@ workflow diann_search { spectral_library, params.diann.params ) + diann_version = DIANN_SEARCH.out.version predicted_speclib = Channel.empty() } else { @@ -37,6 +39,7 @@ workflow diann_search { params.diann.params ) + diann_version = DIANN_SEARCH_LIB_FREE.out.version predicted_speclib = diann_results.predicted_speclib } diff --git a/workflows/encyclopedia_search.nf b/workflows/encyclopedia_search.nf index bb6493e..f62cdd2 100644 --- a/workflows/encyclopedia_search.nf +++ b/workflows/encyclopedia_search.nf @@ -17,6 +17,7 @@ workflow encyclopedia_search { elib peptide_quant protein_quant + encyclopedia_version main: @@ -47,4 +48,5 @@ workflow encyclopedia_search { elib = ENCYCLOPEDIA_CREATE_ELIB.out.elib peptide_quant = ENCYCLOPEDIA_CREATE_ELIB.out.peptide_quant protein_quant = ENCYCLOPEDIA_CREATE_ELIB.out.protein_quant + encyclopedia_version = ENCYCLOPEDIA_CREATE_ELIB.out.version } diff --git a/workflows/generate_qc_report.nf b/workflows/generate_qc_report.nf index 18b3440..6aacd38 100644 --- a/workflows/generate_qc_report.nf +++ b/workflows/generate_qc_report.nf @@ -15,6 +15,7 @@ workflow generate_dia_qc_report { qc_report_qmd qc_report_db qc_tables + dia_qc_version main: @@ -44,5 +45,7 @@ workflow generate_dia_qc_report { } else { qc_tables = Channel.empty() } + + dia_qc_version = PARSE_REPORTS.out.version } diff --git a/workflows/save_run_details.nf b/workflows/save_run_details.nf new file mode 100644 index 0000000..e272680 --- /dev/null +++ b/workflows/save_run_details.nf @@ -0,0 +1,77 @@ + +process WRITE_VERSION_INFO { + label 'process_low_constant' + publishDir "${params.result_dir}", failOnError: true, mode: 'copy' + container params.images.ubuntu + + input: + val workflow_var_names + val workflow_values + val version_file_name + + output: + path(version_file_name), emit: run_details + + shell: + ''' + workflow_var_names=( '!{workflow_var_names.join("' '")}' ) + workflow_values=( '!{workflow_values.join("' '")}' ) + + for i in ${!workflow_var_names[@]} ; do + if [ $i -eq 0 ] ; then + echo "${workflow_var_names[$i]}: ${workflow_values[$i]}" > '!{version_file_name}' + else + echo "${workflow_var_names[$i]}: ${workflow_values[$i]}" >> '!{version_file_name}' + fi + done + ''' +} + +workflow save_run_details { + + take: + input_files + version_files + + emit: + run_details + + main: + + // Read version txt files and create a channel of variable name, value pairs + version_vars = version_files.map{ + program -> program.collect{ it -> + elems = it.split('=').collect{ str -> + str.strip().replaceAll(/^['"]|['"]$/, '') + } + [elems[0], elems[1]] + } + } + + // Create channel of workflow run metadata + workflow_vars = Channel.fromList([["Nextflow run at", workflow.start], + ["Nextflow version", nextflow.version], + ["Workflow git address", "${workflow.repository}"], + ["Workflow git revision (branch)", "${workflow.revision}"], + ["Workflow git commit hash", "${workflow.commitId}"], + ["Run session ID", workflow.sessionId], + ["Command line", workflow.commandLine]]) + + // Create channel of docker image names and paths + docker_images = Channel.fromList(params.images.collect{k, v -> ["${k} docker image", v]}) + + all_vars = workflow_vars.concat( + input_files.flatten().collate(2), + version_vars.flatten().collate(2), + docker_images + ) + + var_names = all_vars.map{ it -> it[0] } + var_values = all_vars.map{ it -> it[1] } + + WRITE_VERSION_INFO(var_names.collect(), var_values.collect(), + 'nextflow_run_details.txt') + + run_details = WRITE_VERSION_INFO.out.run_details +} + diff --git a/workflows/skyline_import.nf b/workflows/skyline_import.nf index a1ae700..3d61db7 100644 --- a/workflows/skyline_import.nf +++ b/workflows/skyline_import.nf @@ -13,6 +13,7 @@ workflow skyline_import { emit: skyline_results + proteowizard_version main: @@ -32,4 +33,5 @@ workflow skyline_import { ) skyline_results = SKYLINE_MERGE_RESULTS.out.final_skyline_zipfile + proteowizard_version = SKYLINE_ADD_LIB.out.version }