Merge pull request #14 from ajmaurais/program_versions

Add informaton to nextflow_run_details file

main.nf

@@ -15,9 +15,9 @@ include { skyline_reports } from "./workflows/skyline_run_reports"
 include { generate_dia_qc_report } from "./workflows/generate_qc_report"
 include { panorama_upload_results } from "./workflows/panorama_upload"
 include { panorama_upload_mzmls } from "./workflows/panorama_upload"
+include { save_run_details } from "./workflows/save_run_details"
 
 // modules
-include { SAVE_RUN_DETAILS } from "./modules/save_run_details"
 include { ENCYCLOPEDIA_BLIB_TO_DLIB } from "./modules/encyclopedia"
 include { ENCYCLOPEDIA_DLIB_TO_TSV } from "./modules/encyclopedia"
 include { BLIB_BUILD_LIBRARY } from "./modules/diann"
@@ -54,6 +54,12 @@ workflow {
     all_elib_ch = null       // hold all elibs generated by encyclopedia
     all_diann_file_ch = null // all files generated by diann to upload
 
+    // version file channles
+    encyclopedia_version = null
+    diann_version = null
+    proteowizard_version = null
+    dia_qc_version = null
+
     config_file = file(workflow.configFiles[1]) // the config file used
 
     // check for old param variable names
@@ -80,9 +86,6 @@ workflow {
         }
     }
 
-    // save details about this run
-    SAVE_RUN_DETAILS()
-    run_details_file = SAVE_RUN_DETAILS.out.run_details
 
     // if accessing panoramaweb and running on aws, set up an aws secret
     if(workflow.profile == 'aws' && is_panorama_used) {
@@ -121,6 +124,13 @@ workflow {
             )
         }
 
+
+        // save details about this run
+        input_files = all_mzml_ch.map{ it -> ['Spectra File', it.baseName] }
+        version_files = Channel.empty()
+        save_run_details(input_files.collect(), version_files.collect())
+        run_details_file = save_run_details.out.run_details
+
         return
     }
 
@@ -149,6 +159,7 @@ workflow {
         }
 
         all_diann_file_ch = Channel.empty()  // will be no diann
+        diann_version = Channel.empty()
 
         // convert blib to dlib if necessary
         if(params.spectral_library.endsWith(".blib")) {
@@ -204,8 +215,9 @@ workflow {
 
         )
 
-        final_elib = encyclopedia_quant.out.elib
+        encyclopedia_version = encyclopedia_quant.out.encyclopedia_version
 
+        final_elib = encyclopedia_quant.out.elib
         all_elib_ch = all_elib_ch.concat(
             encyclopedia_quant.out.individual_elibs,
             encyclopedia_quant.out.elib,
@@ -259,6 +271,7 @@ workflow {
 
 
         all_elib_ch = Channel.empty()  // will be no encyclopedia
+        encyclopedia_version = Channel.empty()
         all_mzml_ch = wide_mzml_ch
 
         diann_search(
@@ -267,6 +280,8 @@ workflow {
             spectral_library_to_use
         )
 
+        diann_version = diann_search.out.diann_version
+
         // create compatible spectral library for Skyline, if needed
         if(!params.skyline.skip) {
             BLIB_BUILD_LIBRARY(diann_search.out.speclib,
@@ -306,6 +321,7 @@ workflow {
                 final_elib,
                 wide_mzml_ch
             )
+            proteowizard_version = skyline_import.out.proteowizard_version
         }
 
         // annotate skyline document if replicate_metadata was specified
@@ -320,6 +336,9 @@ workflow {
         // generate QC report
         if(!params.qc_report.skip) {
             generate_dia_qc_report(final_skyline_file, get_input_files.out.replicate_metadata)
+            dia_qc_version = generate_dia_qc_report.out.dia_qc_version
+        } else {
+            dia_qc_version = Channel.empty()
         }
 
         // run reports if requested
@@ -340,8 +359,21 @@ workflow {
         skyr_file_ch = Channel.empty()
         final_skyline_file = Channel.empty()
         qc_report_files = Channel.empty()
+        proteowizard_version = Channel.empty()
+        dia_qc_version = Channel.empty()
     }
 
+    version_files = encyclopedia_version.concat(diann_version,
+                                                proteowizard_version,
+                                                dia_qc_version).splitText()
+
+    input_files = fasta.map{ it -> ['Fasta file', it.name] }.concat(
+        spectral_library.map{ it -> ['Spectra library', it.baseName] },
+        all_mzml_ch.map{ it -> ['Spectra file', it.baseName] })
+
+    save_run_details(input_files.collect(), version_files.collect())
+    run_details_file = save_run_details.out.run_details
+
     // upload results to Panorama
     if(params.panorama.upload) {
 
@@ -380,7 +412,7 @@ def is_panorama_used() {
            (params.quant_spectra_dir && any_entry_is_panorama(params.quant_spectra_dir)) ||
            (params.chromatogram_library_spectra_dir && any_entry_is_panorama(params.chromatogram_library_spectra_dir)) ||
            (params.skyline_skyr_file && any_entry_is_panorama(params.skyline_skyr_file))
-           
+
 }
 
 //

modules/diann.nf

@@ -15,6 +15,7 @@ process DIANN_SEARCH {
         path("report.tsv.speclib"), emit: speclib
         path("report.tsv"), emit: precursor_tsv
         path("*.quant"), emit: quant_files
+        path("diann_version.txt"), emit: version
 
     script:
 
@@ -34,12 +35,15 @@ process DIANN_SEARCH {
             ${diann_params} \
             > >(tee "diann.stdout") 2> >(tee "diann.stderr" >&2)
         mv -v lib.tsv.speclib report.tsv.speclib
+
+        head -n 1 diann.stdout | egrep -o '[0-9]+\\.[0-9]+\\.[0-9]+' | xargs printf "diann_version=%s\\n" > diann_version.txt
         """
 
     stub:
         """
         touch report.tsv.speclib report.tsv stub.quant
         touch stub.stderr stub.stdout
+        diann | egrep -o '[0-9]+\\.[0-9]+\\.[0-9]+' | xargs printf "diann_version=%s\\n" > diann_version.txt
         """
 }
 
@@ -60,6 +64,7 @@ process DIANN_SEARCH_LIB_FREE {
         path("report.tsv"), emit: precursor_tsv
         path("*.quant"), emit: quant_files
         path("lib.predicted.speclib"), emit: predicted_speclib
+        path("diann_version.txt"), emit: version
 
     script:
 
@@ -80,12 +85,15 @@ process DIANN_SEARCH_LIB_FREE {
             ${diann_params} \
             > >(tee "diann.stdout") 2> >(tee "diann.stderr" >&2)
         mv -v lib.tsv.speclib report.tsv.speclib
+
+        head -n 1 diann.stdout | egrep -o '[0-9]+\\.[0-9]+\\.[0-9]+' | xargs printf "diann_version=%s\\n" > diann_version.txt
         """
 
     stub:
         """
         touch lib.predicted.speclib report.tsv.speclib report.tsv stub.quant
         touch stub.stderr stub.stdout
+        diann | egrep -o '[0-9]+\\.[0-9]+\\.[0-9]+' | xargs printf "diann_version=%s\\n" > diann_version.txt
         """
 }
 

modules/encyclopedia.nf

@@ -76,6 +76,7 @@ process ENCYCLOPEDIA_CREATE_ELIB {
         path("${outputFilePrefix}-combined-results.elib"), emit: elib
         path("${outputFilePrefix}-combined-results.elib.peptides.txt"), emit: peptide_quant, optional: true
         path("${outputFilePrefix}-combined-results.elib.proteins.txt"), emit: protein_quant, optional: true
+        path("encyclopedia_version.txt"), emit: version
 
     script:
     """
@@ -92,6 +93,10 @@ process ENCYCLOPEDIA_CREATE_ELIB {
         -percolatorVersion /usr/local/bin/percolator \\
         ${encyclopedia_params} \\
         > >(tee "${outputFilePrefix}.stdout") 2> >(tee "${outputFilePrefix}.stderr" >&2)
+
+    # get EncyclopeDIA version info
+    ${exec_java_command(task.memory)} --version > version.txt || echo "encyclopedia_version_exit=\$?"
+    echo "encyclopedia_version=\$(cat version.txt| awk '{print \$4}')" > encyclopedia_version.txt
     """
 
     stub:
@@ -100,6 +105,10 @@ process ENCYCLOPEDIA_CREATE_ELIB {
     touch "${outputFilePrefix}-combined-results.elib"
     touch "${outputFilePrefix}-combined-results.elib.peptides.txt"
     touch "${outputFilePrefix}-combined-results.elib.proteins.txt"
+
+    # get EncyclopeDIA version info
+    ${exec_java_command(task.memory)} --version > version.txt || echo "encyclopedia_version_exit=\$?"
+    echo "encyclopedia_version=\$(cat version.txt| awk '{print \$4}')" > encyclopedia_version.txt
     """
 }
 

modules/qc_report.nf

@@ -28,9 +28,7 @@ process MAKE_EMPTY_FILE {
 }
 
 process PARSE_REPORTS {
-    publishDir "${params.result_dir}/qc_report", pattern: '*.db3', failOnError: true, mode: 'copy'
-    publishDir "${params.result_dir}/qc_report", pattern: '*.stdout', failOnError: true, mode: 'copy'
-    publishDir "${params.result_dir}/qc_report", pattern: '*.stderr', failOnError: true, mode: 'copy'
+    publishDir "${params.result_dir}/qc_report", failOnError: true, mode: 'copy'
     label 'process_high_memory'
     container params.images.qc_pipeline
 
@@ -44,6 +42,7 @@ process PARSE_REPORTS {
         path('*.qmd'), emit: qc_report_qmd
         path("*.stdout"), emit: stdout
         path("*.stderr"), emit: stderr
+        path('dia_qc_version.txt'), emit: version
 
     script:
     def metadata_arg = replicate_metadata.name == 'EMPTY' ? '' : "-m $replicate_metadata"
@@ -59,6 +58,10 @@ process PARSE_REPORTS {
             ${format_flags(params.qc_report.color_vars, '--addColorVar')} \
             qc_report_data.db3 \
             > >(tee "make_qmd.stdout") 2> >(tee "make_qmd.stderr")
+
+        # get dia_qc version and git info
+        dia_qc --version|awk '{print \$3}'|xargs -0 printf 'dia_qc_version=%s' > dia_qc_version.txt
+        echo "dia_qc_git_repo='\$GIT_REPO - \$GIT_BRANCH [\$GIT_SHORT_HASH]'" >> dia_qc_version.txt
         """
 
     else
@@ -75,11 +78,19 @@ process PARSE_REPORTS {
             ${format_flags(params.qc_report.color_vars, '--addColorVar')} \
             qc_report_data.db3 \
             > >(tee "make_qmd.stdout") 2> >(tee "make_qmd.stderr")
+
+        # get dia_qc version and git info
+        dia_qc --version|awk '{print \$3}'|xargs -0 printf 'dia_qc_version=%s' > dia_qc_version.txt
+        echo "dia_qc_git_repo='\$GIT_REPO - \$GIT_BRANCH [\$GIT_SHORT_HASH]'" >> dia_qc_version.txt
         """
 
     stub:
     """
     touch stub.stdout stub.stderr stub.db3 stub.qmd
+
+    # get dia_qc version and git info
+    dia_qc --version|awk '{print \$3}'|xargs -0 printf 'dia_qc_version=%s' > dia_qc_version.txt
+    echo "dia_qc_git_repo='\$GIT_REPO - \$GIT_BRANCH [\$GIT_SHORT_HASH]'" >> dia_qc_version.txt
     """
 }
 

modules/skyline.nf

@@ -18,27 +18,70 @@ process SKYLINE_ADD_LIB {
     output:
         path("results.sky.zip"), emit: skyline_zipfile
         path("skyline_add_library.log"), emit: log
+        path("pwiz_versions.txt"), emit: version
 
-    script:
-    """
-    unzip ${skyline_template_zipfile}
+    shell:
+    '''
+    unzip !{skyline_template_zipfile}
 
     wine SkylineCmd \
-        --in="${skyline_template_zipfile.baseName}" \
+        --in="!{skyline_template_zipfile.baseName}" \
         --log-file=skyline_add_library.log \
-        --import-fasta="${fasta}" \
-        --add-library-path="${elib}" \
+        --import-fasta="!{fasta}" \
+        --add-library-path="!{elib}" \
         --out="results.sky" \
         --save \
         --share-zip="results.sky.zip" \
         --share-type="complete"
-    """
+
+    # parse Skyline version info
+    wine SkylineCmd --version > version.txt
+    vars=($(cat version.txt | \
+            tr -cd '\\11\\12\\15\\40-\\176' | \
+            egrep -o 'Skyline.*' | \
+            sed -E "s/(Skyline[-a-z]*) \\((.*)\\) ([.0-9]+) \\(([A-Za-z0-9]{7})\\)/\\1 \\3 \\4/"))
+    skyline_build="${vars[0]}"
+    skyline_version="${vars[1]}"
+    skyline_commit="${vars[2]}"
+
+    # parse msconvert info
+    msconvert_version=$(cat version.txt | \
+                        tr -cd '\\11\\12\\15\\40-\\176' | \
+                        egrep -o 'Proteo[a-zA-Z0-9\\. ]+' | \
+                        egrep -o [0-9].*)
+
+    echo "skyline_build=${skyline_build}" > pwiz_versions.txt
+    echo "skyline_version=${skyline_version}" >> pwiz_versions.txt
+    echo "skyline_commit=${skyline_commit}" >> pwiz_versions.txt
+    echo "msconvert_version=${msconvert_version}" >> pwiz_versions.txt
+    '''
 
     stub:
-    """
+    '''
     touch "results.sky.zip"
     touch "skyline_add_library.log"
-    """
+
+    # parse Skyline version info
+    wine SkylineCmd --version > version.txt
+    vars=(\$(cat version.txt | \
+            tr -cd '\\11\\12\\15\\40-\\176' | \
+            egrep -o 'Skyline.*' | \
+            sed -E "s/(Skyline[-a-z]*) \\((.*)\\) ([.0-9]+) \\(([A-Za-z0-9]{7})\\)/\\1 \\3 \\4/"))
+    skyline_build="\${vars[0]}"
+    skyline_version="\${vars[1]}"
+    skyline_commit="\${vars[2]}"
+
+    # parse msconvert info
+    msconvert_version=\$(cat version.txt | \
+                        tr -cd '\\11\\12\\15\\40-\\176' | \
+                        egrep -o 'Proteo[a-zA-Z0-9\\. ]+' | \
+                        egrep -o [0-9].*)
+
+    echo "skyline_build=\${skyline_build}" > pwiz_versions.txt
+    echo "skyline_version=\${skyline_version}" >> pwiz_versions.txt
+    echo "skyline_commit=\${skyline_commit}" >> pwiz_versions.txt
+    echo "msconvert_version=\${msconvert_version}" >> pwiz_versions.txt
+    '''
 }
 
 process SKYLINE_IMPORT_MZML {

workflows/diann_search.nf

@@ -17,6 +17,7 @@ workflow diann_search {
         stdout
         stderr
         predicted_speclib
+        diann_version
 
     main:
 
@@ -28,6 +29,7 @@ workflow diann_search {
                 spectral_library,
                 params.diann.params
             )
+            diann_version = DIANN_SEARCH.out.version
 
             predicted_speclib = Channel.empty()
         } else {
@@ -37,6 +39,7 @@ workflow diann_search {
                 params.diann.params
             )
 
+            diann_version = DIANN_SEARCH_LIB_FREE.out.version
             predicted_speclib = diann_results.predicted_speclib
         }