diff --git a/assets/schema_mappings.yml b/assets/schema_mappings.yml new file mode 100644 index 00000000..7c9b8452 --- /dev/null +++ b/assets/schema_mappings.yml @@ -0,0 +1,28 @@ +$schema: 'http://json-schema.org/draft-07/schema' +$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/assets/schema_mappings.yml' +title: 'nf-core/fetchngs pipeline - id_mappings.csv schema' +description: 'Schema for the mappings file produced by fetchngs' +type: array +items: + type: object + properties: + sample: + type: string + experiment_accession: + type: string + run_accession: + type: string + sample_accession: + type: string + experiment_alias: + type: string + run_alias: + type: string + sample_alias: + type: string + experiment_title: + type: string + sample_title: + type: string + sample_description: + type: string diff --git a/assets/schema_samplesheet.yml b/assets/schema_samplesheet.yml new file mode 100644 index 00000000..bb9a7e78 --- /dev/null +++ b/assets/schema_samplesheet.yml @@ -0,0 +1,81 @@ +$schema: 'http://json-schema.org/draft-07/schema' +$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/assets/schema_mappings.yml' +title: 'nf-core/fetchngs pipeline - samplesheet.csv schema' +description: 'Schema for the samplesheet file produced by fetchngs' +type: array +items: + type: object + properties: + sample: + type: string + fastq_1: + type: string + format: file-path + pattern: '^\\S+\\.f(ast)?q\\.gz$' + fastq_2: + type: string + format: file-path + pattern: '^\\S+\\.f(ast)?q\\.gz$' + run_accession: + type: string + experiment_accession: + type: string + sample_accession: + type: string + secondary_sample_accession: + type: string + study_accession: + type: string + secondary_study_accession: + type: string + submission_accession: + type: string + run_alias: + type: string + experiment_alias: + type: string + sample_alias: + type: string + study_alias: + type: string + library_layout: + type: string + library_selection: + type: string + library_source: + type: string + library_strategy: + type: string + library_name: + type: string + instrument_model: + type: string + instrument_platform: + type: string + base_count: + type: integer + read_count: + type: integer + tax_id: + type: string + scientific_name: + type: string + sample_title: + type: string + experiment_title: + type: string + study_title: + type: string + sample_description: + type: string + fastq_md5: + type: string + pattern: '^[0-9a-f]{32}$' + fastq_bytes: + type: integer + fastq_ftp: + type: string + fastq_galaxy: + type: string + fastq_aspera: + type: string diff --git a/conf/base.config b/conf/base.config index 6af79a7b..6af45542 100644 --- a/conf/base.config +++ b/conf/base.config @@ -14,12 +14,6 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - publishDir = [ - path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/main.nf b/main.nf index b5b499d5..b11373f5 100644 --- a/main.nf +++ b/main.nf @@ -10,6 +10,7 @@ */ nextflow.enable.dsl = 2 +nextflow.preview.output = true /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -83,6 +84,11 @@ workflow { ) } +output { + directory params.outdir + mode params.publish_dir_mode +} + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ THE END diff --git a/modules/local/aspera_cli/nextflow.config b/modules/local/aspera_cli/nextflow.config index fa2dbd90..9a808242 100644 --- a/modules/local/aspera_cli/nextflow.config +++ b/modules/local/aspera_cli/nextflow.config @@ -1,17 +1,5 @@ process { withName: 'ASPERA_CLI' { ext.args = '-QT -l 300m -P33001' - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] } } diff --git a/modules/local/multiqc_mappings_config/nextflow.config b/modules/local/multiqc_mappings_config/nextflow.config deleted file mode 100644 index 11c58341..00000000 --- a/modules/local/multiqc_mappings_config/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: 'MULTIQC_MAPPINGS_CONFIG' { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} diff --git a/modules/local/sra_fastq_ftp/nextflow.config b/modules/local/sra_fastq_ftp/nextflow.config index 56e43959..26261f26 100644 --- a/modules/local/sra_fastq_ftp/nextflow.config +++ b/modules/local/sra_fastq_ftp/nextflow.config @@ -1,17 +1,5 @@ process { withName: 'SRA_FASTQ_FTP' { ext.args = '-t 5 -nv -c -T 60' - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] } } diff --git a/modules/local/sra_ids_to_runinfo/nextflow.config b/modules/local/sra_ids_to_runinfo/nextflow.config deleted file mode 100644 index 9b9d0b16..00000000 --- a/modules/local/sra_ids_to_runinfo/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: 'SRA_IDS_TO_RUNINFO' { - publishDir = [ - path: { "${params.outdir}/metadata" }, - enabled: false - ] - } -} diff --git a/modules/local/sra_runinfo_to_ftp/nextflow.config b/modules/local/sra_runinfo_to_ftp/nextflow.config deleted file mode 100644 index 43263648..00000000 --- a/modules/local/sra_runinfo_to_ftp/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: 'SRA_RUNINFO_TO_FTP' { - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} diff --git a/modules/local/sra_to_samplesheet/nextflow.config b/modules/local/sra_to_samplesheet/nextflow.config deleted file mode 100644 index da241c1a..00000000 --- a/modules/local/sra_to_samplesheet/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: SRA_TO_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - enabled: false - ] - } -} diff --git a/modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config b/modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config index c4a96e94..df5def04 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config +++ b/modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config @@ -4,8 +4,6 @@ params.settings_file = "${params.settings_path}/user-settings.mkfg" env.NCBI_SETTINGS = params.settings_file process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } withName: CUSTOM_SRATOOLSNCBISETTINGS { containerOptions = { (workflow.containerEngine == 'singularity') ? diff --git a/modules/nf-core/sratools/fasterqdump/nextflow.config b/modules/nf-core/sratools/fasterqdump/nextflow.config index f98b140d..7e1649d1 100644 --- a/modules/nf-core/sratools/fasterqdump/nextflow.config +++ b/modules/nf-core/sratools/fasterqdump/nextflow.config @@ -1,10 +1,5 @@ process { withName: SRATOOLS_FASTERQDUMP { ext.args = '--split-files --include-technical' - publishDir = [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ] } } \ No newline at end of file diff --git a/modules/nf-core/sratools/prefetch/nextflow.config b/modules/nf-core/sratools/prefetch/nextflow.config deleted file mode 100644 index a2ca8848..00000000 --- a/modules/nf-core/sratools/prefetch/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: SRATOOLS_PREFETCH { - publishDir = [ - path: { "${params.outdir}/sra" }, - enabled: false - ] - } -} \ No newline at end of file diff --git a/output.yml b/output.yml new file mode 100644 index 00000000..84d49b17 --- /dev/null +++ b/output.yml @@ -0,0 +1,16 @@ +$schema: 'http://json-schema.org/draft-07/schema' +$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/output.yml' +title: 'nf-core/fetchngs pipeline outputs' +description: '' +type: object +properties: + id_mappings: + type: string + format: file-path + mimetype: text/csv + schema: assets/schema_mappings.yml + samplesheet: + type: string + format: file-path + mimetype: text/csv + schema: assets/schema_samplesheet.yml diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config index de803a38..187faf6d 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config @@ -1,2 +1 @@ -includeConfig '../../../modules/nf-core/sratools/prefetch/nextflow.config' includeConfig '../../../modules/nf-core/sratools/fasterqdump/nextflow.config' diff --git a/workflows/sra/main.nf b/workflows/sra/main.nf index 35ae18dc..854195a8 100644 --- a/workflows/sra/main.nf +++ b/workflows/sra/main.nf @@ -123,6 +123,7 @@ workflow SRA { .fastq .mix(SRA_FASTQ_FTP.out.fastq) .mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads) + .tap { ch_fastq } .map { meta, fastq -> def reads = fastq instanceof List ? fastq.flatten() : [ fastq ] @@ -153,7 +154,7 @@ workflow SRA { .map { it[1] } .collectFile(name:'tmp_samplesheet.csv', newLine: true, keepHeader: true, sort: { it.baseName }) .map { it.text.tokenize('\n').join('\n') } - .collectFile(name:'samplesheet.csv', storeDir: "${params.outdir}/samplesheet") + .collectFile(name:'samplesheet.csv') .set { ch_samplesheet } SRA_TO_SAMPLESHEET @@ -162,7 +163,7 @@ workflow SRA { .map { it[1] } .collectFile(name:'tmp_id_mappings.csv', newLine: true, keepHeader: true, sort: { it.baseName }) .map { it.text.tokenize('\n').join('\n') } - .collectFile(name:'id_mappings.csv', storeDir: "${params.outdir}/samplesheet") + .collectFile(name:'id_mappings.csv') .set { ch_mappings } // @@ -181,7 +182,8 @@ workflow SRA { // Collate and save software versions // softwareVersionsToYAML(ch_versions) - .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true) + .collectFile(name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true) + .set { ch_versions_yml } emit: samplesheet = ch_samplesheet @@ -189,6 +191,16 @@ workflow SRA { sample_mappings = ch_sample_mappings_yml sra_metadata = ch_sra_metadata versions = ch_versions.unique() + + publish: + ch_fastq >> 'fastq/' + ASPERA_CLI.out.md5 >> 'fastq/md5/' + SRA_FASTQ_FTP.out.md5 >> 'fastq/md5/' + SRA_RUNINFO_TO_FTP.out.tsv >> 'metadata/' + ch_versions_yml >> 'pipeline_info/' + ch_samplesheet >> 'samplesheet/' + ch_mappings >> 'samplesheet/' + ch_sample_mappings_yml >> 'samplesheet/' } /* diff --git a/workflows/sra/nextflow.config b/workflows/sra/nextflow.config index d242c238..522b05b8 100644 --- a/workflows/sra/nextflow.config +++ b/workflows/sra/nextflow.config @@ -1,8 +1,3 @@ -includeConfig "../../modules/local/multiqc_mappings_config/nextflow.config" includeConfig "../../modules/local/aspera_cli/nextflow.config" includeConfig "../../modules/local/sra_fastq_ftp/nextflow.config" -includeConfig "../../modules/local/sra_ids_to_runinfo/nextflow.config" -includeConfig "../../modules/local/sra_runinfo_to_ftp/nextflow.config" -includeConfig "../../modules/local/sra_to_samplesheet/nextflow.config" -includeConfig "../../modules/nf-core/sratools/prefetch/nextflow.config" includeConfig "../../subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config"