Added parameter use_all_cpus

CHANGELOG.md

@@ -3,14 +3,15 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## v2.3.0dev - [12-March-2025]
+## v2.3.0dev - [02-April-2025]
 
 ### `Added`
 
 1. Updated nf-core pipeline template to 3.2.0
 2. The minimum required Nextflow version is now 24.10.5
 3. Now the pipeline parameters are presented in separate sections on the PARAMS summary page in the report
 4. Added parameter `hic_alphanumeric_sort` to allow disabling of FASTA sorting by sequence labels
+5. Added parameter `use_all_cpus` to allow multithreaded tasks to consume all the CPUs available on a machine. This behavior is desirable on [MM Cloud](https://www.mmcloud.io) but highly undesirable on AWS Batch, SLURM, etc.
 
 ### `Fixed`
 

conf/modules.config

@@ -417,3 +417,4 @@ process {
         ]
     }
 }
+process { ext.use_all_cpus = params.use_all_cpus }

docs/parameters.md

@@ -131,3 +131,4 @@ Less common options for the pipeline, typically set in a config file.
 | `monochrome_logs`     | Do not use coloured log outputs.                                                                            | `boolean` |         |          | True   |
 | `hook_url`            | Incoming hook URL for messaging service                                                                     | `string`  |         |          | True   |
 | `trace_report_suffix` | Suffix to add to the trace report filename. Default is the date and time in the format yyyy-MM-dd_HH-mm-ss. | `string`  |         |          | True   |
+| `use_all_cpus`        | Use all the available CPUs for each task                                                                    | `boolean` |         |          |        |

modules/gallvp/bwa/mem/main.nf

@@ -35,15 +35,19 @@ process BWA_MEM {
                     "bam"
     def reference = fasta && extension=="cram"  ? "--reference ${fasta}" : ""
     if (!fasta && extension=="cram") error "Fasta reference is required for CRAM output"
-    """
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
+	"""
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     INDEX=`find -L ./ -name "*.amb" | sed 's/\\.amb\$//'`
 
     bwa mem \\
         $args \\
-        -t $task.cpus \\
+        -t \${task_cpus} \\
         \$INDEX \\
         $reads \\
-        | samtools $samtools_command $args2 ${reference} --threads $task.cpus -o ${prefix}.${extension} -
+        | samtools $samtools_command $args2 ${reference} --threads \${task_cpus} -o ${prefix}.${extension} -
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

modules/gallvp/cat/cat/main.nf

@@ -36,12 +36,16 @@ process CAT_CAT {
     out_zip  = prefix.endsWith('.gz')
     in_zip   = file_list[0].endsWith('.gz')
     command1 = (in_zip && !out_zip) ? 'zcat' : 'cat'
-    command2 = (!in_zip && out_zip) ? "| pigz -c -p $task.cpus $args2" : ''
+    command2 = (!in_zip && out_zip) ? "| pigz -c -p \${task_cpus} $args2" : ''
     if(file_list.contains(prefix.trim())) {
         error "The name of the input file can't be the same as for the output prefix in the " +
         "module CAT_CAT (currently `$prefix`). Please choose a different one."
     }
-    """
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
+	"""
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     $command1 \\
         $args \\
         ${file_list.join(' ')} \\

modules/gallvp/ltrfinder/main.nf

@@ -21,10 +21,14 @@ process LTRFINDER {
     script:
     def args            = task.ext.args ?: ''
     def prefix          = task.ext.prefix ?: "${meta.id}"
-    """
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
+	"""
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     LTR_FINDER_parallel \\
         -seq $fasta \\
-        -threads $task.cpus \\
+        -threads \${task_cpus} \\
         $args
 
     mv "${fasta}.finder.combine.scn"    "${prefix}.scn"

modules/gallvp/ltrharvest/main.nf

@@ -21,11 +21,15 @@ process LTRHARVEST {
     script:
     def args = task.ext.args ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
-    """
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
+	"""
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     LTR_HARVEST_parallel \\
         -seq $fasta \\
         $args \\
-        -threads $task.cpus
+        -threads \${task_cpus}
 
     mv "${fasta}.harvest.combine.gff3" \\
         "${prefix}.gff3"

modules/gallvp/ltrretriever/lai/main.nf

@@ -27,12 +27,16 @@ process LTRRETRIEVER_LAI {
     def monoploid_param = monoploid_seqs    ? "-mono $monoploid_seqs"                       : ''
     def lai_output_name = monoploid_seqs    ? "${annotation_out}.${monoploid_seqs}.out.LAI" : "${annotation_out}.LAI"
     def VERSION         = 'beta3.2' // WARN: Version information not provided by tool on CLI. Please update this string when bumping container versions.
-    """
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
+	"""
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     LAI \\
         -genome $fasta \\
         -intact $pass_list \\
         -all $annotation_out \\
-        -t $task.cpus \\
+        -t \${task_cpus} \\
         $monoploid_param \\
         $args \\
         > >(tee "${prefix}.LAI.log") \\

modules/gallvp/ltrretriever/ltrretriever/main.nf

@@ -41,7 +41,11 @@ process LTRRETRIEVER_LTRRETRIEVER {
     //
     // This copy with permissions logic can be removed once https://github.com/oushujun/LTR_retriever/issues/176
     // has been resolved.
-    """
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
+	"""
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     cp \\
         $genome \\
         $writable_genome
@@ -56,7 +60,7 @@ process LTRRETRIEVER_LTRRETRIEVER {
         $infinder \\
         $inmgescan \\
         $non_tgca_file \\
-        -threads $task.cpus \\
+        -threads \${task_cpus} \\
         $args \\
         &> >(tee "${prefix}.log" 2>&1) \\
         || echo "Errors from LTR_retriever printed to ${prefix}.log"

modules/gallvp/minimap2/align/main.nf

@@ -32,19 +32,23 @@ process MINIMAP2_ALIGN {
     def args4 = task.ext.args4 ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
     def bam_index = bam_index_extension ? "${prefix}.bam##idx##${prefix}.bam.${bam_index_extension} --write-index" : "${prefix}.bam"
-    def bam_output = bam_format ? "-a | samtools sort -@ ${task.cpus-1} -o ${bam_index} ${args2}" : "-o ${prefix}.paf"
+    def bam_output = bam_format ? "-a | samtools sort -@ \$((task_cpus-1)) -o ${bam_index} ${args2}" : "-o ${prefix}.paf"
     def cigar_paf = cigar_paf_format && !bam_format ? "-c" : ''
     def set_cigar_bam = cigar_bam && bam_format ? "-L" : ''
     def bam_input = "${reads.extension}".matches('sam|bam|cram')
-    def samtools_reset_fastq = bam_input ? "samtools reset --threads ${task.cpus-1} $args3 $reads | samtools fastq --threads ${task.cpus-1} $args4 |" : ''
+    def samtools_reset_fastq = bam_input ? "samtools reset --threads \$((task_cpus-1)) $args3 $reads | samtools fastq --threads \$((task_cpus-1)) $args4 |" : ''
     def query = bam_input ? "-" : reads
     def target = reference ?: (bam_input ? error("BAM input requires reference") : reads)
 
-    """
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
+	"""
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     $samtools_reset_fastq \\
     minimap2 \\
         $args \\
-        -t $task.cpus \\
+        -t \${task_cpus} \\
         $target \\
         $query \\
         $cigar_paf \\

modules/gallvp/seqkit/seq/main.nf

@@ -29,10 +29,14 @@ process SEQKIT_SEQ {
     extension       = fastx.toString().endsWith('.gz') ? "${extension}.gz" : extension
     def call_gzip   = extension.endsWith('.gz') ? "| gzip -c $args2" : ''
     if("${prefix}.${extension}" == "$fastx") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
-    """
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
+	"""
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     seqkit \\
         seq \\
-        --threads $task.cpus \\
+        --threads \${task_cpus} \\
         $args \\
         $fastx \\
         $call_gzip \\

modules/local/juicer_sort.nf

@@ -17,8 +17,12 @@ process JUICER_SORT {
     task.ext.when == null || task.ext.when
 
     script:
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
     """
-    sort --parallel=${task.cpus} \\
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
+    sort --parallel=\${task_cpus} \\
         -k2,2 -k6,6 \\
         $out_links_txt \\
         > out.sorted.links.txt

modules/local/kraken2.nf

@@ -18,13 +18,17 @@ process KRAKEN2 {
     task.ext.when == null || task.ext.when
 
     script:
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
     """
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     kraken2 \\
         --output "${asm_tag}.kraken2.cut" \\
         --report "${asm_tag}.kraken2.report" \\
         --use-names \\
         --db $db_path \\
-        --threads ${task.cpus} \\
+        --threads \${task_cpus} \\
         $fasta_file > kraken2.log
 
     cat <<-END_VERSIONS > versions.yml

modules/local/mummer.nf

@@ -15,10 +15,14 @@ process MUMMER {
     task.ext.when == null || task.ext.when
 
     script:
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
     """
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
     nucmer \
         --mum \\
-        -t ${task.cpus} \\
+        -t \${task_cpus} \\
         -p "${target}.on.${reference}" \\
         $ref_fasta \\
         $target_fasta

modules/local/ncbi_fcs_gx_screen_samples.nf

@@ -22,8 +22,12 @@ process NCBI_FCS_GX_SCREEN_SAMPLES {
 
     script:
     def VERSION = '0.5.4'
+    def use_all_cpus = task.ext.use_all_cpus ? 'yes' : 'no'
     """
-    export GX_NUM_CORES=$task.cpus
+    n_proc=\$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c)
+    task_cpus=\$([ "$use_all_cpus" = "yes" ] && echo "\$n_proc" || echo "$task.cpus")
+
+    export GX_NUM_CORES=\${task_cpus}
 
     for sample_fasta in $samples;
     do