Merge branch 'master' into aws-sdk-v2-fs-impl

Author: jorgee

.github/stale.yml

@@ -0,0 +1,21 @@
+name: 'Mark stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+  workflow_dispatch:
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-stale: 180
+          days-before-close: -1
+          stale-issue-label: stale
+          stale-issue-message: ''
+          stale-pr-label: stale
+          stale-pr-message: ''
+          exempt-issue-labels: bug,planned,security
+          exempt-all-milestones: true
+          exempt-all-assignees: true

.github/workflows/stale.yml

@@ -15,7 +15,6 @@
  */
 
 plugins {
-    id "io.codearte.nexus-staging" version "0.30.0"
     id 'java'
     id 'idea'
 }
@@ -268,8 +267,8 @@ task exportClasspath {
     }
 }
 
-ext.nexusUsername = project.findProperty('nexusUsername')
-ext.nexusPassword = project.findProperty('nexusPassword')
+ext.nexusUsername = project.findProperty('nexusUsername') ?: System.getenv('AWS_ACCESS_KEY_ID')
+ext.nexusPassword = project.findProperty('nexusPassword') ?: System.getenv('AWS_SECRET_ACCESS_KEY')
 ext.nexusFullName = project.findProperty('nexusFullName')
 ext.nexusEmail = project.findProperty('nexusEmail')
 
@@ -347,8 +346,8 @@ configure(coreProjects) {
         repositories {
             maven {
                 // change URLs to point to your repos, e.g. http://my.org/repo
-                def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2/"
-                def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots/"
+                def releasesRepoUrl = "s3://maven.seqera.io/releases/"
+                def snapshotsRepoUrl = "s3://maven.seqera.io/snapshots/"
                 url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
                 credentials(PasswordCredentials) {
                     username nexusUsername
@@ -393,17 +392,6 @@ task upload {
     dependsOn coreProjects.publish
 }
 
-/*
- * Configure Nextflow staging plugin -- https://github.com/Codearte/gradle-nexus-staging-plugin
- * It adds the tasks
- * - closeRepository
- * - releaseRepository
- * - closeAndReleaseRepository
- */
-nexusStaging {
-    packageGroup = 'io.nextflow'
-    delayBetweenRetriesInMillis = 10_000
-}
 
 if( System.env.BUILD_PACK ) {
     apply from: 'packing.gradle'

build.gradle

@@ -12,7 +12,7 @@ group = "io.nextflow"
 
 dependencies {
     implementation ('com.amazonaws:aws-java-sdk-s3:1.12.777')
-    implementation 'com.google.code.gson:gson:2.10.1'
+    implementation 'com.google.code.gson:gson:2.13.1'
 }
 
 gradlePlugin {

buildSrc/build.gradle

@@ -1,5 +1,11 @@
 NEXTFLOW CHANGE-LOG
 ===================
+25.04.4 - 16 Jun 2024
+- Fix bug in generated Groovy code (#6082) [ead2f320]
+- Fix default imports in included configs (#6096) [831a577b]
+- Fix variable checking in v2 config parser (#6097) [686ff608]
+- Sort linter errors/warnings by source location (#6098) [f51f339c]
+
 24.10.8 - 6 Jun 2025
 - Add Fusion license validation [02b4dd74]
 - Bump [email protected] [cf6af9c5]
@@ -36,6 +42,11 @@ NEXTFLOW CHANGE-LOG
 - Bump [email protected] [0f6498f1]
 - Bump [email protected] [0b491840]
 
+25.04.3 - 2 Jun 2025
+- Add Platform info to Fusion license (#6142) [375db65a]
+- Force overwritting to trace file (#6105) [59e9d88d]
+- Bump [email protected] [f7509bce] 
+
 25.04.2 - 13 May 2025
 - Add check subcommand in lineage (#6074) [5ba67bca]
 - Fix issues with `lint` console output (#6064) [7405f513]

changelog.txt

@@ -2,81 +2,63 @@
 
 # Channels
 
-Nextflow is based on the dataflow programming model in which processes communicate through channels.
+In Nextflow, **channels** are the key data structures that facilitate the dataflow dependencies between each step (i.e. {ref}`process <process-page>`) in a pipeline.
 
-A channel has two major properties:
+There are two kinds of channels, *queue channels* and *value channels*. Channels are created using *channel factories* and transformed using *channel operators*.
 
-1. Sending a message is an *asynchronous* (i.e. non-blocking) operation, which means the sender doesn't have to wait for the receiving process.
-2. Receiving a message is a *synchronous* (i.e. blocking) operation, which means the receiving process must wait until a message has arrived.
-
-(channel-types)=
+(channel-type-queue)=
 
-## Channel types
+## Queue channels
 
-In Nextflow there are two kinds of channels: *queue channels* and *value channels*.
+A *queue channel* is a channel that *emits* an asynchronous sequence of values.
 
-(channel-type-queue)=
+A queue channel can be created by channel factories (e.g., {ref}`channel.of <channel-of>` and {ref}`channel.fromPath <channel-path>`), operators (e.g., {ref}`operator-map` and  {ref}`operator-filter`), and processes (see {ref}`Process outputs <process-output>`).
 
-### Queue channel
+The values in a queue channel cannot be accessed directly -- they can only be accessed by passing the channel as input to an operator or process. For example:
 
-A *queue channel* is a non-blocking unidirectional FIFO queue connecting a *producer* process (i.e. outputting a value)
-to a consumer process or an operator.
+```nextflow
+channel.of(1, 2, 3).view { v -> "queue channel emits ${v}" }
+```
 
-A queue channel can be created by factory methods ({ref}`channel-of`, {ref}`channel-path`, etc), operators ({ref}`operator-map`, {ref}`operator-flatmap`, etc), and processes (see {ref}`Process outputs <process-output>`).
+```console
+queue channel emits 1
+queue channel emits 2
+queue channel emits 3
+```
 
 (channel-type-value)=
 
-### Value channel
+## Value channels
 
-A *value channel* can be bound (i.e. assigned) with one and only one value, and can be consumed any number of times by
-a process or an operator.
+A *value channel* is a channel that is *bound* to an asynchronous value.
 
-A value channel can be created with the {ref}`channel-value` factory method or by any operator that produces a single value
-({ref}`operator-first`, {ref}`operator-collect`, {ref}`operator-reduce`, etc). Additionally, a process will emit value
-channels if it is invoked with all value channels, including simple values which are implicitly wrapped in a value channel.
+A value channel can be created with the {ref}`channel.value <channel-value>` factory, certain operators (e.g., {ref}`operator-collect` and {ref}`operator-reduce`), and processes (under {ref}`certain conditions <process-out-singleton>`).
 
-For example:
+The value in a value channel cannot be accessed directly -- it can only be accessed by passing the channel as input to an operator or process. For example:
 
 ```nextflow
-process echo {
-  input:
-  val x
-
-  output:
-  path 'x.txt'
-
-  script:
-  """
-  echo $x > x.txt
-  """
-}
-
-workflow {
-  result = echo(1)
-  result.view { file -> "Result: ${file}" }
-}
+channel.value(1).view { v -> "value channel is ${v}" }
 ```
 
-In the above example, since the `echo` process is invoked with a simple value instead of a channel, the input is implicitly
-wrapped in a value channel, and the output is also emitted as a value channel.
-
-See also: {ref}`process-multiple-input-channels`.
+```console
+value channel is 1
+```
 
 ## Channel factories
 
-Channel factories are functions that can create channels.
+Channel factories are functions that create channels from regular values.
 
-For example, the `channel.of()` factory can be used to create a channel from an arbitrary list of arguments:
+The `channel.fromPath()` factory creates a channel from a file name or glob pattern, similar to the `files()` function:
 
 ```nextflow
-channel.of(1, 2, 3).view()
+channel.fromPath('input/*.txt').view()
 ```
 
 See {ref}`channel-factory` for the full list of channel factories.
 
 ## Operators
 
-Channel operators, or _operators_ for short, are functions that consume and produce channels. Because channels are asynchronous, operators are necessary to manipulate the values in a channel, aside from using a process. As a result, operators are useful for implementing the _glue logic_ between processes.
+Channel operators, or *operators* for short, are functions that consume and produce channels. Because channels are asynchronous, operators are necessary to manipulate the values in a channel. Operators are particularly useful for implementing glue logic between processes.
 
 Commonly used operators include:
 

docs/channel.md

@@ -108,6 +108,9 @@ The following constants are globally available in a Nextflow configuration file:
 `projectDir: Path`
 : The directory where the main script is located.
 
+`secrets: Map<String,String>`
+: Map of pipeline secrets. See {ref}`secrets-page` for more information.
+
 ## Functions
 
 The following functions are globally available in a Nextflow configuration file:

docs/config.md

@@ -776,9 +776,9 @@ The process `echo` is executed two times because the `x` channel emits only two
 2 and b
 ```
 
-A different semantic is applied when using a {ref}`value channel <channel-type-value>`. This kind of channel is created by the {ref}`channel.value <channel-value>` factory method or implicitly when a process is invoked with an argument that is not a channel. By definition, a value channel is bound to a single value and it can be read an unlimited number of times without consuming its content. Therefore, when mixing a value channel with one or more (queue) channels, it does not affect the process termination because the underlying value is applied repeatedly.
+When a {ref}`value channel <channel-type-value>` is supplied as a process input alongside a queue channel, the process is executed for each value in the queue channel, and the value channel is re-used for each execution.
 
-To better understand this behavior, compare the previous example with the following one:
+For example, compare the previous example with the following:
 
 ```nextflow
 process echo {
@@ -811,7 +811,7 @@ The above example executes the `echo` process three times because `x` is a value
 In general, multiple input channels should be used to process *combinations* of different inputs, using the `each` qualifier or value channels. Having multiple queue channels as inputs is equivalent to using the {ref}`operator-merge` operator, which is not recommended as it may lead to {ref}`non-deterministic process inputs <cache-nondeterministic-inputs>`.
 :::
 
-See also: {ref}`channel-types`.
+See also: {ref}`process-out-singleton`.
 
 (process-output)=
 
@@ -1160,6 +1160,22 @@ In this example, the process is normally expected to produce an `output.txt` fil
 While this option can be used with any process output, it cannot be applied to individual elements of a [tuple](#output-tuples-tuple) output. The entire tuple must be optional or not optional.
 :::
 
+(process-out-singleton)=
+
+### Singleton outputs
+
+When a process is only supplied with value channels, regular values, or no inputs, it returns outputs as value channels. For example:
+
+```{literalinclude} snippets/process-out-singleton.nf
+:language: nextflow
+```
+
+In the above example, the `echo` process is invoked with a regular value that is wrapped in a value channel. As a result, `echo` returns a value channel and `greet` is executed three times.
+
+If the call to `echo` was changed to `echo( channel.of('hello') )`, the process would instead return a queue channel, and `greet` would be executed only once.
+
+See also: {ref}`process-multiple-input-channels`.
+
 (process-when)=
 
 ## When

docs/process.md

@@ -1282,9 +1282,11 @@ The `run` command is used to execute a local pipeline script or remote pipeline
 
   See {ref}`cli-params` for more information about writing custom parameters files.
 
+(cli-secrets)=
+
 ### `secrets`
 
-Manage pipeline secrets.
+Manage {ref}`pipeline secrets <secrets-page>`.
 
 **Usage**
 

docs/reference/cli.md

@@ -8,6 +8,9 @@ This page lists all of the available settings in the {ref}`Nextflow configuratio
 
 ## Unscoped options
 
+`bucketDir`
+: The remote work directory used by hybrid workflows. Equivalent to the `-bucket-dir` option of the `run` command.
+
 `cleanup`
 : If `true`, on a successful completion of a run all files in *work* directory are automatically deleted.
 
@@ -182,7 +185,7 @@ The following settings are available:
 
 `aws.batch.terminateUnschedulableJobs`
 : :::{versionadded} 25.03.0-edge
-:::
+  :::
 : When `true`, jobs that cannot be scheduled for lack of resources or misconfiguration are terminated automatically (default: `false`). The pipeline may complete with an error status depending on the error strategy defined for the corresponding jobs.
 
 `aws.batch.volumes`
@@ -1246,10 +1249,26 @@ Read the {ref}`sharing-page` page to learn how to publish your pipeline to GitHu
 
 ## `nextflow`
 
-:::{deprecated} 24.10.0
-The `nextflow.publish` scope has been renamed to `workflow.output`. See {ref}`config-workflow` for more information.
+:::{versionchanged} 24.10.0
+The `nextflow.publish.retryPolicy` settings were moved to `workflow.output.retryPolicy`.
 :::
 
+:::{versionchanged} 25.06.0-edge
+The `workflow.output.retryPolicy` settings were moved to `nextflow.retryPolicy`.
+:::
+
+`retryPolicy.delay`
+: Delay used for retryable operations (default: `350ms`).
+
+`retryPolicy.jitter`
+: Jitter value used for retryable operations (default: `0.25`).
+
+`retryPolicy.maxAttempts`
+: Max attempts used for retryable operations (default: `5`).
+
+`retryPolicy.maxDelay`
+: Max delay used for retryable operations (default: `90s`).
+
 (config-notification)=
 
 ## `notification`
@@ -1589,7 +1608,7 @@ The following settings are available:
 `wave.retryPolicy.delay`
 : :::{versionadded} 22.06.0-edge
   :::
-: The initial delay when a failing HTTP request is retried (default: `150ms`).
+: The initial delay when a failing HTTP request is retried (default: `450ms`).
 
 `wave.retryPolicy.jitter`
 : :::{versionadded} 22.06.0-edge
@@ -1604,7 +1623,7 @@ The following settings are available:
 `wave.retryPolicy.maxDelay`
 : :::{versionadded} 22.06.0-edge
   :::
-: The max delay when a failing HTTP request is retried (default: `90 seconds`).
+: The max delay when a failing HTTP request is retried (default: `90s`).
 
 `wave.scan.mode`
 : :::{versionadded} 24.09.1-edge
@@ -1705,18 +1724,6 @@ The `workflow` scope provides workflow execution options.
   `'standard'`
   : Overwrite existing files when the file size or last modified timestamp is different.
 
-`workflow.output.retryPolicy.delay`
-: Delay when retrying a failed publish operation (default: `350ms`).
-
-`workflow.output.retryPolicy.jitter`
-: Jitter value when retrying a failed publish operation (default: `0.25`).
-
-`workflow.output.retryPolicy.maxAttempt`
-: Max attempts when retrying a failed publish operation (default: `5`).
-
-`workflow.output.retryPolicy.maxDelay`
-: Max delay when retrying a failed publish operation (default: `90s`).
-
 `workflow.output.storageClass`
 : *Currently only supported for S3.*
 : Specify the storage class for published files.