Upgrade to AWS Java SDK v2

docs/reference/config.md

@@ -216,13 +216,36 @@ The following settings are available:
   :::
 : The retrieval tier to use when restoring objects from Glacier, one of [`Expedited`, `Standard`, `Bulk`].
 
+`aws.client.maxConcurrency`
+: :::{versionadded} 25.06.0-edge
+  :::
+: The maximum number of concurrent S3 transfers when using the asynchronous S3 client. By default, this setting is determined by `aws.client.targetThroughputInGbps`. Modifying this value can affect the amount of memory used for S3 transfers.
+
 `aws.client.maxConnections`
-: The maximum number of allowed open HTTP connections (default: `50`).
+: The maximum number of allowed open HTTP connections when using the synchronous S3 client (default: `50`).
 
 `aws.client.maxErrorRetry`
 : The maximum number of retry attempts for failed retryable requests (default: `-1`).
 
+`aws.client.maxNativeMemory`
+: :::{versionadded} 25.06.0-edge
+  :::
+: The maximum native memory for S3 transfers when using the asynchronous S3 client. By default, this setting is determined by `aws.client.targetThroughputInGbps`. Modifying this value can affect the memory used by the S3 transfers.
+
+`aws.client.minimumPartSize`
+: :::{versionadded} 25.06.0-edge
+  :::
+: The minimum part size for multi-part uploads when using the asynchronous S3 client (default: `8 MB`).
+
+`aws.client.multipartThreshold`
+: :::{versionadded} 25.06.0-edge
+  :::
+: The object size threshold for performing multi-part uploads when using the asynchronous S3 client (default: same as `aws.cllient.minimumPartSize`).
+
 `aws.client.protocol`
+: :::{deprecated} 25.06.0-edge
+  This option is no longer supported.
+  :::
 : The protocol to use when connecting to AWS. Can be `http` or `https` (default: `'https'`).
 
 `aws.client.proxyHost`
@@ -231,6 +254,11 @@ The following settings are available:
 `aws.client.proxyPort`
 : The port to use when connecting through a proxy.
 
+`aws.client.proxyScheme`
+: :::{versionadded} 25.06.0-edge
+  :::
+: The protocol scheme to use when connecting through a proxy. Can be `http` or `https` (default: `'http'`).
+
 `aws.client.proxyUsername`
 : The user name to use when connecting through a proxy.
 
@@ -240,18 +268,27 @@ The following settings are available:
 `aws.client.requesterPays`
 : :::{versionadded} 24.05.0-edge
   :::
-: Use [Rrequester Pays](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html) for S3 buckets (default: `false`).
+: Use [Requester Pays](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html) for S3 buckets (default: `false`).
 
 `aws.client.s3PathStyleAccess`
 : Use the path-based access model to access objects in S3-compatible storage systems (default: `false`).
 
 `aws.client.signerOverride`
+: :::{deprecated} 25.06.0-edge
+  This option is no longer supported.
+  :::
 : The name of the signature algorithm to use for signing requests made by the client.
 
 `aws.client.socketSendBufferSizeHint`
+: :::{deprecated} 25.06.0-edge
+  This option is no longer supported.
+  :::
 : The Size hint (in bytes) for the low level TCP send buffer (default: `0`).
 
 `aws.client.socketRecvBufferSizeHint`
+: :::{deprecated} 25.06.0-edge
+  This option is no longer supported.
+  :::
 : The Size hint (in bytes) for the low level TCP receive buffer (default: `0`).
 
 `aws.client.socketTimeout`
@@ -265,20 +302,45 @@ The following settings are available:
   :::
 : The AWS KMS key Id to be used to encrypt files stored in the target S3 bucket.
 
+`aws.client.targetThroughputInGbps`
+: :::{versionadded} 25.06.0-edge
+  :::
+: The target network throughput (in Gbps) when using the asynchronous S3 client (default: `10`). This setting is not used when `aws.client.maxConcurrency` and `aws.client.maxNativeMemory` are specified.
+
+`aws.client.transferManagerThreads`
+: :::{versionadded} 25.06.0-edge
+  :::
+: Number of threads used by the S3 transfer manager (default `10`).
+
 `aws.client.userAgent`
+: :::{deprecated} 25.06.0-edge
+  This option is no longer supported.
+  :::
 : The HTTP user agent header passed with all HTTP requests.
 
 `aws.client.uploadChunkSize`
 : The size of a single part in a multipart upload (default: `100 MB`).
+: :::{versionchanged} 25.06.0-edge
+  This option only applies when uploading via `Files.newOutputStream()`. Other uploads are managed by the S3 transfer manager.
+  :::
 
 `aws.client.uploadMaxAttempts`
 : The maximum number of upload attempts after which a multipart upload returns an error (default: `5`).
+: :::{versionchanged} 25.06.0-edge
+  This option only applies when uploading via `Files.newOutputStream()`. Other uploads are managed by the S3 transfer manager.
+  :::
 
 `aws.client.uploadMaxThreads`
 : The maximum number of threads used for multipart upload (default: `10`).
+: :::{versionchanged} 25.06.0-edge
+  This option only applies when uploading via `Files.newOutputStream()`. Other uploads are managed by the S3 transfer manager.
+  :::
 
 `aws.client.uploadRetrySleep`
 : The time to wait after a failed upload attempt to retry the part upload (default: `500ms`).
+: :::{versionchanged} 25.06.0-edge
+  This option only applies when uploading via `Files.newOutputStream()`. Other uploads are managed by the S3 transfer manager.
+  :::
 
 `aws.client.uploadStorageClass`
 : The S3 storage class applied to stored objects. Can be `STANDARD`, `STANDARD_IA`, `ONEZONE_IA`, or `INTELLIGENT_TIERING` (default: `STANDARD`).

modules/nf-commons/src/main/nextflow/file/FileHelper.groovy

@@ -880,7 +880,9 @@ class FileHelper {
 
             @Override
             FileVisitResult visitFile(Path fullPath, BasicFileAttributes attrs) throws IOException {
-                final path = folder.relativize(fullPath)
+                final path = fullPath.isAbsolute()
+                    ? folder.relativize(fullPath)
+                    : fullPath
                 log.trace "visitFiles > file=$path; includeFile=$includeFile; matches=${matcher.matches(path)}; isRegularFile=${attrs.isRegularFile()}"
 
                 if (includeFile && matcher.matches(path) && (attrs.isRegularFile() || (options.followLinks == false && attrs.isSymbolicLink())) && (includeHidden || !isHidden(fullPath))) {
@@ -912,7 +914,9 @@ class FileHelper {
     }
 
     static protected Path relativize0(Path folder, Path fullPath) {
-        def result = folder.relativize(fullPath)
+        final result = fullPath.isAbsolute()
+            ? folder.relativize(fullPath)
+            : fullPath
         String str
         if( folder.is(FileSystems.default) || !(str=result.toString()).endsWith('/') )
             return result

modules/nf-lang/src/main/java/nextflow/config/scopes/AwsClientConfig.java

@@ -49,6 +49,12 @@ The amount of time to wait (in milliseconds) when initially establishing a conne
     """)
     public String endpoint;
 
+    @ConfigOption
+    @Description("""
+        The maximum number of concurrency in S3 async clients.
+    """)
+    public int maxConcurrency;
+
     @ConfigOption
     @Description("""
         The maximum number of allowed open HTTP connections.
@@ -61,6 +67,24 @@ The amount of time to wait (in milliseconds) when initially establishing a conne
     """)
     public int maxErrorRetry;
 
+    @ConfigOption
+    @Description("""
+        The maximum native memory used by the S3 asynchronous client for S3 transfers.
+    """)
+    public MemoryUnit maxNativeMemory;
+
+    @ConfigOption
+    @Description("""
+        The minimum size of a single part in a multipart upload (default: `8 MB`).
+    """)
+    public MemoryUnit minimumPartSize;
+
+    @ConfigOption
+    @Description("""
+        The S3 Async client threshold to create multipart S3 transfers. Default is the same as `minimumPartSize`.
+    """)
+    public MemoryUnit multipartThreshold;
+
     @ConfigOption
     @Description("""
         The protocol (i.e. HTTP or HTTPS) to use when connecting to AWS.
@@ -79,6 +103,12 @@ The protocol (i.e. HTTP or HTTPS) to use when connecting to AWS.
     """)
     public int proxyPort;
 
+    @ConfigOption
+    @Description("""
+        The protocol scheme to use when connecting through a proxy (http/https).
+    """)
+    public String proxyScheme;
+
     @ConfigOption
     @Description("""
         The user name to use when connecting through a proxy.
@@ -139,6 +169,18 @@ The amount of time to wait (in milliseconds) for data to be transferred over an
     """)
     public String storageKmsKeyId;
 
+    @ConfigOption
+    @Description("""
+        The S3 Async client target network throughput in Gbps. This value is used to automatically set `maxConcurrency` and `maxNativeMemory` (default: `10`).
+    """)
+    public Double targetThroughputInGbps;
+
+    @ConfigOption
+    @Description("""
+        The number of threads used by the S3 transfer manager (default: `10`).
+    """)
+    public int transferManagerThreads;
+
     @ConfigOption
     @Description("""
         The HTTP user agent header passed with all HTTP requests.

plugins/nf-amazon/build.gradle

@@ -38,17 +38,20 @@ dependencies {
     compileOnly 'org.pf4j:pf4j:3.12.0'
 
     api ('javax.xml.bind:jaxb-api:2.4.0-b180830.0359')
-    api ('com.amazonaws:aws-java-sdk-s3:1.12.777')
-    api ('com.amazonaws:aws-java-sdk-ec2:1.12.777')
-    api ('com.amazonaws:aws-java-sdk-batch:1.12.777')
-    api ('com.amazonaws:aws-java-sdk-iam:1.12.777')
-    api ('com.amazonaws:aws-java-sdk-ecs:1.12.777')
-    api ('com.amazonaws:aws-java-sdk-logs:1.12.777')
-    api ('com.amazonaws:aws-java-sdk-codecommit:1.12.777')
-    api ('com.amazonaws:aws-java-sdk-sts:1.12.777')
-    api ('com.amazonaws:aws-java-sdk-ses:1.12.777')
-    api ('software.amazon.awssdk:sso:2.26.26')
-    api ('software.amazon.awssdk:ssooidc:2.26.26')
+    api ('software.amazon.awssdk:s3:2.31.64')
+    api ('software.amazon.awssdk:ec2:2.31.64')
+    api ('software.amazon.awssdk:batch:2.31.64')
+    api ('software.amazon.awssdk:iam:2.31.64')
+    api ('software.amazon.awssdk:ecs:2.31.64')
+    api ('software.amazon.awssdk:cloudwatchlogs:2.31.64')
+    api ('software.amazon.awssdk:codecommit:2.31.64')
+    api ('software.amazon.awssdk:sts:2.31.64')
+    api ('software.amazon.awssdk:ses:2.31.64')
+    api ('software.amazon.awssdk:sso:2.31.64')
+    api ('software.amazon.awssdk:ssooidc:2.31.64')
+    api ('software.amazon.awssdk:s3-transfer-manager:2.31.64')
+    api ('software.amazon.awssdk:apache-client:2.31.64')
+    api ('software.amazon.awssdk:aws-crt-client:2.31.64')
 
     constraints {
         api 'com.fasterxml.jackson.core:jackson-databind:2.12.7.1'

plugins/nf-amazon/src/main/nextflow/cloud/aws/AmazonPlugin.groovy

@@ -35,8 +35,6 @@ class AmazonPlugin extends BasePlugin {
     @Override
     void start() {
         super.start()
-        // disable aws sdk v1 warning
-        System.setProperty("aws.java.v1.disableDeprecationAnnouncement", "true")
         FileHelper.getOrInstallProvider(S3FileSystemProvider)
     }