Skip to content

Add git response max length #6190

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 6, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright 2013-2025, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package nextflow.exception

import groovy.transform.CompileStatic
import groovy.transform.InheritConstructors

/**
* Exception thrown when the http response length exceed
* the max allowed size.
*
* @author Paolo Di Tommaso <[email protected]>
*/
@CompileStatic
@InheritConstructors
class HttpResponseLengthExceedException extends IOException {
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ import groovy.transform.CompileStatic
import groovy.transform.Memoized
import groovy.util.logging.Slf4j
import nextflow.Const
import nextflow.SysEnv
import nextflow.exception.AbortOperationException
import nextflow.exception.HttpResponseLengthExceedException
import nextflow.exception.RateLimitExceededException
import nextflow.util.RetryConfig
import nextflow.util.Threads
Expand Down Expand Up @@ -242,6 +244,7 @@ abstract class RepositoryProvider {
final HttpResponse<byte[]> resp = httpSend0(request)
// check the response code
checkResponse(resp)
checkMaxLength(resp)
// return the body as byte array
return resp.body()
}
Expand Down Expand Up @@ -310,6 +313,17 @@ abstract class RepositoryProvider {
}
}

protected void checkMaxLength(HttpResponse<byte[]> response) {
final max = SysEnv.getLong("NXF_GIT_RESPONSE_MAX_LENGTH", 0)
if( max<=0 )
return
final length = response.headers().firstValueAsLong('Content-Length').orElse(0)
if( length<=0 )
return
if( length>max )
throw new HttpResponseLengthExceedException("HTTP response '${response.uri()}' is too big - response length: ${length}; max allowed length: ${max}")
}

protected <T> List<T> invokeAndResponseWithPaging(String request, Closure<T> parse) {
// this is needed because apparently bytebuddy used by testing framework is not able
// to handle properly this method signature using both generics and `@Memoized` annotation.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,13 @@
package nextflow.scm

import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.nio.channels.UnresolvedAddressException
import javax.net.ssl.SSLSession

import nextflow.SysEnv
import nextflow.exception.HttpResponseLengthExceedException
import nextflow.util.RetryConfig
import spock.lang.Specification
/**
Expand Down Expand Up @@ -169,4 +173,82 @@ class RepositoryProviderTest extends Specification {
false | new SocketTimeoutException()
}

def 'should not validate when max length is not configured via SysEnv' () {
given:
def provider = Spy(RepositoryProvider)
def response = createMockResponseWithContentLength(1500)

when:
SysEnv.push([:])
provider.checkMaxLength(response)
then:
noExceptionThrown()

cleanup:
SysEnv.pop()
}

def 'should validate and pass when content is within limit via SysEnv' () {
given:
def provider = Spy(RepositoryProvider)
def response = createMockResponseWithContentLength(1500)

when:
SysEnv.push(['NXF_GIT_RESPONSE_MAX_LENGTH': '2000'])
provider.checkMaxLength(response)
then:
noExceptionThrown()

cleanup:
SysEnv.pop()
}

def 'should validate and fail when content exceeds limit via SysEnv' () {
given:
def provider = Spy(RepositoryProvider)
def response = createMockResponseWithContentLength(1500)

when:
SysEnv.push(['NXF_GIT_RESPONSE_MAX_LENGTH': '1000'])
provider.checkMaxLength(response)
then:
thrown(HttpResponseLengthExceedException)

cleanup:
SysEnv.pop()
}

private createMockResponseWithContentLength(long contentLength) {
return new HttpResponse<byte[]>() {
@Override
int statusCode() { return 200 }

@Override
HttpRequest request() { return null }

@Override
Optional<HttpResponse<byte[]>> previousResponse() { return Optional.empty() }

@Override
java.net.http.HttpHeaders headers() {
return java.net.http.HttpHeaders.of(
['Content-Length': [contentLength.toString()]],
(a, b) -> true
)
}

@Override
byte[] body() { return new byte[0] }

@Override
Optional<SSLSession> sslSession() { return Optional.empty() }

@Override
URI uri() { return new URI('https://api.github.com/repos/test/repo') }

@Override
HttpClient.Version version() { return HttpClient.Version.HTTP_1_1 }
}
}

}
10 changes: 10 additions & 0 deletions modules/nf-commons/src/main/nextflow/SysEnv.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,16 @@ class SysEnv {
return Boolean.parseBoolean(result)
}

static Integer getInteger(String name, Integer defValue) {
final result = get(name, defValue!=null ? String.valueOf(defValue) : null)
return result!=null ? Integer.valueOf(result) : null
}

static Long getLong(String name, Long defValue) {
final result = get(name, defValue!=null ? String.valueOf(defValue) : null)
return result!=null ? Long.valueOf(result) : null
}

static void push(Map<String,String> env) {
history.push(holder.getTarget())
holder.setTarget(env)
Expand Down
45 changes: 45 additions & 0 deletions modules/nf-commons/src/test/nextflow/SysEnvTest.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,51 @@ class SysEnvTest extends Specification {
[:] | true | true
[FOO:'false'] | false | false
[FOO:'true'] | true | true
}

@Unroll
def 'should get integer value' () {
given:
SysEnv.push(STATE)

expect:
SysEnv.getInteger('FOO', DEF) == EXPECTED

where:
STATE | DEF | EXPECTED
[:] | null | null
[FOO:'0'] | null | 0
[FOO:'1'] | null | 1
and:
[:] | 0 | 0
[FOO:'0'] | 0 | 0
[FOO:'100'] | 0 | 100
and:
[:] | 1 | 1
[FOO:'0'] | 1 | 0
[FOO:'100'] | 1 | 100
}

@Unroll
def 'should get long value' () {
given:
SysEnv.push(STATE)

expect:
SysEnv.getLong('FOO', DEF) == EXPECTED

where:
STATE | DEF | EXPECTED
[:] | null | null
[FOO:'0'] | null | 0
[FOO:'1'] | null | 1
and:
[:] | 0 | 0
[FOO:'0'] | 0 | 0
[FOO:'100'] | 0 | 100
and:
[:] | 1 | 1
[FOO:'0'] | 1 | 0
[FOO:'100'] | 1 | 100
}
}