Skip to content

Commit df9c535

Browse files
swingingsimianNathan Johnsonpditommaso
committed
Fix readBytes for non-GitHub providers (#6243) [ci fast]
Signed-off-by: Paolo Di Tommaso <[email protected]> Co-authored-by: Nathan Johnson <[email protected]> Co-authored-by: Paolo Di Tommaso <[email protected]>
1 parent 46ff6f0 commit df9c535

13 files changed

+123
-66
lines changed

‎modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,8 +159,7 @@ final class BitbucketRepositoryProvider extends RepositoryProvider {
159159

160160
@Override
161161
byte[] readBytes(String path) {
162-
163-
def url = getContentUrl(path)
164-
invoke(url)?.getBytes()
162+
final url = getContentUrl(path)
163+
return invokeBytes(url)
165164
}
166165
}

‎modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,8 @@ final class BitbucketServerRepositoryProvider extends RepositoryProvider {
107107

108108
@Override
109109
byte[] readBytes(String path) {
110-
def url = getContentUrl(path)
111-
invoke(url)?.getBytes()
110+
final url = getContentUrl(path)
111+
return invokeBytes(url)
112112
}
113113

114114
@Override

‎modules/nextflow/src/main/groovy/nextflow/scm/GiteaRepositoryProvider.groovy

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,8 @@ final class GiteaRepositoryProvider extends RepositoryProvider {
102102
/** {@inheritDoc} */
103103
@Override
104104
byte[] readBytes(String path) {
105-
106105
def url = getContentUrl(path)
107-
def contents = invoke(url)
108-
return contents?.getBytes()
106+
return invokeBytes(url)
109107
}
110108

111109
}

‎modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,9 @@ class GithubRepositoryProvider extends RepositoryProvider {
112112
/** {@inheritDoc} */
113113
@Override
114114
byte[] readBytes(String path) {
115-
116115
def url = getContentUrl(path)
117116
Map response = invokeAndParseResponse(url)
118117
response.get('content')?.toString()?.decodeBase64()
119-
120118
}
121119

122120
}

‎modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,10 +113,8 @@ class GitlabRepositoryProvider extends RepositoryProvider {
113113
/** {@inheritDoc} */
114114
@Override
115115
byte[] readBytes(String path) {
116-
117116
def url = getContentUrl(path)
118117
Map response = invokeAndParseResponse(url)
119118
response.get('content')?.toString()?.decodeBase64()
120-
121119
}
122120
}

‎modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -224,14 +224,25 @@ abstract class RepositoryProvider {
224224
* @return The remote service response as a text
225225
*/
226226
protected String invoke( String api ) {
227+
final result = invokeBytes(api)
228+
return result!=null ? new String(result) : null
229+
}
230+
231+
/**
232+
* Invoke the API request specified and return binary content
233+
*
234+
* @param api A API request url e.g. https://api.github.com/repos/nextflow-io/hello/raw/image.png
235+
* @return The remote service response as byte array
236+
*/
237+
protected byte[] invokeBytes( String api ) {
227238
assert api
228239
log.debug "Request [credentials ${getAuthObfuscated() ?: '-'}] -> $api"
229240
final request = newRequest(api)
230241
// submit the request
231-
final HttpResponse<String> resp = httpSend(request)
242+
final HttpResponse<byte[]> resp = httpSend0(request)
232243
// check the response code
233244
checkResponse(resp)
234-
// return the body as string
245+
// return the body as byte array
235246
return resp.body()
236247
}
237248

@@ -263,7 +274,7 @@ abstract class RepositoryProvider {
263274
*
264275
* @param response A {@link HttpURLConnection} response instance
265276
*/
266-
protected checkResponse( HttpResponse<String> response ) {
277+
protected checkResponse( HttpResponse<?> response ) {
267278
final code = response.statusCode()
268279
if( code==401 ) {
269280
log.debug "Response status: $code -- ${response.body()}"
@@ -447,6 +458,7 @@ abstract class RepositoryProvider {
447458
return isCausedByUnresolvedAddressException(t.cause)
448459
}
449460

461+
@Deprecated
450462
protected HttpResponse<String> httpSend(HttpRequest request) {
451463
if( httpClient==null )
452464
httpClient = newHttpClient()
@@ -460,10 +472,24 @@ abstract class RepositoryProvider {
460472
}
461473
}
462474

475+
private HttpResponse<byte[]> httpSend0(HttpRequest request) {
476+
if( httpClient==null )
477+
httpClient = newHttpClient()
478+
if( retryConfig==null )
479+
retryConfig = new RetryConfig()
480+
try {
481+
safeApply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()))
482+
}
483+
catch (FailsafeException e) {
484+
throw e.cause
485+
}
486+
}
487+
463488
private HttpClient newHttpClient() {
464489
final builder = HttpClient.newBuilder()
465490
.version(HttpClient.Version.HTTP_1_1)
466491
.connectTimeout(Duration.ofSeconds(60))
492+
.followRedirects(HttpClient.Redirect.NORMAL)
467493
// use virtual threads executor if enabled
468494
if( Threads.useVirtual() )
469495
builder.executor(Executors.newVirtualThreadPerTaskExecutor())

‎modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ class AzureRepositoryProviderTest extends Specification {
5151
't-neumann/hello' | ['t-neumann', 'hello', 'hello']
5252
'ORGANIZATION/PROJECT/hello' | ['ORGANIZATION','PROJECT','hello']
5353
'ORGANIZATION/PROJECT/_git/hello' | ['ORGANIZATION','PROJECT','hello']
54-
5554
}
5655

5756
def 'should throw exception if wrong path' () {
@@ -69,7 +68,6 @@ class AzureRepositoryProviderTest extends Specification {
6968
}
7069

7170
def 'should return repo url' () {
72-
7371
given:
7472
def config = new ConfigSlurper().parse(CONFIG)
7573
def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject)
@@ -85,7 +83,6 @@ class AzureRepositoryProviderTest extends Specification {
8583
}
8684

8785
def 'should return project URL' () {
88-
8986
given:
9087
def config = new ConfigSlurper().parse(CONFIG)
9188
def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject)
@@ -97,29 +94,24 @@ class AzureRepositoryProviderTest extends Specification {
9794
't-neumann/hello' | 'https://dev.azure.com/t-neumann/hello'
9895
'ORGANIZATION/PROJECT/hello' | 'https://dev.azure.com/ORGANIZATION/PROJECT/hello'
9996
'ORGANIZATION/PROJECT/_git/hello' | 'https://dev.azure.com/ORGANIZATION/PROJECT/_git/hello'
100-
10197
}
10298

10399
def 'should return content URL' () {
104-
105100
given:
106101
def config = new ConfigSlurper().parse(CONFIG)
107102
def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject)
108103

109104
expect:
110105
new AzureRepositoryProvider('t-neumann/hello', obj).getContentUrl('main.nf') == 'https://dev.azure.com/t-neumann/hello/_apis/git/repositories/hello/items?download=false&includeContent=true&includeContentMetadata=false&api-version=6.0&\$format=json&path=main.nf'
111-
112106
}
113107

114108
def 'should return content URL for revision' () {
115-
116109
given:
117110
def config = new ConfigSlurper().parse(CONFIG)
118111
def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject)
119112

120113
expect:
121114
new AzureRepositoryProvider('t-neumann/hello', obj).setRevision("a-branch").getContentUrl('main.nf') == 'https://dev.azure.com/t-neumann/hello/_apis/git/repositories/hello/items?download=false&includeContent=true&includeContentMetadata=false&api-version=6.0&\$format=json&path=main.nf&versionDescriptor.version=a-branch'
122-
123115
}
124116

125117
/*
@@ -141,7 +133,6 @@ class AzureRepositoryProviderTest extends Specification {
141133
def result = repo.readText('main.nf')
142134
then:
143135
result == 'println "Hello from Azure repos!"'
144-
145136
}
146137

147138
@IgnoreIf({System.getenv('NXF_SMOKE')})

‎modules/nextflow/src/test/groovy/nextflow/scm/BitbucketRepositoryProviderTest.groovy

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ class BitbucketRepositoryProviderTest extends Specification {
2828

2929
@Requires( { System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') } )
3030
def testBitbucketCloneURL() {
31-
3231
given:
3332
def token = System.getenv('NXF_BITBUCKET_ACCESS_TOKEN')
3433
def config = new ProviderConfig('bitbucket').setAuth(token)
@@ -39,16 +38,13 @@ class BitbucketRepositoryProviderTest extends Specification {
3938
url == "https://${config.user}@bitbucket.org/pditommaso/tutorial.git".toString()
4039
}
4140

42-
4341
def testGetHomePage() {
4442
expect:
4543
new BitbucketRepositoryProvider('pditommaso/tutorial').getRepositoryUrl() == "https://bitbucket.org/pditommaso/tutorial"
4644
}
4745

48-
4946
@Requires( { System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') } )
5047
def testReadContent() {
51-
5248
given:
5349
def token = System.getenv('NXF_BITBUCKET_ACCESS_TOKEN')
5450
def config = new ProviderConfig('bitbucket').setAuth(token)
@@ -59,7 +55,21 @@ class BitbucketRepositoryProviderTest extends Specification {
5955

6056
then:
6157
result.trim().startsWith('#!/usr/bin/env nextflow')
58+
}
6259

60+
@Requires( { System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') } )
61+
def 'should read binary data'() {
62+
given:
63+
def token = System.getenv('NXF_BITBUCKET_ACCESS_TOKEN')
64+
def config = new ProviderConfig('bitbucket').setAuth(token)
65+
def DATA = this.class.getResourceAsStream('/test-sandbucket.jpg').bytes
66+
67+
when:
68+
def repo = new BitbucketRepositoryProvider('pditommaso/tutorial', config)
69+
def result = repo.readBytes('sandbucket.jpg')
70+
71+
then:
72+
result == DATA
6373
}
6474

6575
@Requires( { System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') } )
@@ -160,6 +170,5 @@ class BitbucketRepositoryProviderTest extends Specification {
160170
then:
161171
!data.contains('world')
162172
data.contains('mundo')
163-
164173
}
165174
}

‎modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package nextflow.scm
1919
import spock.lang.IgnoreIf
2020
import spock.lang.Requires
2121
import spock.lang.Specification
22-
2322
/**
2423
*
2524
* @author Akira Sekiguchi <[email protected]>
@@ -28,76 +27,83 @@ class GiteaRepositoryProviderTest extends Specification {
2827

2928
static final String CONFIG = '''
3029
providers {
31-
3230
mygitea {
33-
server = 'https://git.seqera.io'
34-
endpoint = 'https://git.seqera.io/api/v1'
31+
server = 'https://gitea.com'
32+
endpoint = 'https://gitea.com/api/v1'
3533
platform = 'gitea'
36-
user = 'myname'
37-
password = 'mypassword'
3834
}
39-
4035
}
4136
'''
4237

4338
def 'should return repo url' () {
44-
4539
given:
4640
def config = new ConfigSlurper().parse(CONFIG)
4741
def obj = new ProviderConfig('gitea', config.providers.mygitea as ConfigObject)
4842

4943
expect:
50-
new GiteaRepositoryProvider('pditommaso/hello', obj).getEndpointUrl() == 'https://git.seqera.io/api/v1/repos/pditommaso/hello'
44+
new GiteaRepositoryProvider('pditommaso/hello', obj).getEndpointUrl() == 'https://gitea.com/api/v1/repos/pditommaso/hello'
5145
}
5246

5347
def 'should return project URL' () {
54-
5548
given:
5649
def config = new ConfigSlurper().parse(CONFIG)
5750
def obj = new ProviderConfig('gitea', config.providers.mygitea as ConfigObject)
5851

5952
expect:
60-
new GiteaRepositoryProvider('pditommaso/hello', obj).getRepositoryUrl() == 'https://git.seqera.io/pditommaso/hello'
61-
53+
new GiteaRepositoryProvider('pditommaso/hello', obj).getRepositoryUrl() == 'https://gitea.com/pditommaso/hello'
6254
}
6355

6456
def 'should return content URL' () {
65-
6657
given:
6758
def config = new ConfigSlurper().parse(CONFIG)
6859
def obj = new ProviderConfig('gitea', config.providers.mygitea as ConfigObject)
6960

7061
expect:
7162
new GiteaRepositoryProvider('pditommaso/hello', obj)
72-
.getContentUrl('main.nf') == 'https://git.seqera.io/api/v1/repos/pditommaso/hello/raw/main.nf'
63+
.getContentUrl('main.nf') == 'https://gitea.com/api/v1/repos/pditommaso/hello/raw/main.nf'
7364
and:
7465
new GiteaRepositoryProvider('pditommaso/hello', obj)
7566
.setRevision('12345')
76-
.getContentUrl('main.nf') == 'https://git.seqera.io/api/v1/repos/pditommaso/hello/raw/main.nf?ref=12345'
67+
.getContentUrl('main.nf') == 'https://gitea.com/api/v1/repos/pditommaso/hello/raw/main.nf?ref=12345'
7768

7869
}
7970

8071
@IgnoreIf({System.getenv('NXF_SMOKE')})
8172
@Requires({System.getenv('NXF_GITEA_ACCESS_TOKEN')})
8273
def 'should read file content'() {
83-
8474
given:
85-
def token = System.getenv('NXF_GITEA_ACCESS_TOKEN')
86-
def config = new ProviderConfig('gitea', new ConfigSlurper().parse(CONFIG).providers.mygitea as ConfigObject).setAuth(token)
75+
def token = System.getenv('NXF_GITEA_ACCESS_TOKEN')
76+
def config = new ProviderConfig('gitea', new ConfigSlurper().parse(CONFIG).providers.mygitea as ConfigObject) .setAuth(token)
8777

8878
when:
89-
def repo = new GiteaRepositoryProvider('test-org/nextflow-ci-repo', config)
79+
def repo = new GiteaRepositoryProvider('pditommaso/test-hello', config)
9080
def result = repo.readText('README.md')
9181
then:
92-
result.contains('nextflow-ci-repo')
82+
result.contains('Basic Nextflow script')
9383

9484
// when:
9585
// repo = new GiteaRepositoryProvider('test-org/nextflow-ci-repo', config)
9686
// .setRevision('foo')
9787
// result = repo.readText('README.md')
9888
// then:
9989
// result.contains("foo branch")
90+
}
91+
92+
@IgnoreIf({System.getenv('NXF_SMOKE')})
93+
@Requires({System.getenv('NXF_GITEA_ACCESS_TOKEN')})
94+
def 'should read bytes gitea content'() {
95+
given:
96+
def token = System.getenv('NXF_GITEA_ACCESS_TOKEN')
97+
def config = new ProviderConfig('gitea', new ConfigSlurper().parse(CONFIG).providers.mygitea as ConfigObject) .setAuth(token)
98+
def repo = new GiteaRepositoryProvider('pditommaso/test-hello', config)
99+
and:
100+
def DATA = this.class.getResourceAsStream('/test-asset.bin').bytes
101+
102+
when:
103+
def result = repo.readBytes('test/test-asset.bin')
100104

105+
then:
106+
result == DATA
101107
}
102108

103109
}

0 commit comments

Comments
 (0)