diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy index dcc9f7579b..97e392cb96 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy @@ -159,8 +159,7 @@ final class BitbucketRepositoryProvider extends RepositoryProvider { @Override byte[] readBytes(String path) { - - def url = getContentUrl(path) - invoke(url)?.getBytes() + final url = getContentUrl(path) + return invokeBytes(url) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy index d7627bf6e5..0bc0220bd4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy @@ -107,8 +107,8 @@ final class BitbucketServerRepositoryProvider extends RepositoryProvider { @Override byte[] readBytes(String path) { - def url = getContentUrl(path) - invoke(url)?.getBytes() + final url = getContentUrl(path) + return invokeBytes(url) } @Override diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/GiteaRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/GiteaRepositoryProvider.groovy index 5766cbb0d2..65be6c04f8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/GiteaRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/GiteaRepositoryProvider.groovy @@ -102,10 +102,8 @@ final class GiteaRepositoryProvider extends RepositoryProvider { /** {@inheritDoc} */ @Override byte[] readBytes(String path) { - def url = getContentUrl(path) - def contents = invoke(url) - return contents?.getBytes() + return invokeBytes(url) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy index 55c71a74db..f48bcd5341 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy @@ -112,11 +112,9 @@ class GithubRepositoryProvider extends RepositoryProvider { /** {@inheritDoc} */ @Override byte[] readBytes(String path) { - def url = getContentUrl(path) Map response = invokeAndParseResponse(url) response.get('content')?.toString()?.decodeBase64() - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy index e90d0b0603..0f57cc4988 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy @@ -113,10 +113,8 @@ class GitlabRepositoryProvider extends RepositoryProvider { /** {@inheritDoc} */ @Override byte[] readBytes(String path) { - def url = getContentUrl(path) Map response = invokeAndParseResponse(url) response.get('content')?.toString()?.decodeBase64() - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy index 4cd52ef9d3..50f0b86d2b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy @@ -224,14 +224,25 @@ abstract class RepositoryProvider { * @return The remote service response as a text */ protected String invoke( String api ) { + final result = invokeBytes(api) + return result!=null ? new String(result) : null + } + + /** + * Invoke the API request specified and return binary content + * + * @param api A API request url e.g. https://api.github.com/repos/nextflow-io/hello/raw/image.png + * @return The remote service response as byte array + */ + protected byte[] invokeBytes( String api ) { assert api log.debug "Request [credentials ${getAuthObfuscated() ?: '-'}] -> $api" final request = newRequest(api) // submit the request - final HttpResponse resp = httpSend(request) + final HttpResponse resp = httpSend0(request) // check the response code checkResponse(resp) - // return the body as string + // return the body as byte array return resp.body() } @@ -263,7 +274,7 @@ abstract class RepositoryProvider { * * @param response A {@link HttpURLConnection} response instance */ - protected checkResponse( HttpResponse response ) { + protected checkResponse( HttpResponse response ) { final code = response.statusCode() if( code==401 ) { log.debug "Response status: $code -- ${response.body()}" @@ -447,6 +458,7 @@ abstract class RepositoryProvider { return isCausedByUnresolvedAddressException(t.cause) } + @Deprecated protected HttpResponse httpSend(HttpRequest request) { if( httpClient==null ) httpClient = newHttpClient() @@ -460,10 +472,24 @@ abstract class RepositoryProvider { } } + private HttpResponse httpSend0(HttpRequest request) { + if( httpClient==null ) + httpClient = newHttpClient() + if( retryConfig==null ) + retryConfig = new RetryConfig() + try { + safeApply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray())) + } + catch (FailsafeException e) { + throw e.cause + } + } + private HttpClient newHttpClient() { final builder = HttpClient.newBuilder() .version(HttpClient.Version.HTTP_1_1) .connectTimeout(Duration.ofSeconds(60)) + .followRedirects(HttpClient.Redirect.NORMAL) // use virtual threads executor if enabled if( Threads.useVirtual() ) builder.executor(Executors.newVirtualThreadPerTaskExecutor()) diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy index 0322568b84..e279e55866 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/AzureRepositoryProviderTest.groovy @@ -51,7 +51,6 @@ class AzureRepositoryProviderTest extends Specification { 't-neumann/hello' | ['t-neumann', 'hello', 'hello'] 'ORGANIZATION/PROJECT/hello' | ['ORGANIZATION','PROJECT','hello'] 'ORGANIZATION/PROJECT/_git/hello' | ['ORGANIZATION','PROJECT','hello'] - } def 'should throw exception if wrong path' () { @@ -69,7 +68,6 @@ class AzureRepositoryProviderTest extends Specification { } def 'should return repo url' () { - given: def config = new ConfigSlurper().parse(CONFIG) def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject) @@ -85,7 +83,6 @@ class AzureRepositoryProviderTest extends Specification { } def 'should return project URL' () { - given: def config = new ConfigSlurper().parse(CONFIG) def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject) @@ -97,29 +94,24 @@ class AzureRepositoryProviderTest extends Specification { 't-neumann/hello' | 'https://dev.azure.com/t-neumann/hello' 'ORGANIZATION/PROJECT/hello' | 'https://dev.azure.com/ORGANIZATION/PROJECT/hello' 'ORGANIZATION/PROJECT/_git/hello' | 'https://dev.azure.com/ORGANIZATION/PROJECT/_git/hello' - } def 'should return content URL' () { - given: def config = new ConfigSlurper().parse(CONFIG) def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject) expect: new AzureRepositoryProvider('t-neumann/hello', obj).getContentUrl('main.nf') == 'https://dev.azure.com/t-neumann/hello/_apis/git/repositories/hello/items?download=false&includeContent=true&includeContentMetadata=false&api-version=6.0&\$format=json&path=main.nf' - } def 'should return content URL for revision' () { - given: def config = new ConfigSlurper().parse(CONFIG) def obj = new ProviderConfig('azurerepos', config.providers.azurerepos as ConfigObject) expect: new AzureRepositoryProvider('t-neumann/hello', obj).setRevision("a-branch").getContentUrl('main.nf') == 'https://dev.azure.com/t-neumann/hello/_apis/git/repositories/hello/items?download=false&includeContent=true&includeContentMetadata=false&api-version=6.0&\$format=json&path=main.nf&versionDescriptor.version=a-branch' - } /* @@ -141,7 +133,6 @@ class AzureRepositoryProviderTest extends Specification { def result = repo.readText('main.nf') then: result == 'println "Hello from Azure repos!"' - } @IgnoreIf({System.getenv('NXF_SMOKE')}) diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/BitbucketRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/BitbucketRepositoryProviderTest.groovy index 7dc933f793..b38bd76d1f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/BitbucketRepositoryProviderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/BitbucketRepositoryProviderTest.groovy @@ -28,7 +28,6 @@ class BitbucketRepositoryProviderTest extends Specification { @Requires( { System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') } ) def testBitbucketCloneURL() { - given: def token = System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') def config = new ProviderConfig('bitbucket').setAuth(token) @@ -39,16 +38,13 @@ class BitbucketRepositoryProviderTest extends Specification { url == "https://${config.user}@bitbucket.org/pditommaso/tutorial.git".toString() } - def testGetHomePage() { expect: new BitbucketRepositoryProvider('pditommaso/tutorial').getRepositoryUrl() == "https://bitbucket.org/pditommaso/tutorial" } - @Requires( { System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') } ) def testReadContent() { - given: def token = System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') def config = new ProviderConfig('bitbucket').setAuth(token) @@ -59,7 +55,21 @@ class BitbucketRepositoryProviderTest extends Specification { then: result.trim().startsWith('#!/usr/bin/env nextflow') + } + @Requires( { System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') } ) + def 'should read binary data'() { + given: + def token = System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') + def config = new ProviderConfig('bitbucket').setAuth(token) + def DATA = this.class.getResourceAsStream('/test-sandbucket.jpg').bytes + + when: + def repo = new BitbucketRepositoryProvider('pditommaso/tutorial', config) + def result = repo.readBytes('sandbucket.jpg') + + then: + result == DATA } @Requires( { System.getenv('NXF_BITBUCKET_ACCESS_TOKEN') } ) @@ -160,6 +170,5 @@ class BitbucketRepositoryProviderTest extends Specification { then: !data.contains('world') data.contains('mundo') - } } diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy index 27d050d960..f1791e2b6a 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy @@ -19,7 +19,6 @@ package nextflow.scm import spock.lang.IgnoreIf import spock.lang.Requires import spock.lang.Specification - /** * * @author Akira Sekiguchi @@ -28,68 +27,59 @@ class GiteaRepositoryProviderTest extends Specification { static final String CONFIG = ''' providers { - mygitea { - server = 'https://git.seqera.io' - endpoint = 'https://git.seqera.io/api/v1' + server = 'https://gitea.com' + endpoint = 'https://gitea.com/api/v1' platform = 'gitea' - user = 'myname' - password = 'mypassword' } - } ''' def 'should return repo url' () { - given: def config = new ConfigSlurper().parse(CONFIG) def obj = new ProviderConfig('gitea', config.providers.mygitea as ConfigObject) expect: - new GiteaRepositoryProvider('pditommaso/hello', obj).getEndpointUrl() == 'https://git.seqera.io/api/v1/repos/pditommaso/hello' + new GiteaRepositoryProvider('pditommaso/hello', obj).getEndpointUrl() == 'https://gitea.com/api/v1/repos/pditommaso/hello' } def 'should return project URL' () { - given: def config = new ConfigSlurper().parse(CONFIG) def obj = new ProviderConfig('gitea', config.providers.mygitea as ConfigObject) expect: - new GiteaRepositoryProvider('pditommaso/hello', obj).getRepositoryUrl() == 'https://git.seqera.io/pditommaso/hello' - + new GiteaRepositoryProvider('pditommaso/hello', obj).getRepositoryUrl() == 'https://gitea.com/pditommaso/hello' } def 'should return content URL' () { - given: def config = new ConfigSlurper().parse(CONFIG) def obj = new ProviderConfig('gitea', config.providers.mygitea as ConfigObject) expect: new GiteaRepositoryProvider('pditommaso/hello', obj) - .getContentUrl('main.nf') == 'https://git.seqera.io/api/v1/repos/pditommaso/hello/raw/main.nf' + .getContentUrl('main.nf') == 'https://gitea.com/api/v1/repos/pditommaso/hello/raw/main.nf' and: new GiteaRepositoryProvider('pditommaso/hello', obj) .setRevision('12345') - .getContentUrl('main.nf') == 'https://git.seqera.io/api/v1/repos/pditommaso/hello/raw/main.nf?ref=12345' + .getContentUrl('main.nf') == 'https://gitea.com/api/v1/repos/pditommaso/hello/raw/main.nf?ref=12345' } @IgnoreIf({System.getenv('NXF_SMOKE')}) @Requires({System.getenv('NXF_GITEA_ACCESS_TOKEN')}) def 'should read file content'() { - given: - def token = System.getenv('NXF_GITEA_ACCESS_TOKEN') - def config = new ProviderConfig('gitea', new ConfigSlurper().parse(CONFIG).providers.mygitea as ConfigObject).setAuth(token) + def token = System.getenv('NXF_GITEA_ACCESS_TOKEN') + def config = new ProviderConfig('gitea', new ConfigSlurper().parse(CONFIG).providers.mygitea as ConfigObject) .setAuth(token) when: - def repo = new GiteaRepositoryProvider('test-org/nextflow-ci-repo', config) + def repo = new GiteaRepositoryProvider('pditommaso/test-hello', config) def result = repo.readText('README.md') then: - result.contains('nextflow-ci-repo') + result.contains('Basic Nextflow script') // when: // repo = new GiteaRepositoryProvider('test-org/nextflow-ci-repo', config) @@ -97,7 +87,23 @@ class GiteaRepositoryProviderTest extends Specification { // result = repo.readText('README.md') // then: // result.contains("foo branch") + } + + @IgnoreIf({System.getenv('NXF_SMOKE')}) + @Requires({System.getenv('NXF_GITEA_ACCESS_TOKEN')}) + def 'should read bytes gitea content'() { + given: + def token = System.getenv('NXF_GITEA_ACCESS_TOKEN') + def config = new ProviderConfig('gitea', new ConfigSlurper().parse(CONFIG).providers.mygitea as ConfigObject) .setAuth(token) + def repo = new GiteaRepositoryProvider('pditommaso/test-hello', config) + and: + def DATA = this.class.getResourceAsStream('/test-asset.bin').bytes + + when: + def result = repo.readBytes('test/test-asset.bin') + then: + result == DATA } } diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/GithubRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/GithubRepositoryProviderTest.groovy index 74ea1800c8..b642222efb 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/GithubRepositoryProviderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/GithubRepositoryProviderTest.groovy @@ -26,36 +26,47 @@ class GithubRepositoryProviderTest extends Specification { @Requires({System.getenv('NXF_GITHUB_ACCESS_TOKEN')}) def testGitCloneUrl() { - given: def token = System.getenv('NXF_GITHUB_ACCESS_TOKEN') def config = new ProviderConfig('github').setAuth(token) when: - def url = new GithubRepositoryProvider('nextflow-io/hello',config).getCloneUrl() + def url = new GithubRepositoryProvider('nextflow-io/test-hello',config).getCloneUrl() then: - url == 'https://github.com/nextflow-io/hello.git' - + url == 'https://github.com/nextflow-io/test-hello.git' } def testGetHomePage() { expect: - new GithubRepositoryProvider('nextflow-io/hello').getRepositoryUrl() == "https://github.com/nextflow-io/hello" + new GithubRepositoryProvider('nextflow-io/test-hello').getRepositoryUrl() == "https://github.com/nextflow-io/test-hello" } @Requires({System.getenv('NXF_GITHUB_ACCESS_TOKEN')}) def testReadContent() { - given: def token = System.getenv('NXF_GITHUB_ACCESS_TOKEN') def config = new ProviderConfig('github').setAuth(token) + def repo = new GithubRepositoryProvider('nextflow-io/test-hello', config) when: - def repo = new GithubRepositoryProvider('nextflow-io/hello', config) def result = repo.readText('main.nf') then: result.trim().startsWith('#!/usr/bin/env nextflow') + } + @Requires({System.getenv('NXF_GITHUB_ACCESS_TOKEN')}) + def 'should read bytes github content'() { + given: + def token = System.getenv('NXF_GITHUB_ACCESS_TOKEN') + def config = new ProviderConfig('github').setAuth(token) + def repo = new GithubRepositoryProvider('nextflow-io/test-hello', config) + and: + def DATA = this.class.getResourceAsStream('/test-asset.bin').bytes + + when: + def result = repo.readBytes('/test/test-asset.bin') + then: + result == DATA } def 'should return content URL' () { @@ -139,5 +150,22 @@ class GithubRepositoryProviderTest extends Specification { cleanup: SysEnv.pop() } + + @Requires({System.getenv('NXF_GITHUB_ACCESS_TOKEN')}) + def 'should read content from renamed repository'() { + given: + def token = System.getenv('NXF_GITHUB_ACCESS_TOKEN') + def config = new ProviderConfig('github').setAuth(token) + // this repository has been renamed to `pditommaso/ciao` + // nevertheless the read content should work, because the + // client needs to be able to follow the http redirection + // returned by the github backend + def repo = new GithubRepositoryProvider('pditommaso/hello', config) + + when: + def result = repo.readText('main.nf') + then: + result.trim().startsWith(/println "I'm the main"/) + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy index 6330ca4c89..1cc348c650 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/GitlabRepositoryProviderTest.groovy @@ -28,22 +28,17 @@ import spock.lang.Specification class GitlabRepositoryProviderTest extends Specification { def 'should return repo url' () { - expect: new GitlabRepositoryProvider('pditommaso/hello').getEndpointUrl() == 'https://gitlab.com/api/v4/projects/pditommaso%2Fhello' - } def 'should return project URL' () { - expect: new GitlabRepositoryProvider('pditommaso/hello').getRepositoryUrl() == 'https://gitlab.com/pditommaso/hello' - } @Requires({System.getenv('NXF_GITLAB_ACCESS_TOKEN')}) def 'should return clone url'() { - given: def token = System.getenv('NXF_GITLAB_ACCESS_TOKEN') def config = new ProviderConfig('gitlab').setAuth(token) @@ -52,13 +47,10 @@ class GitlabRepositoryProviderTest extends Specification { def url = new GitlabRepositoryProvider('pditommaso/hello', config).getCloneUrl() then: url == 'https://gitlab.com/pditommaso/hello.git' - } - @Requires({System.getenv('NXF_GITLAB_ACCESS_TOKEN')}) def 'should read file content'() { - given: def token = System.getenv('NXF_GITLAB_ACCESS_TOKEN') def config = new ProviderConfig('gitlab').setAuth(token) @@ -68,12 +60,25 @@ class GitlabRepositoryProviderTest extends Specification { def result = repo.readText('main.nf') then: result.trim().startsWith('#!/usr/bin/env nextflow') + } + + @Requires({System.getenv('NXF_GITLAB_ACCESS_TOKEN')}) + def 'should read binary content'() { + given: + def token = System.getenv('NXF_GITLAB_ACCESS_TOKEN') + def config = new ProviderConfig('gitlab').setAuth(token) + and: + def DATA = this.class.getResourceAsStream('/test-asset.bin').bytes + when: + def repo = new GitlabRepositoryProvider('pditommaso/hello', config) + def result = repo.readBytes('test/test-asset.bin') + then: + result == DATA } @Requires({System.getenv('NXF_GITLAB_ACCESS_TOKEN')}) def 'should return default branch' () { - given: def token = System.getenv('NXF_GITLAB_ACCESS_TOKEN') def config = new ProviderConfig('gitlab').setAuth(token) @@ -82,7 +87,6 @@ class GitlabRepositoryProviderTest extends Specification { def provider = new GitlabRepositoryProvider('pditommaso/hello', config) then: provider.getDefaultBranch() == 'master' - } @Requires({System.getenv('NXF_GITLAB_ACCESS_TOKEN')}) diff --git a/modules/nextflow/src/test/resources/test-asset.bin b/modules/nextflow/src/test/resources/test-asset.bin new file mode 100644 index 0000000000..9e1844d303 Binary files /dev/null and b/modules/nextflow/src/test/resources/test-asset.bin differ diff --git a/modules/nextflow/src/test/resources/test-sandbucket.jpg b/modules/nextflow/src/test/resources/test-sandbucket.jpg new file mode 100644 index 0000000000..a2e8af6cf7 Binary files /dev/null and b/modules/nextflow/src/test/resources/test-sandbucket.jpg differ