diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index ae295e534..2afdf3d22 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -204,7 +204,7 @@ jobs: env: VERSION: ${{ matrix.cwl-version }} CONTAINER: ${{ matrix.container }} - GIT_TARGET: ${{ matrix.cwl-version == 'v1.2' && '1.2.1_proposed' || 'main' }} + GIT_TARGET: main CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }} run: ./conformance-test.sh - name: Archive test results @@ -266,7 +266,7 @@ jobs: macos: name: Test on macos-latest - runs-on: macos-latest + runs-on: macos-13 # not latest, that is now an Apple Silicon M1, for which seqtk is not yet built on bioconda env: TOXENV: py312-unit steps: diff --git a/conformance-test.sh b/conformance-test.sh index 9506c28bb..cdc57f69f 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -13,15 +13,15 @@ venv() { } # Set these variables when running the script, e.g.: -# VERSION=v1.2 GIT_TARGET=1.2.1_proposed CONTAINER=podman ./conformance_test.sh +# VERSION=v1.2 GIT_TARGET=main CONTAINER=podman ./conformance_test.sh # Version of the standard to test against # Current options: v1.0, v1.1, v1.2 VERSION=${VERSION:-"v1.2"} # Which commit of the standard's repo to use -# Defaults to the last commit of the 1.2.1_proposed branch -GIT_TARGET=${GIT_TARGET:-"1.2.1_proposed"} +# Defaults to the last commit of the main branch +GIT_TARGET=${GIT_TARGET:-"main"} # Which container runtime to use # Valid options: docker, singularity diff --git a/cwltool/builder.py b/cwltool/builder.py index f5d6824e0..ff1c099e5 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -86,7 +86,7 @@ def content_limit_respected_read(f: IO[bytes]) -> str: :returns: the file contents :raises WorkflowException: if the file is too large """ - return content_limit_respected_read_bytes(f).decode("utf-8") + return str(content_limit_respected_read_bytes(f), "utf-8") def substitute(value: str, replace: str) -> str: diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 5b2bea5b2..b11bb4bc3 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -1369,8 +1369,8 @@ def collect_output( else: if binding.get("loadContents"): with fs_access.open(cast(str, rfile["location"]), "rb") as f: - files["contents"] = content_limit_respected_read_bytes(f).decode( - "utf-8" + files["contents"] = str( + content_limit_respected_read_bytes(f), "utf-8" ) if compute_checksum: with fs_access.open(cast(str, rfile["location"]), "rb") as f: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index c34e32082..7c6eaf5d6 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -106,6 +106,7 @@ def __str__(self) -> str: return f"ResearchObject <{self.ro_uuid}> in <{self.folder}>" def _initialize(self) -> None: + """Initialize the bagit folder structure.""" for research_obj_folder in ( METADATA, DATA, @@ -359,6 +360,7 @@ def add_annotation( return uri def _ro_annotations(self) -> List[Annotation]: + """Append base RO and provenance annotations to the list of annotations.""" annotations: List[Annotation] = [] annotations.append( { @@ -414,6 +416,7 @@ def _ro_annotations(self) -> List[Annotation]: def _authored_by(self) -> Optional[AuthoredBy]: authored_by: AuthoredBy = {} + """Returns the authoredBy metadata if it was supplied on CLI""" if self.orcid: authored_by["orcid"] = self.orcid if self.full_name: @@ -542,6 +545,7 @@ def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None: checksum_file.write(line) def _add_to_bagit(self, rel_path: str, **checksums: str) -> None: + """Compute file size and checksums and adds to bagit manifest.""" if PurePosixPath(rel_path).is_absolute(): raise ValueError(f"rel_path must be relative: {rel_path}") lpath = os.path.join(self.folder, local_path(rel_path)) diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index 7dcf85cbc..dbe9e2f97 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -27,7 +27,7 @@ def printrdf(wflow: Process, ctx: ContextType, style: str) -> str: rdf = gather(wflow, ctx).serialize(format=style, encoding="utf-8") if not rdf: return "" - return rdf.decode("utf-8") + return str(rdf, "utf-8") def lastpart(uri: Any) -> str: diff --git a/cwltool/docker.py b/cwltool/docker.py index a1b9c8a37..d0f628b15 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -116,36 +116,38 @@ def get_image( if (docker_image_id := docker_requirement.get("dockerImageId")) is not None: try: manifest = json.loads( - subprocess.check_output( - [self.docker_exec, "inspect", docker_image_id] - ).decode( # nosec - "utf-8" + str( + subprocess.check_output( + [self.docker_exec, "inspect", docker_image_id] + ), # nosec + "utf-8", ) ) found = manifest is not None except (OSError, subprocess.CalledProcessError, UnicodeError): pass + cmd: List[str] = [] + if "dockerFile" in docker_requirement: + dockerfile_dir = create_tmp_dir(tmp_outdir_prefix) + with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile: + dfile.write(docker_requirement["dockerFile"]) + cmd = [ + self.docker_exec, + "build", + "--tag=%s" % str(docker_requirement["dockerImageId"]), + dockerfile_dir, + ] + _logger.info(str(cmd)) + subprocess.check_call(cmd, stdout=sys.stderr) # nosec + found = True + if (force_pull or not found) and pull_image: - cmd: List[str] = [] if "dockerPull" in docker_requirement: cmd = [self.docker_exec, "pull", str(docker_requirement["dockerPull"])] _logger.info(str(cmd)) subprocess.check_call(cmd, stdout=sys.stderr) # nosec found = True - elif "dockerFile" in docker_requirement: - dockerfile_dir = create_tmp_dir(tmp_outdir_prefix) - with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile: - dfile.write(docker_requirement["dockerFile"]) - cmd = [ - self.docker_exec, - "build", - "--tag=%s" % str(docker_requirement["dockerImageId"]), - dockerfile_dir, - ] - _logger.info(str(cmd)) - subprocess.check_call(cmd, stdout=sys.stderr) # nosec - found = True elif "dockerLoad" in docker_requirement: cmd = [self.docker_exec, "load"] _logger.info(str(cmd)) diff --git a/cwltool/utils.py b/cwltool/utils.py index 219f91830..c8620994a 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -200,7 +200,7 @@ def bytes2str_in_dicts( # if value is bytes, return decoded string, elif isinstance(inp, bytes): - return inp.decode("utf-8") + return str(inp, "utf-8") # simply return elements itself return inp diff --git a/lint-requirements.txt b/lint-requirements.txt index 576703b32..cad3da694 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<24.3 -black~=24.3 +black~=24.4 codespell diff --git a/mypy-requirements.txt b/mypy-requirements.txt index c2db73673..c2f3de419 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.9.0 # also update pyproject.toml +mypy==1.10.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 types-requests diff --git a/pyproject.toml b/pyproject.toml index 09789e45a..e845a4e5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.9.0", # also update mypy-requirements.txt + "mypy==1.10.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", diff --git a/release-test.sh b/release-test.sh index a5e620391..46c1e08eb 100755 --- a/release-test.sh +++ b/release-test.sh @@ -71,7 +71,7 @@ pushd src/${package} pip install -rtest-requirements.txt build make dist #make test -cp dist/${package}*tar.gz ../../../testenv3/ +cp dist/${module}*tar.gz ../../../testenv3/ pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install popd # ../.. no subdir named ${proj} here, safe for py.testing the installed module # shellcheck disable=SC2086 @@ -87,13 +87,13 @@ source bin/activate rm -f lib/python-wheels/setuptools* \ && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel -package_tar=$(find . -name "${package}*tar.gz") +package_tar=$(find . -name "${module}*tar.gz") pip install "-r${DIR}/test-requirements.txt" udocker build pip install "${package_tar}${extras}" udocker install mkdir out -tar --extract --directory=out -z -f ${package}*.tar.gz -pushd out/${package}* +tar --extract --directory=out -z -f ${module}*.tar.gz +pushd out/${module}* make dist make test pip install "-r${DIR}/mypy-requirements.txt" diff --git a/setup.py b/setup.py index ddd5ed454..896dd7a61 100644 --- a/setup.py +++ b/setup.py @@ -151,7 +151,7 @@ test_suite="tests", tests_require=[ "bagit >= 1.6.4, < 1.9", - "pytest >= 6.2, < 8.2", + "pytest >= 6.2, < 8.3", "mock >= 2.0.0", "pytest-mock >= 1.10.0", "pytest-httpserver", diff --git a/test-requirements.txt b/test-requirements.txt index d7901c3a5..cffff68a3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ bagit>=1.6.4,<1.9 -pytest>= 6.2,< 8.2 +pytest>= 6.2,< 8.3 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" pytest-httpserver diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 018a92120..0532f3cde 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -12,7 +12,9 @@ from cwltool.main import main from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION -from cwltool.utils import CWLObjectType +from cwltool.utils import CWLObjectType, CONTENT_LIMIT, bytes2str_in_dicts +from cwltool.builder import content_limit_respected_read +from cwltool.errors import WorkflowException from .util import needs_docker @@ -214,3 +216,26 @@ def test_clt_returns_specialchar_names(tmp_path: Path) -> None: result["location"] == "keep:ae755cd1b3cff63152ff4200f4dea7e9+52/%3A%3F%23%5B%5D%40%21%24%26%27%28%29%2A%2B%2C%3B%3D" ) + + +def test_content_limit_respected_read() -> None: + b1 = b"abcd" * 100 + b1io = BytesIO(b1) + + assert len(b1) < CONTENT_LIMIT + assert content_limit_respected_read(b1io) == str("abcd" * 100) + + b2 = b"abcd" * 20000 + b2io = BytesIO(b2) + + assert len(b2) > CONTENT_LIMIT + with pytest.raises(WorkflowException): + content_limit_respected_read(b2io) + + +def test_bytes2str_in_dicts() -> None: + assert bytes2str_in_dicts({"foo": b"bar"}) == {"foo": "bar"} + + assert bytes2str_in_dicts({"foo": [b"bar"]}) == {"foo": ["bar"]} + + assert bytes2str_in_dicts({"foo": {"foo2": b"bar"}}) == {"foo": {"foo2": "bar"}} diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index af830834b..2470a2515 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -168,6 +168,64 @@ def test_dockerfile_tmpdir_prefix(tmp_path: Path, monkeypatch: pytest.MonkeyPatc assert (subdir / "Dockerfile").exists() +@needs_docker +def test_dockerfile_build(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Test that DockerCommandLineJob.get_image builds a Dockerfile.""" + (tmp_path / "out").mkdir() + tmp_outdir_prefix = tmp_path / "out" / "1" + (tmp_path / "3").mkdir() + tmpdir_prefix = str(tmp_path / "3" / "ttmp") + runtime_context = RuntimeContext( + {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None} + ) + builder = Builder( + {}, + [], + [], + {}, + schema.Names(), + [], + [], + {}, + None, + None, + StdFsAccess, + StdFsAccess(""), + None, + 0.1, + False, + False, + False, + "no_listing", + runtime_context.get_outdir(), + runtime_context.get_tmpdir(), + runtime_context.get_stagedir(), + INTERNAL_VERSION, + "docker", + ) + + docker_image_id = sys._getframe().f_code.co_name + + assert DockerCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + { + "class": "DockerRequirement", + "dockerFile": "FROM debian:stable-slim", + "dockerImageId": docker_image_id, + }, + pull_image=False, + force_pull=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + ) + output = subprocess.check_output( + ["docker", "images", "--quiet", docker_image_id], stderr=subprocess.STDOUT, text=True + ) + + # If the output is empty, the image doesn't exist + assert output.strip(), f"Docker image {docker_image_id} does not exist" + + @needs_singularity def test_dockerfile_singularity_build(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test that SingularityCommandLineJob.get_image builds a Dockerfile with Singularity."""