Update branch to sync with main

.github/workflows/ci-tests.yml

@@ -204,7 +204,7 @@ jobs:
         env:
           VERSION: ${{ matrix.cwl-version }}
           CONTAINER: ${{ matrix.container }}
-          GIT_TARGET: ${{ matrix.cwl-version == 'v1.2' && '1.2.1_proposed' || 'main' }}
+          GIT_TARGET: main
           CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }}
         run: ./conformance-test.sh
       - name: Archive test results
@@ -266,7 +266,7 @@ jobs:
 
   macos:
     name: Test on macos-latest
-    runs-on: macos-latest
+    runs-on: macos-13  # not latest, that is now an Apple Silicon M1, for which seqtk is not yet built on bioconda
     env:
       TOXENV: py312-unit
     steps:

conformance-test.sh

@@ -13,15 +13,15 @@ venv() {
 }
 
 # Set these variables when running the script, e.g.:
-# VERSION=v1.2 GIT_TARGET=1.2.1_proposed CONTAINER=podman ./conformance_test.sh
+# VERSION=v1.2 GIT_TARGET=main CONTAINER=podman ./conformance_test.sh
 
 # Version of the standard to test against
 # Current options: v1.0, v1.1, v1.2
 VERSION=${VERSION:-"v1.2"}
 
 # Which commit of the standard's repo to use
-# Defaults to the last commit of the 1.2.1_proposed branch
-GIT_TARGET=${GIT_TARGET:-"1.2.1_proposed"}
+# Defaults to the last commit of the main branch
+GIT_TARGET=${GIT_TARGET:-"main"}
 
 # Which container runtime to use
 # Valid options: docker, singularity

cwltool/builder.py

@@ -86,7 +86,7 @@ def content_limit_respected_read(f: IO[bytes]) -> str:
     :returns: the file contents
     :raises WorkflowException: if the file is too large
     """
-    return content_limit_respected_read_bytes(f).decode("utf-8")
+    return str(content_limit_respected_read_bytes(f), "utf-8")
 
 
 def substitute(value: str, replace: str) -> str:

cwltool/command_line_tool.py

@@ -1369,8 +1369,8 @@ def collect_output(
                     else:
                         if binding.get("loadContents"):
                             with fs_access.open(cast(str, rfile["location"]), "rb") as f:
-                                files["contents"] = content_limit_respected_read_bytes(f).decode(
-                                    "utf-8"
+                                files["contents"] = str(
+                                    content_limit_respected_read_bytes(f), "utf-8"
                                 )
                         if compute_checksum:
                             with fs_access.open(cast(str, rfile["location"]), "rb") as f:

cwltool/cwlprov/ro.py

@@ -106,6 +106,7 @@ def __str__(self) -> str:
         return f"ResearchObject <{self.ro_uuid}> in <{self.folder}>"
 
     def _initialize(self) -> None:
+        """Initialize the bagit folder structure."""
         for research_obj_folder in (
             METADATA,
             DATA,
@@ -359,6 +360,7 @@ def add_annotation(
         return uri
 
     def _ro_annotations(self) -> List[Annotation]:
+        """Append base RO and provenance annotations to the list of annotations."""
         annotations: List[Annotation] = []
         annotations.append(
             {
@@ -414,6 +416,7 @@ def _ro_annotations(self) -> List[Annotation]:
 
     def _authored_by(self) -> Optional[AuthoredBy]:
         authored_by: AuthoredBy = {}
+        """Returns the authoredBy metadata if it was supplied on CLI"""
         if self.orcid:
             authored_by["orcid"] = self.orcid
         if self.full_name:
@@ -542,6 +545,7 @@ def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None:
                 checksum_file.write(line)
 
     def _add_to_bagit(self, rel_path: str, **checksums: str) -> None:
+        """Compute file size and checksums and adds to bagit manifest."""
         if PurePosixPath(rel_path).is_absolute():
             raise ValueError(f"rel_path must be relative: {rel_path}")
         lpath = os.path.join(self.folder, local_path(rel_path))

cwltool/cwlrdf.py

@@ -27,7 +27,7 @@ def printrdf(wflow: Process, ctx: ContextType, style: str) -> str:
     rdf = gather(wflow, ctx).serialize(format=style, encoding="utf-8")
     if not rdf:
         return ""
-    return rdf.decode("utf-8")
+    return str(rdf, "utf-8")
 
 
 def lastpart(uri: Any) -> str:

cwltool/docker.py

@@ -116,36 +116,38 @@ def get_image(
         if (docker_image_id := docker_requirement.get("dockerImageId")) is not None:
             try:
                 manifest = json.loads(
-                    subprocess.check_output(
-                        [self.docker_exec, "inspect", docker_image_id]
-                    ).decode(  # nosec
-                        "utf-8"
+                    str(
+                        subprocess.check_output(
+                            [self.docker_exec, "inspect", docker_image_id]
+                        ),  # nosec
+                        "utf-8",
                     )
                 )
                 found = manifest is not None
             except (OSError, subprocess.CalledProcessError, UnicodeError):
                 pass
 
+        cmd: List[str] = []
+        if "dockerFile" in docker_requirement:
+            dockerfile_dir = create_tmp_dir(tmp_outdir_prefix)
+            with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile:
+                dfile.write(docker_requirement["dockerFile"])
+            cmd = [
+                self.docker_exec,
+                "build",
+                "--tag=%s" % str(docker_requirement["dockerImageId"]),
+                dockerfile_dir,
+            ]
+            _logger.info(str(cmd))
+            subprocess.check_call(cmd, stdout=sys.stderr)  # nosec
+            found = True
+
         if (force_pull or not found) and pull_image:
-            cmd: List[str] = []
             if "dockerPull" in docker_requirement:
                 cmd = [self.docker_exec, "pull", str(docker_requirement["dockerPull"])]
                 _logger.info(str(cmd))
                 subprocess.check_call(cmd, stdout=sys.stderr)  # nosec
                 found = True
-            elif "dockerFile" in docker_requirement:
-                dockerfile_dir = create_tmp_dir(tmp_outdir_prefix)
-                with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile:
-                    dfile.write(docker_requirement["dockerFile"])
-                cmd = [
-                    self.docker_exec,
-                    "build",
-                    "--tag=%s" % str(docker_requirement["dockerImageId"]),
-                    dockerfile_dir,
-                ]
-                _logger.info(str(cmd))
-                subprocess.check_call(cmd, stdout=sys.stderr)  # nosec
-                found = True
             elif "dockerLoad" in docker_requirement:
                 cmd = [self.docker_exec, "load"]
                 _logger.info(str(cmd))

cwltool/utils.py

@@ -200,7 +200,7 @@ def bytes2str_in_dicts(
 
     # if value is bytes, return decoded string,
     elif isinstance(inp, bytes):
-        return inp.decode("utf-8")
+        return str(inp, "utf-8")
 
     # simply return elements itself
     return inp

lint-requirements.txt

@@ -1,3 +1,3 @@
 flake8-bugbear<24.3
-black~=24.3
+black~=24.4
 codespell

mypy-requirements.txt

@@ -1,4 +1,4 @@
-mypy==1.9.0  # also update pyproject.toml
+mypy==1.10.0  # also update pyproject.toml
 ruamel.yaml>=0.16.0,<0.19
 cwl-utils>=0.32
 types-requests

pyproject.toml

@@ -2,7 +2,7 @@
 requires = [
     "setuptools>=45",
     "setuptools_scm[toml]>=8.0.4,<9",
-    "mypy==1.9.0",  # also update mypy-requirements.txt
+    "mypy==1.10.0",  # also update mypy-requirements.txt
     "types-requests",
     "types-psutil",
     "importlib_resources>=1.4;python_version<'3.9'",

release-test.sh

@@ -71,7 +71,7 @@ pushd src/${package}
 pip install -rtest-requirements.txt build
 make dist
 #make test
-cp dist/${package}*tar.gz ../../../testenv3/
+cp dist/${module}*tar.gz ../../../testenv3/
 pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 popd # ../.. no subdir named ${proj} here, safe for py.testing the installed module
 # shellcheck disable=SC2086
@@ -87,13 +87,13 @@ source bin/activate
 rm -f lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
-package_tar=$(find . -name "${package}*tar.gz")
+package_tar=$(find . -name "${module}*tar.gz")
 pip install "-r${DIR}/test-requirements.txt" udocker build
 pip install "${package_tar}${extras}"
 udocker install
 mkdir out
-tar --extract --directory=out -z -f ${package}*.tar.gz
-pushd out/${package}*
+tar --extract --directory=out -z -f ${module}*.tar.gz
+pushd out/${module}*
 make dist
 make test
 pip install "-r${DIR}/mypy-requirements.txt"

setup.py

@@ -151,7 +151,7 @@
     test_suite="tests",
     tests_require=[
         "bagit >= 1.6.4, < 1.9",
-        "pytest >= 6.2, < 8.2",
+        "pytest >= 6.2, < 8.3",
         "mock >= 2.0.0",
         "pytest-mock >= 1.10.0",
         "pytest-httpserver",

test-requirements.txt

@@ -1,5 +1,5 @@
 bagit>=1.6.4,<1.9
-pytest>= 6.2,< 8.2
+pytest>= 6.2,< 8.3
 pytest-xdist>=3.2.0 # for the worksteal scheduler
 psutil # enhances pytest-xdist to allow "-n logical"
 pytest-httpserver

tests/test_path_checks.py

@@ -12,7 +12,9 @@
 from cwltool.main import main
 from cwltool.stdfsaccess import StdFsAccess
 from cwltool.update import INTERNAL_VERSION
-from cwltool.utils import CWLObjectType
+from cwltool.utils import CWLObjectType, CONTENT_LIMIT, bytes2str_in_dicts
+from cwltool.builder import content_limit_respected_read
+from cwltool.errors import WorkflowException
 
 from .util import needs_docker
 
@@ -214,3 +216,26 @@ def test_clt_returns_specialchar_names(tmp_path: Path) -> None:
         result["location"]
         == "keep:ae755cd1b3cff63152ff4200f4dea7e9+52/%3A%3F%23%5B%5D%40%21%24%26%27%28%29%2A%2B%2C%3B%3D"
     )
+
+
+def test_content_limit_respected_read() -> None:
+    b1 = b"abcd" * 100
+    b1io = BytesIO(b1)
+
+    assert len(b1) < CONTENT_LIMIT
+    assert content_limit_respected_read(b1io) == str("abcd" * 100)
+
+    b2 = b"abcd" * 20000
+    b2io = BytesIO(b2)
+
+    assert len(b2) > CONTENT_LIMIT
+    with pytest.raises(WorkflowException):
+        content_limit_respected_read(b2io)
+
+
+def test_bytes2str_in_dicts() -> None:
+    assert bytes2str_in_dicts({"foo": b"bar"}) == {"foo": "bar"}
+
+    assert bytes2str_in_dicts({"foo": [b"bar"]}) == {"foo": ["bar"]}
+
+    assert bytes2str_in_dicts({"foo": {"foo2": b"bar"}}) == {"foo": {"foo2": "bar"}}

tests/test_tmpdir.py

@@ -168,6 +168,64 @@ def test_dockerfile_tmpdir_prefix(tmp_path: Path, monkeypatch: pytest.MonkeyPatc
     assert (subdir / "Dockerfile").exists()
 
 
+@needs_docker
+def test_dockerfile_build(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    """Test that DockerCommandLineJob.get_image builds a Dockerfile."""
+    (tmp_path / "out").mkdir()
+    tmp_outdir_prefix = tmp_path / "out" / "1"
+    (tmp_path / "3").mkdir()
+    tmpdir_prefix = str(tmp_path / "3" / "ttmp")
+    runtime_context = RuntimeContext(
+        {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None}
+    )
+    builder = Builder(
+        {},
+        [],
+        [],
+        {},
+        schema.Names(),
+        [],
+        [],
+        {},
+        None,
+        None,
+        StdFsAccess,
+        StdFsAccess(""),
+        None,
+        0.1,
+        False,
+        False,
+        False,
+        "no_listing",
+        runtime_context.get_outdir(),
+        runtime_context.get_tmpdir(),
+        runtime_context.get_stagedir(),
+        INTERNAL_VERSION,
+        "docker",
+    )
+
+    docker_image_id = sys._getframe().f_code.co_name
+
+    assert DockerCommandLineJob(
+        builder, {}, CommandLineTool.make_path_mapper, [], [], ""
+    ).get_image(
+        {
+            "class": "DockerRequirement",
+            "dockerFile": "FROM debian:stable-slim",
+            "dockerImageId": docker_image_id,
+        },
+        pull_image=False,
+        force_pull=False,
+        tmp_outdir_prefix=str(tmp_outdir_prefix),
+    )
+    output = subprocess.check_output(
+        ["docker", "images", "--quiet", docker_image_id], stderr=subprocess.STDOUT, text=True
+    )
+
+    # If the output is empty, the image doesn't exist
+    assert output.strip(), f"Docker image {docker_image_id} does not exist"
+
+
 @needs_singularity
 def test_dockerfile_singularity_build(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     """Test that SingularityCommandLineJob.get_image builds a Dockerfile with Singularity."""