Update schema-salad dependency for bug fix (#1463)

* Update schema-salad dependency for bug fix

Specifically common-workflow-language/schema_salad#408

* Use vocabulary in schema salad validation
* Fixes internal Avro type change from File to org.w3id.cwl.cwl.File

Co-authored-by: Michael R. Crusoe <[email protected]>
Co-authored-by: Michael R. Crusoe <[email protected]>

Author: tetron

cwltool/builder.py

@@ -46,6 +46,12 @@
     from .pathmapper import PathMapper
     from .provenance_profile import ProvenanceProfile  # pylint: disable=unused-import
 
+INPUT_OBJ_VOCAB: Dict[str, str] = {
+    "Any": "https://w3id.org/cwl/salad#Any",
+    "File": "https://w3id.org/cwl/cwl#File",
+    "Directory": "https://w3id.org/cwl/cwl#Directory",
+}
+
 
 def content_limit_respected_read_bytes(f):  # type: (IO[bytes]) -> bytes
     contents = f.read(CONTENT_LIMIT + 1)
@@ -272,7 +278,7 @@ def bind_input(
                     avsc = self.names.get_name(cast(str, t["name"]), None)
                 if not avsc:
                     avsc = make_avsc_object(convert_to_dict(t), self.names)
-                if validate(avsc, datum):
+                if validate(avsc, datum, vocab=INPUT_OBJ_VOCAB):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t
                     if not value_from_expression:
@@ -376,7 +382,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType:
                 self.files.append(f)
                 return f
 
-            if schema["type"] == "File":
+            if schema["type"] == "org.w3id.cwl.cwl.File":
                 datum = cast(CWLObjectType, datum)
                 self.files.append(datum)
 
@@ -516,7 +522,7 @@ def addsf(
                     _capture_files,
                 )
 
-            if schema["type"] == "Directory":
+            if schema["type"] == "org.w3id.cwl.cwl.Directory":
                 datum = cast(CWLObjectType, datum)
                 ll = schema.get("loadListing") or self.loadListing
                 if ll and ll != "no_listing":

cwltool/command_line_tool.py

@@ -40,7 +40,12 @@
 from schema_salad.validate import validate_ex
 from typing_extensions import TYPE_CHECKING, Type
 
-from .builder import Builder, content_limit_respected_read_bytes, substitute
+from .builder import (
+    Builder,
+    content_limit_respected_read_bytes,
+    substitute,
+    INPUT_OBJ_VOCAB,
+)
 from .context import LoadingContext, RuntimeContext, getdefault
 from .docker import DockerCommandLineJob
 from .errors import UnsupportedRequirement, WorkflowException
@@ -1110,7 +1115,11 @@ def collect_output_ports(
                 Schema, self.names.get_name("outputs_record_schema", None)
             )
             validate_ex(
-                expected_schema, ret, strict=False, logger=_logger_validation_warnings
+                expected_schema,
+                ret,
+                strict=False,
+                logger=_logger_validation_warnings,
+                vocab=INPUT_OBJ_VOCAB,
             )
             if ret is not None and builder.mutation_manager is not None:
                 adjustFileObjs(ret, builder.mutation_manager.set_generation)

cwltool/expression.py

@@ -134,7 +134,7 @@ def next_seg(
         m = segment_re.match(remaining_string)
         if not m:
             return current_value
-        next_segment_str = m.group(0)
+        next_segment_str = m.group(1)
 
         key = None  # type: Optional[Union[str, int]]
         if next_segment_str[0] == ".":
@@ -146,7 +146,7 @@ def next_seg(
             if (
                 isinstance(current_value, MutableSequence)
                 and key == "length"
-                and not remaining_string[m.end(0) :]
+                and not remaining_string[m.end(1) :]
             ):
                 return len(current_value)
             if not isinstance(current_value, MutableMapping):
@@ -175,7 +175,7 @@ def next_seg(
             try:
                 return next_seg(
                     parsed_string + remaining_string,
-                    remaining_string[m.end(0) :],
+                    remaining_string[m.end(1) :],
                     cast(CWLOutputType, current_value[cast(str, key)]),
                 )
             except KeyError:
@@ -184,7 +184,7 @@ def next_seg(
             try:
                 return next_seg(
                     parsed_string + remaining_string,
-                    remaining_string[m.end(0) :],
+                    remaining_string[m.end(1) :],
                     current_value[key],
                 )
             except KeyError:

cwltool/process.py

@@ -44,12 +44,12 @@
 from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
 from schema_salad.schema import load_schema, make_avro_schema, make_valid_avro
 from schema_salad.sourceline import SourceLine, strip_dup_lineno
-from schema_salad.utils import convert_to_dict
-from schema_salad.validate import validate_ex
+from schema_salad.utils import ContextType, convert_to_dict
+from schema_salad.validate import validate_ex, avro_type_name
 from typing_extensions import TYPE_CHECKING
 
 from . import expression
-from .builder import Builder, HasReqsHints
+from .builder import Builder, HasReqsHints, INPUT_OBJ_VOCAB
 from .context import LoadingContext, RuntimeContext, getdefault
 from .errors import UnsupportedRequirement, WorkflowException
 from .loghandler import _logger
@@ -457,30 +457,34 @@ def fill_in_defaults(
 
 
 def avroize_type(
-    field_type: Union[
-        CWLObjectType, MutableSequence[CWLOutputType], CWLOutputType, None
-    ],
+    field_type: Union[CWLObjectType, MutableSequence[Any], CWLOutputType, None],
     name_prefix: str = "",
-) -> None:
+) -> Union[CWLObjectType, MutableSequence[Any], CWLOutputType, None]:
     """Add missing information to a type so that CWL types are valid."""
     if isinstance(field_type, MutableSequence):
-        for field in field_type:
-            avroize_type(field, name_prefix)
+        for i, field in enumerate(field_type):
+            field_type[i] = avroize_type(field, name_prefix)
     elif isinstance(field_type, MutableMapping):
         if field_type["type"] in ("enum", "record"):
             if "name" not in field_type:
                 field_type["name"] = name_prefix + str(uuid.uuid4())
         if field_type["type"] == "record":
-            avroize_type(
+            field_type["fields"] = avroize_type(
                 cast(MutableSequence[CWLOutputType], field_type["fields"]), name_prefix
             )
-        if field_type["type"] == "array":
-            avroize_type(
+        elif field_type["type"] == "array":
+            field_type["items"] = avroize_type(
                 cast(MutableSequence[CWLOutputType], field_type["items"]), name_prefix
             )
-        if isinstance(field_type["type"], MutableSequence):
-            for ctype in field_type["type"]:
-                avroize_type(cast(CWLOutputType, ctype), name_prefix)
+        else:
+            field_type["type"] = avroize_type(
+                cast(CWLOutputType, field_type["type"]), name_prefix
+            )
+    elif field_type == "File":
+        return "org.w3id.cwl.cwl.File"
+    elif field_type == "Directory":
+        return "org.w3id.cwl.cwl.Directory"
+    return field_type
 
 
 def get_overrides(
@@ -635,6 +639,7 @@ def __init__(
                 sdtypes,
                 {cast(str, t["name"]): cast(Dict[str, Any], t) for t in sdtypes},
                 set(),
+                vocab=INPUT_OBJ_VOCAB,
             )
             for i in av:
                 self.schemaDefs[i["name"]] = i  # type: ignore
@@ -669,7 +674,8 @@ def __init__(
                     c["type"] = nullable
                 else:
                     c["type"] = c["type"]
-                avroize_type(c["type"], c["name"])
+
+                c["type"] = avroize_type(c["type"], c["name"])
                 if key == "inputs":
                     cast(
                         List[CWLObjectType], self.inputs_record_schema["fields"]
@@ -709,9 +715,13 @@ def __init__(
                     )
                     raise
             if self.doc_schema is not None:
+                classname = toolpath_object["class"]
+                avroname = classname
+                if self.doc_loader and classname in self.doc_loader.vocab:
+                    avroname = avro_type_name(self.doc_loader.vocab[classname])
                 validate_js_expressions(
                     toolpath_object,
-                    self.doc_schema.names[toolpath_object["class"]],
+                    self.doc_schema.names[avroname],
                     validate_js_options,
                 )
 
@@ -778,7 +788,13 @@ def _init_job(
                 raise WorkflowException(
                     "Missing input record schema: " "{}".format(self.names)
                 )
-            validate_ex(schema, job, strict=False, logger=_logger_validation_warnings)
+            validate_ex(
+                schema,
+                job,
+                strict=False,
+                logger=_logger_validation_warnings,
+                vocab=INPUT_OBJ_VOCAB,
+            )
 
             if load_listing and load_listing != "no_listing":
                 get_listing(fs_access, job, recursive=(load_listing == "deep_listing"))
@@ -1015,13 +1031,16 @@ def evalResources(
     def validate_hints(
         self, avsc_names: Names, hints: List[CWLObjectType], strict: bool
     ) -> None:
+        if self.doc_loader is None:
+            return
         for i, r in enumerate(hints):
             sl = SourceLine(hints, i, ValidationException)
             with sl:
-                if (
-                    avsc_names.get_name(cast(str, r["class"]), None) is not None
-                    and self.doc_loader is not None
-                ):
+                classname = cast(str, r["class"])
+                avroname = classname
+                if classname in self.doc_loader.vocab:
+                    avroname = avro_type_name(self.doc_loader.vocab[classname])
+                if avsc_names.get_name(avroname, None) is not None:
                     plain_hint = {
                         key: r[key]
                         for key in r
@@ -1030,10 +1049,11 @@ def validate_hints(
                     validate_ex(
                         cast(
                             Schema,
-                            avsc_names.get_name(cast(str, plain_hint["class"]), None),
+                            avsc_names.get_name(avroname, None),
                         ),
                         plain_hint,
                         strict=strict,
+                        vocab=self.doc_loader.vocab,
                     )
                 elif r["class"] in ("NetworkAccess", "LoadListingRequirement"):
                     pass

cwltool/validate_js.py

@@ -39,7 +39,7 @@ def is_expression(tool, schema):
     # type: (Any, Optional[Schema]) -> bool
     return (
         isinstance(schema, EnumSchema)
-        and schema.name == "Expression"
+        and schema.name == "org.w3id.cwl.cwl.Expression"
         and isinstance(tool, str)
     )
 
@@ -76,6 +76,7 @@ def get_expressions(
                 tool,
                 raise_ex=False,
                 logger=_logger_validation_warnings,
+                vocab={},
             ):
                 valid_schema = possible_schema
 

requirements.txt

@@ -2,7 +2,7 @@ requests>=2.4.3
 ruamel.yaml>=0.15,<0.17.11
 rdflib>=4.2.2,<5.1
 shellescape>=3.4.1,<3.9
-schema-salad>=7.1.20210518142926,<8
+schema-salad>=8.1.20210627200047,<9
 prov==1.5.1
 bagit==1.8.1
 mypy-extensions

setup.py

@@ -113,7 +113,7 @@
         "shellescape >= 3.4.1, < 3.9",
         # 7.1.20210518142926 or later required due to
         # https://github.com/common-workflow-language/schema_salad/issues/385
-        "schema-salad >= 7.1.20210518142926, < 8",
+        "schema-salad >= 8.1.20210627200047, < 9",
         "mypy-extensions",
         "psutil >= 5.6.6",
         "prov == 1.5.1",

tests/test_procgenerator.py

@@ -20,6 +20,7 @@ def test_missing_enable_ext(monkeypatch: pytest.MonkeyPatch) -> None:
     assert (
         main(
             [
+                "--debug",
                 "--enable-ext",
                 "--enable-dev",
                 get_data("tests/wf/generator/zing.cwl"),

tests/test_validate_js.py

@@ -28,7 +28,7 @@ def test_get_expressions() -> None:
     test_cwl_yaml = yaml.load(TEST_CWL)
     schema = process.get_schema("v1.0")[1]
     assert isinstance(schema, Names)
-    clt_schema = schema.names["CommandLineTool"]
+    clt_schema = schema.names["org.w3id.cwl.cwl.CommandLineTool"]
 
     exprs = validate_js.get_expressions(test_cwl_yaml, clt_schema)
 
@@ -40,7 +40,7 @@ def test_validate_js_expressions(mocker: Any) -> None:
     test_cwl_yaml = yaml.load(TEST_CWL)
     schema = process.get_schema("v1.0")[1]
     assert isinstance(schema, Names)
-    clt_schema = schema.names["CommandLineTool"]
+    clt_schema = schema.names["org.w3id.cwl.cwl.CommandLineTool"]
 
     mocker.patch("cwltool.validate_js._logger")
     # mocker.patch("cwltool.validate_js.print_js_hint_messages")

tests/wf/generator/pytoolgen.cwl

@@ -2,7 +2,7 @@
 cwlVersion: v1.0
 $namespaces:
   cwltool: "http://commonwl.org/cwltool#"
-class: ProcessGenerator
+class: cwltool:ProcessGenerator
 inputs:
   script: string
   dir: Directory