mypyc: enable subclassing and/or pickling of popular classes

Fixes: DataBiosphere/toil#4394

Author: mr-c

cwltool/builder.py

@@ -19,15 +19,15 @@
 
 from cwl_utils import expression
 from cwl_utils.file_formats import check_format
+from mypy_extensions import mypyc_attr
 from rdflib import Graph
+from ruamel.yaml.comments import CommentedMap
 from schema_salad.avro.schema import Names, Schema, make_avsc_object
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import convert_to_dict, json_dumps
 from schema_salad.validate import validate
 
-from ruamel.yaml.comments import CommentedMap
-
 from .errors import WorkflowException
 from .loghandler import _logger
 from .mutation import MutationManager
@@ -94,6 +94,7 @@ def substitute(value: str, replace: str) -> str:
     return value + replace
 
 
+@mypyc_attr(allow_interpreted_subclasses=True)
 class Builder(HasReqsHints):
     """Helper class to construct a command line from a CWL CommandLineTool."""
 

cwltool/command_line_tool.py

@@ -32,15 +32,15 @@
 )
 
 import shellescape
+from mypy_extensions import mypyc_attr
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import Schema
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import file_uri, uri_file_path
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
 from schema_salad.validate import validate_ex
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-
 from .builder import (
     INPUT_OBJ_VOCAB,
     Builder,
@@ -208,6 +208,7 @@ def run(
                 self.output_callback({}, "permanentFail")
 
 
+@mypyc_attr(allow_interpreted_subclasses=True)
 class ExpressionTool(Process):
     def job(
         self,
@@ -315,6 +316,7 @@ def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> Optional[CWL
     )
 
 
+@mypyc_attr(serializable=True)
 class CallbackJob:
     """Callback Job class, used by :py:func:`CommandLineTool.job`."""
 
@@ -401,6 +403,7 @@ def __init__(self, msg: str, port: CWLObjectType, **kwargs: Any) -> None:
         )
 
 
+@mypyc_attr(allow_interpreted_subclasses=True)
 class CommandLineTool(Process):
     def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext) -> None:
         """Initialize this CommandLineTool."""
@@ -470,8 +473,8 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]:
             )
         return CommandLineJob
 
+    @staticmethod
     def make_path_mapper(
-        self,
         reffiles: List[CWLObjectType],
         stagedir: str,
         runtimeContext: RuntimeContext,

cwltool/context.py

@@ -18,13 +18,12 @@
     Union,
 )
 
+from ruamel.yaml.comments import CommentedMap
 from schema_salad.avro.schema import Names
 from schema_salad.ref_resolver import Loader
 from schema_salad.utils import FetcherCallableType
 from typing_extensions import Literal
 
-from ruamel.yaml.comments import CommentedMap
-
 from .mpi import MpiConfig
 from .pathmapper import PathMapper
 from .stdfsaccess import StdFsAccess

cwltool/cwlrdf.py

@@ -4,11 +4,10 @@
 
 from rdflib import Graph
 from rdflib.query import ResultRow
+from ruamel.yaml.comments import CommentedMap
 from schema_salad.jsonld_context import makerdf
 from schema_salad.utils import ContextType
 
-from ruamel.yaml.comments import CommentedMap
-
 from .cwlviewer import CWLViewer
 from .process import Process
 

cwltool/docker.py

@@ -83,7 +83,7 @@ def __init__(
         self,
         builder: Builder,
         joborder: CWLObjectType,
-        make_path_mapper: Callable[..., PathMapper],
+        make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper],
         requirements: List[CWLObjectType],
         hints: List[CWLObjectType],
         name: str,
@@ -452,7 +452,7 @@ def __init__(
         self,
         builder: Builder,
         joborder: CWLObjectType,
-        make_path_mapper: Callable[..., PathMapper],
+        make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper],
         requirements: List[CWLObjectType],
         hints: List[CWLObjectType],
         name: str,

cwltool/executors.py

@@ -20,6 +20,7 @@
 )
 
 import psutil
+from mypy_extensions import mypyc_attr
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import SourceLine
 
@@ -40,6 +41,7 @@
 TMPDIR_LOCK = Lock()
 
 
+@mypyc_attr(allow_interpreted_subclasses=True)
 class JobExecutor(metaclass=ABCMeta):
     """Abstract base job executor."""
 
@@ -178,6 +180,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None:
         return (None, "permanentFail")
 
 
+@mypyc_attr(allow_interpreted_subclasses=True)
 class SingleJobExecutor(JobExecutor):
     """Default single-threaded CWL reference executor."""
 

cwltool/job.py

@@ -118,7 +118,7 @@ def __init__(
         self,
         builder: Builder,
         joborder: CWLObjectType,
-        make_path_mapper: Callable[..., PathMapper],
+        make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper],
         requirements: List[CWLObjectType],
         hints: List[CWLObjectType],
         name: str,

cwltool/load_tool.py

@@ -21,6 +21,7 @@
 )
 
 from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
 from schema_salad.fetcher import Fetcher
 from schema_salad.ref_resolver import Loader, file_uri
@@ -34,8 +35,6 @@
     json_dumps,
 )
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-
 from . import CWL_CONTENT_TYPES, process, update
 from .context import LoadingContext
 from .errors import GraphTargetMissingException

cwltool/main.py

@@ -34,6 +34,9 @@
 import argcomplete
 import coloredlogs
 import pkg_resources  # part of setuptools
+import ruamel.yaml
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+from ruamel.yaml.main import YAML
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
 from schema_salad.sourceline import cmap, strip_dup_lineno
@@ -45,10 +48,6 @@
     yaml_no_ts,
 )
 
-import ruamel.yaml
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from ruamel.yaml.main import YAML
-
 from . import CWL_CONTENT_TYPES, workflow
 from .argparser import arg_parser, generate_parser, get_default_args
 from .context import LoadingContext, RuntimeContext, getdefault

cwltool/pack.py

@@ -14,11 +14,10 @@
     cast,
 )
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.ref_resolver import Loader, SubLoader
 from schema_salad.utils import ResolveType
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-
 from .context import LoadingContext
 from .load_tool import fetch_document, resolve_and_validate_document
 from .process import shortname, uniquename