Merge pull request #109 from manu-chroma/modernize

Python3 compatible codebase !

Author: mr-c

.gitignore

@@ -2,3 +2,11 @@
 .eggs
 *.egg-info/
 *pyc
+
+build/
+dist/
+ruamel
+typeshed/2and3/ruamel/yaml
+# virtualenv 
+venv/
+.cache/

.travis.yml

@@ -2,7 +2,10 @@ sudo: false
 language: python
 python:
   - 2.7
+  - 3.3
+  - 3.4
   - 3.5
+  - 3.6
 os:
   - linux
 

Makefile

@@ -165,12 +165,20 @@ list-author-emails:
 	@echo 'name, E-Mail Address'
 	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
 
-mypy: ${PYSOURCES}
-	rm -Rf typeshed/2.7/ruamel/yaml
+mypy2: ${PYSOURCES}
+	rm -Rf typeshed/2and3/ruamel/yaml
 	ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-		typeshed/2.7/ruamel/
-	MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
-		 --fast-parser --warn-redundant-casts --warn-unused-ignores \
+		typeshed/2and3/ruamel/
+	MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
+		 --warn-redundant-casts --warn-unused-ignores \
+		 schema_salad
+
+mypy3: ${PYSOURCES}
+	rm -Rf typeshed/2and3/ruamel/yaml
+	ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
+		typeshed/2and3/ruamel/
+	MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
+		 --warn-redundant-casts \
 		 schema_salad
 
 jenkins:

mypy.ini

@@ -0,0 +1,2 @@
+[mypy-ruamel.*]
+ignore_errors = True

mypy_requirements.txt

@@ -1,2 +1 @@
-mypy==0.470 ;  python_version>="3"
-typed-ast==0.6.3 ; python_version>="3"
+mypy==0.511 ;  python_version>="3"

requirements.txt

@@ -1,7 +1,7 @@
-typing==3.5.2.2 ; python_version>="2.7"
+typing==3.5.3
 avro-python3 ; python_version>="3"
 avro==1.8.1 ; python_version<"3"
-ruamel.yaml==0.13.7
+ruamel.yaml==0.15.2
 rdflib==4.2.2
 rdflib-jsonld==0.4.0
 mistune==0.7.3

schema_salad/__init__.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import logging
 import sys
 import typing

schema_salad/__main__.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from . import main
 import sys
 import typing

schema_salad/jsonld_context.py

@@ -1,6 +1,11 @@
+from __future__ import absolute_import
 import collections
 import shutil
 import json
+
+import six
+from six.moves import urllib
+
 import ruamel.yaml as yaml
 try:
     from ruamel.yaml import CSafeLoader as SafeLoader
@@ -16,7 +21,6 @@
 from rdflib import Graph, URIRef
 import rdflib.namespace
 from rdflib.namespace import RDF, RDFS
-import urlparse
 import logging
 from schema_salad.utils import aslist
 from typing import (cast, Any, Dict, Iterable, List, Optional, Text, Tuple,
@@ -34,16 +38,16 @@ def pred(datatype,      # type: Dict[str, Union[Dict, str]]
          namespaces     # type: Dict[str, rdflib.namespace.Namespace]
          ):
     # type: (...) -> Union[Dict, Text]
-    split = urlparse.urlsplit(name)
+    split = urllib.parse.urlsplit(name)
 
-    vee = None  # type: Optional[Union[str, unicode]]
+    vee = None  # type: Optional[Text]
 
     if split.scheme != '':
         vee = name
-        (ns, ln) = rdflib.namespace.split_uri(unicode(vee))
+        (ns, ln) = rdflib.namespace.split_uri(six.text_type(vee))
         name = ln
         if ns[0:-1] in namespaces:
-            vee = unicode(namespaces[ns[0:-1]][ln])
+            vee = six.text_type(namespaces[ns[0:-1]][ln])
         _logger.debug("name, v %s %s", name, vee)
 
     v = None  # type: Optional[Dict]
@@ -102,11 +106,11 @@ def process_type(t,             # type: Dict[str, Any]
         classnode = URIRef(recordname)
         g.add((classnode, RDF.type, RDFS.Class))
 
-        split = urlparse.urlsplit(recordname)
+        split = urllib.parse.urlsplit(recordname)
         predicate = recordname
         if t.get("inVocab", True):
             if split.scheme:
-                (ns, ln) = rdflib.namespace.split_uri(unicode(recordname))
+                (ns, ln) = rdflib.namespace.split_uri(six.text_type(recordname))
                 predicate = recordname
                 recordname = ln
             else:
@@ -129,15 +133,15 @@ def process_type(t,             # type: Dict[str, Any]
             _logger.debug("Processing field %s", i)
 
             v = pred(t, i, fieldname, context, defaultPrefix,
-                    namespaces)  # type: Union[Dict[Any, Any], unicode, None]
+                    namespaces)  # type: Union[Dict[Any, Any], Text, None]
 
-            if isinstance(v, basestring):
+            if isinstance(v, six.string_types):
                 v = v if v[0] != "@" else None
             elif v is not None:
                 v = v["_@id"] if v.get("_@id", "@")[0] != "@" else None
 
             if bool(v):
-                (ns, ln) = rdflib.namespace.split_uri(unicode(v))
+                (ns, ln) = rdflib.namespace.split_uri(six.text_type(v))
                 if ns[0:-1] in namespaces:
                     propnode = namespaces[ns[0:-1]][ln]
                 else:
@@ -188,8 +192,8 @@ def salad_to_jsonld_context(j, schema_ctx):
     return (context, g)
 
 
-def fix_jsonld_ids(obj,     # type: Union[Dict[unicode, Any], List[Dict[unicode, Any]]]
-                   ids      # type: List[unicode]
+def fix_jsonld_ids(obj,     # type: Union[Dict[Text, Any], List[Dict[Text, Any]]]
+                   ids      # type: List[Text]
                    ):
     # type: (...) -> None
     if isinstance(obj, dict):
@@ -203,22 +207,22 @@ def fix_jsonld_ids(obj,     # type: Union[Dict[unicode, Any], List[Dict[unicode,
             fix_jsonld_ids(entry, ids)
 
 
-def makerdf(workflow,       # type: Union[str, unicode]
-            wf,             # type: Union[List[Dict[unicode, Any]], Dict[unicode, Any]]
+def makerdf(workflow,       # type: Text
+            wf,             # type: Union[List[Dict[Text, Any]], Dict[Text, Any]]
             ctx,            # type: ContextType
             graph=None      # type: Graph
             ):
     # type: (...) -> Graph
     prefixes = {}
     idfields = []
-    for k, v in ctx.iteritems():
+    for k, v in six.iteritems(ctx):
         if isinstance(v, dict):
             url = v["@id"]
         else:
             url = v
         if url == "@id":
             idfields.append(k)
-        doc_url, frg = urlparse.urldefrag(url)
+        doc_url, frg = urllib.parse.urldefrag(url)
         if "/" in frg:
             p = frg.split("/")[0]
             prefixes[p] = u"%s#%s/" % (doc_url, p)
@@ -242,7 +246,7 @@ def makerdf(workflow,       # type: Union[str, unicode]
     for sub, pred, obj in g.triples((None, URIRef("@id"), None)):
         g.remove((sub, pred, obj))
 
-    for k2, v2 in prefixes.iteritems():
+    for k2, v2 in six.iteritems(prefixes):
         g.namespace_manager.bind(k2, v2)
 
     return g

schema_salad/main.py

@@ -1,15 +1,17 @@
 from __future__ import print_function
+from __future__ import absolute_import
 import argparse
 import logging
 import sys
 import traceback
 import json
 import os
-import urlparse
+
+from six.moves import urllib
 
 import pkg_resources  # part of setuptools
 
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Union, Text
 
 from rdflib import Graph, plugin
 from rdflib.serializer import Serializer
@@ -20,6 +22,7 @@
 from . import validate
 from .sourceline import strip_dup_lineno
 from .ref_resolver import Loader
+import six
 
 _logger = logging.getLogger("salad")
 
@@ -28,8 +31,8 @@
 
 
 def printrdf(workflow,  # type: str
-             wf,        # type: Union[List[Dict[unicode, Any]], Dict[unicode, Any]]
-             ctx,       # type: Dict[unicode, Any]
+             wf,        # type: Union[List[Dict[Text, Any]], Dict[Text, Any]]
+             ctx,       # type: Dict[Text, Any]
              sr         # type: str
              ):
     # type: (...) -> None
@@ -108,7 +111,7 @@ def main(argsl=None):  # type: (List[str]) -> int
     # Load schema document and resolve refs
 
     schema_uri = args.schema
-    if not urlparse.urlparse(schema_uri)[0]:
+    if not urllib.parse.urlparse(schema_uri)[0]:
         schema_uri = "file://" + os.path.abspath(schema_uri)
     schema_raw_doc = metaschema_loader.fetch(schema_uri)
 
@@ -118,8 +121,8 @@ def main(argsl=None):  # type: (List[str]) -> int
     except (validate.ValidationException) as e:
         _logger.error("Schema `%s` failed link checking:\n%s",
                       args.schema, e, exc_info=(True if args.debug else False))
-        _logger.debug("Index is %s", metaschema_loader.idx.keys())
-        _logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys())
+        _logger.debug("Index is %s", list(metaschema_loader.idx.keys()))
+        _logger.debug("Vocabulary is %s", list(metaschema_loader.vocab.keys()))
         return 1
     except (RuntimeError) as e:
         _logger.error("Schema `%s` read error:\n%s",
@@ -132,7 +135,7 @@ def main(argsl=None):  # type: (List[str]) -> int
         return 0
 
     if not args.document and args.print_index:
-        print(json.dumps(metaschema_loader.idx.keys(), indent=4))
+        print(json.dumps(list(metaschema_loader.idx.keys()), indent=4))
         return 0
 
     # Validate the schema document against the metaschema
@@ -205,12 +208,12 @@ def main(argsl=None):  # type: (List[str]) -> int
     # Load target document and resolve refs
     try:
         uri = args.document
-        if not urlparse.urlparse(uri)[0]:
+        if not urllib.parse.urlparse(uri)[0]:
             doc = "file://" + os.path.abspath(uri)
         document, doc_metadata = document_loader.resolve_ref(uri)
     except (validate.ValidationException, RuntimeError) as e:
         _logger.error("Document `%s` failed validation:\n%s",
-                      args.document, strip_dup_lineno(unicode(e)), exc_info=args.debug)
+                      args.document, strip_dup_lineno(six.text_type(e)), exc_info=args.debug)
         return 1
 
     # Optionally print the document after ref resolution
@@ -219,7 +222,7 @@ def main(argsl=None):  # type: (List[str]) -> int
         return 0
 
     if args.print_index:
-        print(json.dumps(document_loader.idx.keys(), indent=4))
+        print(json.dumps(list(document_loader.idx.keys()), indent=4))
         return 0
 
     # Validate the schema document against the metaschema