Merge branch 'develop' of https://github.com/TIA-Lab/tiatoolbox into example-semantic

Author: mostafajahanifar

.pre-commit-config.yaml

@@ -8,38 +8,30 @@ repos:
     hooks:
       - id: black
         language_version: python3 # Should be a command that runs python3.7+
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
-    hooks:
-      - id: flake8
-        additional_dependencies: [
-          flake8-spellcheck, # Spelling checker
-        ]
-        args: [--exit-zero]  # These checks will not cause the commit to fail
-        verbose: true
   - repo: https://gitlab.com/pycqa/flake8
     rev: 3.9.2
     hooks:
       - id: flake8
         additional_dependencies: [
           flake8-bugbear, # Detect potential bugs
-          flake8-isort, # Check import ordering
-          flake8-comprehensions, # Suggestions for better list/set/dict comprehensions 
-          flake8-mutable, # Check for mutable default arguments
-          flake8-use-fstring, # Encourages use of f-strings vs old style
-          flake8-simplify, # Suggestions to simplify code
-          pep8-naming, # Check PEP8 class naming
-          flake8-eradicate, # Find dead/commented out code
           flake8-builtins, # Check for built-ins being used as variables
           flake8-cognitive-complexity, # Check max function complexity
+          flake8-comprehensions, # Suggestions for better list/set/dict comprehensions 
+          flake8-eradicate, # Find dead/commented out code
           flake8-expression-complexity, # Check max expression complexity
-          flake8-return, # Check return statements
-          flake8-pytest-style, # Check against pytest style guide
           flake8-fixme, # Check for FIXME, TODO, and XXX left in comments
-          flake8-sql, # Check SQL statement style
-          nitpick, # Check that black, flake8, isort have compaible options
+          flake8-isort, # Check import ordering
           flake8-logging-format, # Validate (lack of) logging format strings
+          flake8-mutable, # Check for mutable default arguments
           flake8-pie, # Misc. linting rules
+          flake8-pytest-style, # Check against pytest style guide
+          flake8-return, # Check return statements
+          flake8-simplify, # Suggestions to simplify code
+          flake8-spellcheck, # Spelling checker
+          flake8-sql, # Check SQL statement style
+          flake8-use-fstring, # Encourages use of f-strings vs old style
+          nitpick, # Check that black, flake8, isort have compaible options
+          pep8-naming, # Check PEP8 class naming
         ]
   - repo: local
     hooks:

requirements.conda.yml

@@ -8,24 +8,28 @@ dependencies:
   - python=3.7
   - Click
   - cython
+  - defusedxml
+  - glymur
   - h5py
+  - imagecodecs
+  - jupyterlab
   - matplotlib
   - numpy
   - opencv>=4.0
   - openjpeg>=2.3.0
+  - openslide
+  - pandas
   - pillow
   - pip
+  - pixman<0.38.0
+  - pytorch
   - pyyaml>=5.1
   - requests
-  - pixman<0.38.0
-  - openslide
-  - pandas
-  - glymur
-  - scikit-learn>=0.23.2
   - scikit-image
+  - scikit-learn>=0.23.2
+  - tifffile
   - torchvision
-  - pytorch
   - tqdm
-  - jupyterlab
+  - zarr
   - pip:
     - openslide-python==1.1.2

requirements.dev.conda.yml

@@ -5,36 +5,40 @@ channels:
   - pytorch
   - defaults
 dependencies:
-  - python=3.7
+  - black
   - Click
   - cython
+  - defusedxml
+  - flake8
+  - flake8-bugbear
+  - glymur
   - h5py
+  - imagecodecs
+  - jupyterlab
   - matplotlib
   - numpy
   - opencv>=4.0
+  - openjpeg>=2.3.0
+  - openslide
+  - pandas
   - pillow
   - pip
-  - pyyaml>=5.1
-  - requests
   - pixman<0.38.0
-  - openslide
-  - openjpeg>=2.3.0
+  - pre-commit
+  - pytest-cov
+  - pytest-runner==5.2
+  - pytest==6.2.5
+  - python=3.7
+  - pytorch
+  - pyyaml>=5.1
   - recommonmark
-  - pandas
-  - glymur
+  - requests
+  - scikit-image
   - scikit-learn>=0.23.2
+  - tifffile
+  - torchvision
   - torchvision>=0.10.1
-  - pytorch
   - tqdm
-  - pytest==6.2.5
-  - pytest-runner==5.2
-  - scikit-image
-  - pre-commit
-  - flake8
-  - flake8-bugbear
-  - black
-  - pytest-cov
-  - jupyterlab
-  - openjpeg
+  - zarr
   - pip:
       - openslide-python==1.1.2

requirements.txt

@@ -1,17 +1,23 @@
 Click>=7.0
-numpy
-pillow
+defusedxml
+glymur
+imagecodecs
+jupyterlab
 matplotlib
+numpy
 opencv-python>=4.0
 openslide-python==1.1.2
-pyyaml>=5.1
 pandas
-glymur
-scikit-learn>=0.23.2
+pillow
+pyyaml>=5.1
+requests
 scikit-image
+scikit-learn>=0.23.2
 sphinx==4.1.2
+tifffile
 torchvision==0.10.1
 torch==1.9.1
 tqdm==4.60.0
 jupyterlab
 requests
+zarr

requirements.win64.conda.yml

@@ -8,23 +8,27 @@ dependencies:
   - python=3.7
   - Click
   - cython
+  - defusedxml
+  - glymur
   - h5py
+  - imagecodecs
+  - jupyterlab
   - matplotlib
   - numpy
   - opencv>=4.0
+  - pandas
   - pillow
   - openjpeg>=2.3.0
   - pip
+  - pixman<0.38.0
+  - pytorch
   - pyyaml>=5.1
   - requests
-  - pixman<0.38.0
-  - pandas
-  - glymur
-  - scikit-learn>=0.23.2
   - scikit-image
+  - scikit-learn>=0.23.2
+  - tifffile
   - torchvision
-  - pytorch
   - tqdm
-  - jupyterlab
+  - zarr
   - pip:
     - openslide-python

requirements_dev.txt

@@ -1,32 +1,35 @@
-pip>=20.0.2
+black
 bump2version==0.5.11
-wheel==0.33.6
-watchdog==0.9.0
-flake8==3.7.8
-tox==3.14.0
-coverage==5.1
-sphinx==4.1.2
-twine==1.14.0
 Click>=7.0
-pytest==6.2.5
-pytest-runner==5.2
+coverage==5.1
+flake8==3.7.8
+flake8-bugbear
+glymur
+imagecodecs
+jupyterlab
+matplotlib
+numpy
 opencv-python>=4.0
 openslide-python==1.1.2
-pytest-cov==2.9.0
-numpy
+pandas
+pip>=20.0.2
 pillow
-matplotlib
+pre-commit
+pytest==6.2.5
+pytest-runner==5.2
+pytest-cov==2.9.0
 pyyaml>=5.1
-pandas
-glymur
+recommonmark
+requests
 scikit-image
 scikit-learn>=0.23.2
-recommonmark
-pre-commit
-black
-flake8-bugbear
-torchvision==0.10.1
+sphinx==4.1.2
+tifffile
+tox==3.14.0
 torch==1.9.1
+torchvision==0.10.1
 tqdm==4.60.0
-jupyterlab
-requests
+twine==1.14.0
+wheel==0.33.6
+watchdog==0.9.0
+zarr

setup.cfg

@@ -20,7 +20,7 @@ max-line-length = 88
 extend-ignore = E203 # For black compatibility
 spellcheck-targets = comments
 dictionaries = en_US,python,technical
-max-cognitive-complexity = 7
+max-cognitive-complexity = 11
 max-expression-complexity = 7
 
 [aliases]

setup.py

@@ -25,6 +25,9 @@
     "torchvision==0.10.1",
     "torch==1.9.1",
     "tqdm==4.60.0",
+    "tifffile",
+    "imagecodecs",
+    "zarr",
     "requests",
 ]
 

tests/models/test_abc.py

@@ -27,6 +27,7 @@
 from tiatoolbox.models.architecture import get_pretrained_model
 
 
+@pytest.mark.skip(reason="Local test, not applicable for travis.")
 def test_get_pretrained_model():
     """Test for downloading and creating pretrained models."""
     pretrained_info = rcParam["pretrained_model_info"]

tests/test_data.py

@@ -17,21 +17,48 @@ def test_fetch_sample(tmp_path):
     tmp_path = pathlib.Path(tmp_path)
     path = _fetch_remote_sample("stainnorm-source")
     assert os.path.exists(path)
-    # test if corrupted
+    # Test if corrupted
     get_wsireader(path)
 
     path = _fetch_remote_sample("stainnorm-source", tmp_path)
-    # assuming Path has no trailing '/'
+    # Assuming Path has no trailing '/'
     assert os.path.exists(path)
     assert str(tmp_path) in str(path)
 
-    # test not directory path
+    # Test not directory path
     test_path = pathlib.Path(f"{tmp_path}/dummy.npy")
     np.save(test_path, np.zeros([3, 3, 3]))
     with pytest.raises(ValueError, match=r".*tmp_path must be a directory.*"):
         path = _fetch_remote_sample("wsi1_8k_8k_svs", test_path)
 
-    #  very tiny so temporary hook here also
+    # Very tiny so temporary hook here also
+    arr = stainnorm_target()
+    assert isinstance(arr, np.ndarray)
+
+
+def test_fetch_sample_skip(tmp_path):
+    """Test skipping fetching sample via code name if already downloaded."""
+    # Fetch the remote file twice
+    tmp_path = pathlib.Path(tmp_path)
+    path_a = _fetch_remote_sample("wsi1_8k_8k_svs")
+    path_b = _fetch_remote_sample("wsi1_8k_8k_svs")
+    assert os.path.exists(path_a)
+    assert path_a == path_b
+    # Test if corrupted
+    get_wsireader(path_a)
+
+    path = _fetch_remote_sample("wsi1_8k_8k_svs", tmp_path)
+    # Assuming Path has no trailing '/'
+    assert os.path.exists(path)
+    assert str(tmp_path) in str(path)
+
+    # Test not directory path
+    test_path = pathlib.Path(f"{tmp_path}/dummy.npy")
+    np.save(test_path, np.zeros([3, 3, 3]))
+    with pytest.raises(ValueError, match=r".*tmp_path must be a directory.*"):
+        path = _fetch_remote_sample("wsi1_8k_8k_svs", test_path)
+
+    # Very tiny so temporary hook here also
     arr = stainnorm_target()
     assert isinstance(arr, np.ndarray)