Merge pull request #1646 from mathbunnyru/asalikhov/update_mypy_config

Update mypy config and make it work well with pre-commit

Author: mathbunnyru

.pre-commit-config.yaml

@@ -22,7 +22,17 @@ repos:
     rev: v0.931
     hooks:
       - id: mypy
-        additional_dependencies: ["pytest", "types-requests", "types-tabulate"]
+        args: [--config, ./mypy.ini]
+        additional_dependencies:
+          ["numpy", "pytest", "requests", "types-requests", "types-tabulate"]
+        # Unfortunately, `pre-commit` only runs on changed files
+        # This doesn't work well with `mypy --follow-imports error`
+        # See: https://github.com/pre-commit/mirrors-mypy/issues/34#issuecomment-1062160321
+        #
+        # To workaround this we run `mypy` only in manual mode
+        # So it won't run as part of `git commit` command
+        # But it will still be run as part of `pre-commit` workflow and give expected results
+        stages: [manual]
 
   # Autoformat: YAML, JSON, Markdown, etc.
   - repo: https://github.com/pre-commit/mirrors-prettier

Makefile

@@ -170,7 +170,7 @@ img-rm-dang: ## remove dangling images (tagged None)
 
 
 pre-commit-all: ## run pre-commit hook on all files
-	@pre-commit run --all-files || (printf "\n\n\n" && git --no-pager diff --color=always)
+	@pre-commit run --all-files --hook-stage manual || (printf "\n\n\n" && git --no-pager diff --color=always)
 pre-commit-install: ## set up the git hook scripts
 	@pre-commit --version
 	@pre-commit install

mypy.ini

@@ -1,8 +1,25 @@
+# Mypy is an optional static type checker for Python that aims to combine
+# the benefits of dynamic (or "duck") typing and static typing.
+#
+# Documentation: http://www.mypy-lang.org
+# Project: https://github.com/python/mypy
+# Config reference: https://mypy.readthedocs.io/en/stable/config_file.html
+#
+# We use mypy as part of pre-commit checks
+
 [mypy]
 python_version = 3.9
-follow_imports = normal
-strict = False
+follow_imports = error
+strict = True
 no_incremental = True
+# This allows us to use pytest decorators, which are not typed yet
+disallow_untyped_decorators = False
+
+# These sections allow us to ignore mypy errors for packages
+# which are not (hopefully yet) statically typed
+
+[mypy-Cython.*]
+ignore_missing_imports = True
 
 [mypy-docker.*]
 ignore_missing_imports = True
@@ -22,5 +39,11 @@ ignore_missing_imports = True
 [mypy-pyspark.*]
 ignore_missing_imports = True
 
+[mypy-setuptools.*]
+ignore_missing_imports = True
+
 [mypy-tensorflow.*]
 ignore_missing_imports = True
+
+[mypy-urllib3.*]
+ignore_missing_imports = True

tests/base-notebook/test_packages.py

@@ -111,7 +111,9 @@ def r_package_predicate(package: str) -> bool:
     return not excluded_package_predicate(package) and package.startswith("r-")
 
 
-def _check_import_package(package_helper: CondaPackageHelper, command: list[str]):
+def _check_import_package(
+    package_helper: CondaPackageHelper, command: list[str]
+) -> None:
     """Generic function executing a command"""
     LOGGER.debug(f"Trying to import a package with [{command}] ...")
     exec_result = package_helper.running_container.exec_run(command)
@@ -120,12 +122,14 @@ def _check_import_package(package_helper: CondaPackageHelper, command: list[str]
     ), f"Import package failed, output: {exec_result.output}"
 
 
-def check_import_python_package(package_helper: CondaPackageHelper, package: str):
+def check_import_python_package(
+    package_helper: CondaPackageHelper, package: str
+) -> None:
     """Try to import a Python package from the command line"""
     _check_import_package(package_helper, ["python", "-c", f"import {package}"])
 
 
-def check_import_r_package(package_helper: CondaPackageHelper, package: str):
+def check_import_r_package(package_helper: CondaPackageHelper, package: str) -> None:
     """Try to import a R package from the command line"""
     _check_import_package(package_helper, ["R", "--slave", "-e", f"library({package})"])
 

tests/conftest.py

@@ -11,7 +11,7 @@
 import pytest  # type: ignore
 import requests
 
-from requests.packages.urllib3.util.retry import Retry
+from urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
 
 
@@ -23,7 +23,7 @@ def find_free_port() -> str:
     with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
         s.bind(("", 0))
         s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        return s.getsockname()[1]
+        return s.getsockname()[1]  # type: ignore
 
 
 @pytest.fixture(scope="session")

tests/images_hierarchy.py

@@ -23,7 +23,7 @@ def get_test_dirs(
     short_image_name: Optional[str],
 ) -> list[Path]:
     if short_image_name is None:
-        return []  # type: ignore
+        return []
 
     test_dirs = get_test_dirs(ALL_IMAGES[short_image_name])
     if (current_image_tests_dir := THIS_DIR / short_image_name).exists():