From 57ce839479c365d77a56c3e87e4c3ed9f36335b9 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 7 Mar 2022 08:55:53 +0300 Subject: [PATCH 1/4] Update mypy config --- mypy.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypy.ini b/mypy.ini index c9c1a225b1..8a5f7812e2 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,6 +4,9 @@ follow_imports = normal strict = False no_incremental = True +[mypy-Cython.*] +ignore_missing_imports = True + [mypy-docker.*] ignore_missing_imports = True @@ -22,5 +25,8 @@ ignore_missing_imports = True [mypy-pyspark.*] ignore_missing_imports = True +[mypy-setuptools.*] +ignore_missing_imports = True + [mypy-tensorflow.*] ignore_missing_imports = True From b487d1afbe9b5a40940a31300157d6b3211e545e Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 8 Mar 2022 23:23:13 +0300 Subject: [PATCH 2/4] Make mypy config strict and work on pre-commit --- .pre-commit-config.yaml | 5 ++++- mypy.ini | 21 +++++++++++++++++++-- tests/base-notebook/test_packages.py | 10 +++++++--- tests/conftest.py | 4 ++-- tests/images_hierarchy.py | 2 +- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e803ac91e5..0a333947da 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,10 @@ repos: rev: v0.931 hooks: - id: mypy - additional_dependencies: ["pytest", "types-requests", "types-tabulate"] + args: [--config, ./mypy.ini] + additional_dependencies: + ["numpy", "pytest", "requests", "types-requests", "types-tabulate"] + stages: [manual] # Autoformat: YAML, JSON, Markdown, etc. - repo: https://github.com/pre-commit/mirrors-prettier diff --git a/mypy.ini b/mypy.ini index 8a5f7812e2..607647ee92 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,8 +1,22 @@ +# MMypy is an optional static type checker for Python that aims to combine +# the benefits of dynamic (or "duck") typing and static typing. +# +# Documentation: http://www.mypy-lang.org +# Project: https://github.com/python/mypy +# Config reference: https://mypy.readthedocs.io/en/stable/config_file.html +# +# We use mypy as part of pre-commit checks + [mypy] python_version = 3.9 -follow_imports = normal -strict = False +follow_imports = error +strict = True no_incremental = True +# This allows us to use pytest decorators, which are not typed yet +disallow_untyped_decorators = False + +# These sections allow us to ignore mypy errors for packages +# which are not (hopefully yet) statically typed [mypy-Cython.*] ignore_missing_imports = True @@ -30,3 +44,6 @@ ignore_missing_imports = True [mypy-tensorflow.*] ignore_missing_imports = True + +[mypy-urllib3.*] +ignore_missing_imports = True diff --git a/tests/base-notebook/test_packages.py b/tests/base-notebook/test_packages.py index 7109b6ae15..1c085320e2 100644 --- a/tests/base-notebook/test_packages.py +++ b/tests/base-notebook/test_packages.py @@ -111,7 +111,9 @@ def r_package_predicate(package: str) -> bool: return not excluded_package_predicate(package) and package.startswith("r-") -def _check_import_package(package_helper: CondaPackageHelper, command: list[str]): +def _check_import_package( + package_helper: CondaPackageHelper, command: list[str] +) -> None: """Generic function executing a command""" LOGGER.debug(f"Trying to import a package with [{command}] ...") exec_result = package_helper.running_container.exec_run(command) @@ -120,12 +122,14 @@ def _check_import_package(package_helper: CondaPackageHelper, command: list[str] ), f"Import package failed, output: {exec_result.output}" -def check_import_python_package(package_helper: CondaPackageHelper, package: str): +def check_import_python_package( + package_helper: CondaPackageHelper, package: str +) -> None: """Try to import a Python package from the command line""" _check_import_package(package_helper, ["python", "-c", f"import {package}"]) -def check_import_r_package(package_helper: CondaPackageHelper, package: str): +def check_import_r_package(package_helper: CondaPackageHelper, package: str) -> None: """Try to import a R package from the command line""" _check_import_package(package_helper, ["R", "--slave", "-e", f"library({package})"]) diff --git a/tests/conftest.py b/tests/conftest.py index adbada58e0..18d7435f2f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ import pytest # type: ignore import requests -from requests.packages.urllib3.util.retry import Retry +from urllib3.util.retry import Retry from requests.adapters import HTTPAdapter @@ -23,7 +23,7 @@ def find_free_port() -> str: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(("", 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - return s.getsockname()[1] + return s.getsockname()[1] # type: ignore @pytest.fixture(scope="session") diff --git a/tests/images_hierarchy.py b/tests/images_hierarchy.py index 093883e25c..744f9c2fb9 100644 --- a/tests/images_hierarchy.py +++ b/tests/images_hierarchy.py @@ -23,7 +23,7 @@ def get_test_dirs( short_image_name: Optional[str], ) -> list[Path]: if short_image_name is None: - return [] # type: ignore + return [] test_dirs = get_test_dirs(ALL_IMAGES[short_image_name]) if (current_image_tests_dir := THIS_DIR / short_image_name).exists(): From 2067fbc1bc5c9f075900357c766320746c1f50f7 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 8 Mar 2022 23:25:57 +0300 Subject: [PATCH 3/4] Fix running hooks --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f98bf3181f..c9ebcd3bf7 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ img-rm-dang: ## remove dangling images (tagged None) pre-commit-all: ## run pre-commit hook on all files - @pre-commit run --all-files || (printf "\n\n\n" && git --no-pager diff --color=always) + @pre-commit run --all-files --hook-stage manual || (printf "\n\n\n" && git --no-pager diff --color=always) pre-commit-install: ## set up the git hook scripts @pre-commit --version @pre-commit install From d502cbe7bc5478a99a2f6bf5a765a806d0c7ad03 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 8 Mar 2022 23:30:45 +0300 Subject: [PATCH 4/4] Add docs --- .pre-commit-config.yaml | 7 +++++++ mypy.ini | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a333947da..67ccef62c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,6 +25,13 @@ repos: args: [--config, ./mypy.ini] additional_dependencies: ["numpy", "pytest", "requests", "types-requests", "types-tabulate"] + # Unfortunately, `pre-commit` only runs on changed files + # This doesn't work well with `mypy --follow-imports error` + # See: https://github.com/pre-commit/mirrors-mypy/issues/34#issuecomment-1062160321 + # + # To workaround this we run `mypy` only in manual mode + # So it won't run as part of `git commit` command + # But it will still be run as part of `pre-commit` workflow and give expected results stages: [manual] # Autoformat: YAML, JSON, Markdown, etc. diff --git a/mypy.ini b/mypy.ini index 607647ee92..38c7c7fe29 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,6 @@ -# MMypy is an optional static type checker for Python that aims to combine +# Mypy is an optional static type checker for Python that aims to combine # the benefits of dynamic (or "duck") typing and static typing. -# +# # Documentation: http://www.mypy-lang.org # Project: https://github.com/python/mypy # Config reference: https://mypy.readthedocs.io/en/stable/config_file.html