From 59272441e2f2dd03fabdfd7f451988d74249b183 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:41:04 +0000 Subject: [PATCH 1/2] Go full ruff (black and bandit) and disable bandit on codeclimate Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .bandit.yml | 406 ---------------------------- .codeclimate.yml | 2 - .github/workflows/python-bandit.yml | 32 --- Makefile | 7 +- pyproject.toml | 43 ++- scripts/gen_pages.py | 2 +- tox.ini | 9 +- 7 files changed, 35 insertions(+), 466 deletions(-) delete mode 100644 .bandit.yml delete mode 100644 .github/workflows/python-bandit.yml diff --git a/.bandit.yml b/.bandit.yml deleted file mode 100644 index 8732e87..0000000 --- a/.bandit.yml +++ /dev/null @@ -1,406 +0,0 @@ - -### This config may optionally select a subset of tests to run or skip by -### filling out the 'tests' and 'skips' lists given below. If no tests are -### specified for inclusion then it is assumed all tests are desired. The skips -### set will remove specific tests from the include set. This can be controlled -### using the -t/-s CLI options. Note that the same test ID should not appear -### in both 'tests' and 'skips', this would be nonsensical and is detected by -### Bandit at runtime. - -# Available tests: -# B101 : assert_used -# B102 : exec_used -# B103 : set_bad_file_permissions -# B104 : hardcoded_bind_all_interfaces -# B105 : hardcoded_password_string -# B106 : hardcoded_password_funcarg -# B107 : hardcoded_password_default -# B108 : hardcoded_tmp_directory -# B110 : try_except_pass -# B112 : try_except_continue -# B113 : request_without_timeout -# B201 : flask_debug_true -# B202 : tarfile_unsafe_members -# B301 : pickle -# B302 : marshal -# B303 : md5 -# B304 : ciphers -# B305 : cipher_modes -# B306 : mktemp_q -# B307 : eval -# B308 : mark_safe -# B310 : urllib_urlopen -# B311 : random -# B312 : telnetlib -# B313 : xml_bad_cElementTree -# B314 : xml_bad_ElementTree -# B315 : xml_bad_expatreader -# B316 : xml_bad_expatbuilder -# B317 : xml_bad_sax -# B318 : xml_bad_minidom -# B319 : xml_bad_pulldom -# B320 : xml_bad_etree -# B321 : ftplib -# B323 : unverified_context -# B324 : hashlib_insecure_functions -# B401 : import_telnetlib -# B402 : import_ftplib -# B403 : import_pickle -# B404 : import_subprocess -# B405 : import_xml_etree -# B406 : import_xml_sax -# B407 : import_xml_expat -# B408 : import_xml_minidom -# B409 : import_xml_pulldom -# B410 : import_lxml -# B411 : import_xmlrpclib -# B412 : import_httpoxy -# B413 : import_pycrypto -# B415 : import_pyghmi -# B501 : request_with_no_cert_validation -# B502 : ssl_with_bad_version -# B503 : ssl_with_bad_defaults -# B504 : ssl_with_no_version -# B505 : weak_cryptographic_key -# B506 : yaml_load -# B507 : ssh_no_host_key_verification -# B508 : snmp_insecure_version -# B509 : snmp_weak_cryptography -# B601 : paramiko_calls -# B602 : subprocess_popen_with_shell_equals_true -# B603 : subprocess_without_shell_equals_true -# B604 : any_other_function_with_shell_equals_true -# B605 : start_process_with_a_shell -# B606 : start_process_with_no_shell -# B607 : start_process_with_partial_path -# B608 : hardcoded_sql_expressions -# B609 : linux_commands_wildcard_injection -# B610 : django_extra_used -# B611 : django_rawsql_used -# B612 : logging_config_insecure_listen -# B701 : jinja2_autoescape_false -# B702 : use_of_mako_templates -# B703 : django_mark_safe - -# (optional) list included test IDs here, eg '[B101, B406]': -tests: - -# (optional) list skipped test IDs here, eg '[B101, B406]': -skips: - -### (optional) plugin settings - some test plugins require configuration data -### that may be given here, per-plugin. All bandit test plugins have a built in -### set of sensible defaults and these will be used if no configuration is -### provided. It is not necessary to provide settings for every (or any) plugin -### if the defaults are acceptable. - -exclude_dirs: ['.tox'] - -any_other_function_with_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -assert_used: - skips: - - tests/*.py - - ./tests/*.py -hardcoded_tmp_directory: - tmp_dirs: - - /tmp - - /var/tmp - - /dev/shm -linux_commands_wildcard_injection: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -ssl_with_bad_defaults: - bad_protocol_versions: - - PROTOCOL_SSLv2 - - SSLv2_METHOD - - SSLv23_METHOD - - PROTOCOL_SSLv3 - - PROTOCOL_TLSv1 - - SSLv3_METHOD - - TLSv1_METHOD - - PROTOCOL_TLSv1_1 - - TLSv1_1_METHOD -ssl_with_bad_version: - bad_protocol_versions: - - PROTOCOL_SSLv2 - - SSLv2_METHOD - - SSLv23_METHOD - - PROTOCOL_SSLv3 - - PROTOCOL_TLSv1 - - SSLv3_METHOD - - TLSv1_METHOD - - PROTOCOL_TLSv1_1 - - TLSv1_1_METHOD -start_process_with_a_shell: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -start_process_with_no_shell: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -start_process_with_partial_path: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -subprocess_popen_with_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -subprocess_without_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -try_except_continue: - check_typed_exception: false -try_except_pass: - check_typed_exception: false -weak_cryptographic_key: - weak_key_size_dsa_high: 1024 - weak_key_size_dsa_medium: 2048 - weak_key_size_ec_high: 160 - weak_key_size_ec_medium: 224 - weak_key_size_rsa_high: 1024 - weak_key_size_rsa_medium: 2048 diff --git a/.codeclimate.yml b/.codeclimate.yml index 70bd741..a68c916 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,6 +1,4 @@ version: "2" plugins: - bandit: - enabled: true sonar-python: enabled: true diff --git a/.github/workflows/python-bandit.yml b/.github/workflows/python-bandit.yml deleted file mode 100644 index 50a6991..0000000 --- a/.github/workflows/python-bandit.yml +++ /dev/null @@ -1,32 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: Bandit checks - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - bandit: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Security check - Bandit - uses: ioggstream/bandit-report-artifacts@v1.7.4 - with: - project_path: . - config_file: .bandit.yml - - # This is optional - - name: Security check report artifacts - uses: actions/upload-artifact@v4 - with: - name: Security report - path: output/security_report.txt diff --git a/Makefile b/Makefile index be9ea44..59852d5 100644 --- a/Makefile +++ b/Makefile @@ -18,11 +18,8 @@ format: lint: tox -e lint -bandit: - tox -e bandit - format-fix: - poetry run black . + poetry run ruff format . lint-fix: poetry run ruff . --fix @@ -33,7 +30,7 @@ dev-dependencies: update-dependencies: poetry update --with dev -fix: format-fix lint-fix +fix: lint-fix format-fix check: poetry export -f requirements.txt --output /tmp/requirements.txt --with dev tox diff --git a/pyproject.toml b/pyproject.toml index 662b992..9218da8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,8 +46,6 @@ optional = true [tool.poetry.group.dev.dependencies] coverage = ">=6.5.0" -bandit = ">=1.7.6" -black = ">=22.10.0" mkdocs = ">=1.4.3" mkdocstrings = { version = ">=0.24.0", extras = ["python"] } mkdocs-gen-files = ">=0.5.0" @@ -67,15 +65,6 @@ tox = ">=4.12.1" ### Tools configuration ### ############################ -[tool.black] -target-version = ["py38", "py39", "py310", "py311", "py312"] -extend-exclude = ''' -( - /docs, - .tox -) -''' - [tool.coverage.run] branch = true source = ["bootstrap_python_package"] @@ -104,9 +93,37 @@ testpaths = [ [tool.ruff] extend-exclude = ["docs", ".tox"] +target-version = "py38" [tool.ruff.lint] -select = ["E", "F", "I"] +select = [ + "E", # pycodestyle + "W", # pycodestyle + "F", # pyflakes + "I", # isort + "N", # pep8-naming + "S", # flake8-bandit + "RUF", # ruff-specific-rules +] +# Ignoring rules problematic with formatter +# https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules +ignore = [ + "W191", + "E111", + "E114", + "E117", + "D206", + "D300", + "Q000", + "Q001", + "Q002", + "Q003", + "COM812", + "COM819", + "ISC001", + "ISC002", +] [tool.ruff.lint.per-file-ignores] -"__init__.py" = ["F401"] +"__init__.py" = ["F401"] # Ignore unused imports on init files +"tests/**/*.py" = ["S101"] # Allow assert usage on tests diff --git a/scripts/gen_pages.py b/scripts/gen_pages.py index 5559e08..e36627c 100644 --- a/scripts/gen_pages.py +++ b/scripts/gen_pages.py @@ -35,7 +35,7 @@ full_doc_path = Path(nav_pages_path, doc_path) # Handle edge cases - parts = (src_dir,) + tuple(module_path.parts) + parts = (src_dir, *tuple(module_path.parts)) if parts[-1] == "__init__": parts = parts[:-1] doc_path = doc_path.with_name("index.md") diff --git a/tox.ini b/tox.ini index 8284a8e..555774b 100644 --- a/tox.ini +++ b/tox.ini @@ -7,9 +7,8 @@ env_list = py39 py38 typing - format lint - bandit + format [testenv] ; The file /tmp/requirements.txt is created automatically if you run tox @@ -34,12 +33,8 @@ commands = [testenv:format] commands = - black --check . + ruff format --check . [testenv:lint] commands = ruff . - -[testenv:bandit] -commands = - bandit -c .bandit.yml -r . From b184b1489b176fa2f96faea0a9c0e4ca83285036 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:59:31 +0000 Subject: [PATCH 2/2] Update workflows and Makefile Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .github/workflows/python-code-style.yml | 5 +-- .github/workflows/python-lint.yml | 5 +-- .github/workflows/python-tests.yml | 3 -- .github/workflows/python-typing.yml | 30 +++++++++++++ Makefile | 58 +++++++++++++++---------- 5 files changed, 68 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/python-typing.yml diff --git a/.github/workflows/python-code-style.yml b/.github/workflows/python-code-style.yml index 536b2e5..ab3fb94 100644 --- a/.github/workflows/python-code-style.yml +++ b/.github/workflows/python-code-style.yml @@ -24,9 +24,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install poetry - poetry config virtualenvs.create false - poetry install --no-root --with dev + python -m pip install poetry tox + make poetry-export - name: Check code style with black run: | make format diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 280860d..4cc679c 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -24,8 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install poetry - poetry config virtualenvs.create false - poetry install --no-root --with dev + python -m pip install poetry tox + make poetry-export - name: Lint with ruff run: make lint diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 0aa4db3..a5a694b 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -33,6 +33,3 @@ jobs: - name: Test with pytest run: | make ci-test - - name: Check typing - run: | - make typing diff --git a/.github/workflows/python-typing.yml b/.github/workflows/python-typing.yml new file mode 100644 index 0000000..68a7cae --- /dev/null +++ b/.github/workflows/python-typing.yml @@ -0,0 +1,30 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python lint + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + typing: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry tox + make poetry-export + - name: Check typing + run: make typing diff --git a/Makefile b/Makefile index 59852d5..1473d7d 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,49 @@ -.PHONY: docs +.PHONY: dev-dependencies update-dependencies test docs fix check typing lint format ci-test ci-coverage poetry-export + +######################### +###### dev commands ##### +######################### +dev-dependencies: + poetry install --with dev --no-root + +update-dependencies: + poetry update --with dev test: poetry run pytest -n auto --cov -ci-test: - poetry run pytest +docs: + poetry run mkdocs serve -ci-coverage: - poetry run pytest --cov --cov-report lcov +fix: + poetry run ruff . --fix + poetry run ruff format . -typing: - tox -e typing +check: poetry-export + tox -format: - tox -e format +typing: poetry-export + tox -e typing -lint: +lint: poetry-export tox -e lint -format-fix: - poetry run ruff format . +format: poetry-export + tox -e format -lint-fix: - poetry run ruff . --fix -dev-dependencies: - poetry install --with dev --no-root +######################### +#### Helper commands #### +######################### +poetry-export: + poetry export -f requirements.txt --output /tmp/requirements.txt --with dev -update-dependencies: - poetry update --with dev -fix: lint-fix format-fix -check: - poetry export -f requirements.txt --output /tmp/requirements.txt --with dev - tox +######################### +###### CI commands ###### +######################### +ci-test: + poetry run pytest -docs: - poetry run mkdocs serve +ci-coverage: + poetry run pytest --cov --cov-report lcov