diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 00000000..7c510094 --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,3 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ diff --git a/.gitattributes b/.gitattributes index af7fe598..d1575bd8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ papermill/_version.py export-subst +.git_archival.txt export-subst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c59ba97..ae841666 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,13 +29,13 @@ jobs: set -xe python -VV python -m site - python -m pip install --upgrade pip setuptools wheel coverage[toml] virtualenv tox tox-gh-actions + python -m pip install --upgrade pip wheel coverage[toml] virtualenv tox tox-gh-actions - name: "Run tox targets for ${{ matrix.python-version }}" run: python -m tox - name: "Generate coverage XML" - if: "contains(env.USING_COVERAGE, matrix.python-version)" + if: contains(env.USING_COVERAGE, matrix.python-version) run: python -m coverage xml - name: "Upload coverage to Codecov" @@ -46,7 +46,7 @@ jobs: strategy: fail-fast: false matrix: - toxenv: ["manifest", "docs", "binder"] + toxenv: ["docs", "binder"] env: TOXENV: ${{ matrix.toxenv }} steps: diff --git a/.gitignore b/.gitignore index a580829b..30942df9 100644 --- a/.gitignore +++ b/.gitignore @@ -128,3 +128,6 @@ pip-wheel-metadata # lint .ruff_cache/ local.ipynb + +# hatch build +papermill/version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0bfe0a4..1d73a259 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,14 +26,14 @@ repos: - id: codespell additional_dependencies: [tomli] - - repo: https://github.com/psf/black + - repo: https://github.com/psf/black-pre-commit-mirror rev: 25.11.0 hooks: - id: black name: Black code args: [--skip-string-normalization] - - repo: https://github.com/executablebooks/mdformat + - repo: https://github.com/hukkin/mdformat rev: 1.0.0 hooks: - id: mdformat diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d32a2282..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,33 +0,0 @@ -recursive-include papermill *.py -recursive-include papermill *.ipynb -recursive-include papermill *.json -recursive-include papermill *.yaml -recursive-include papermill *.yml -recursive-include papermill *.keep -recursive-include papermill *.txt - -include requirements.txt -recursive-include requirements *.txt -include README.md -include LICENSE -include MANIFEST.in -include *.md -include *.toml - -# Documentation -prune docs - -exclude .pre-commit-config.yaml -# exclude sample notebooks for binder -prune binder -# Scripts -graft scripts -# Test env -prune .tox -# Exclude notebooks checkpoints generated by testing -recursive-exclude papermill/.ipynb_checkpoints *.ipynb -recursive-exclude papermill/tests/notebooks/.ipynb_checkpoints *.ipynb - -# Build files -exclude .github -exclude .readthedocs.yaml diff --git a/papermill/version.py b/papermill/version.py deleted file mode 100644 index e7426207..00000000 --- a/papermill/version.py +++ /dev/null @@ -1 +0,0 @@ -version = '2.6.0' diff --git a/pyproject.toml b/pyproject.toml index 57cb957c..c9f56813 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,28 @@ [build-system] -requires = ["setuptools>=61.0", "wheel"] -build-backend = "setuptools.build_meta" +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" [project] name = "papermill" dynamic = ["version"] description = "Parameterize and run Jupyter and nteract Notebooks" readme = "README.md" -license = {text = "BSD"} +license = {text = "BSD-3-Clause"} +requires-python = ">=3.10" authors = [ - {name = "nteract contributors", email = "nteract@googlegroups.com"} + { name = "nteract contributors", email = "nteract@googlegroups.com" }, +] +keywords = [ + "jupyter", + "mapreduce", + "notebook", + "nteract", + "pipeline", ] -keywords = ["jupyter", "mapreduce", "nteract", "pipeline", "notebook"] classifiers = [ "Intended Audience :: Developers", - "Intended Audience :: System Administrators", "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", + "Intended Audience :: System Administrators", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", @@ -24,142 +30,141 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] -requires-python = ">=3.10" dependencies = [ + "aiohttp >=3.9.0; python_version == '3.12'", "click", + "entrypoints", + "nbclient >= 0.2.0", + "nbformat >= 5.2.0", "pyyaml", - "nbformat>=5.2.0", - "nbclient>=0.2.0", - "tqdm>=4.32.2", "requests", - "entrypoints", - "tenacity>=5.0.2", - "aiohttp>=3.9.0; python_version=='3.12'", + "tenacity >= 5.0.2", + "tqdm >= 4.32.2", ] -[project.urls] -Documentation = "https://papermill.readthedocs.io" -Funding = "https://nteract.io" -Source = "https://github.com/nteract/papermill/" -Tracker = "https://github.com/nteract/papermill/issues" - -[project.scripts] -papermill = "papermill.__main__:papermill" - [project.optional-dependencies] -s3 = ["boto3"] +all = [ + "papermill[azure,binder,gcs,github,hdfs,s3]" +] azure = [ - "azure-datalake-store>=0.0.30,<1.0.0a0", - "azure-storage-blob>=12.1.0", - "requests>=2.21.0", + "azure-datalake-store >= 0.0.30,<1.0.0a0", # newer versions require auth changes see https://github.com/nteract/papermill/issues/825 "azure-identity>=1.3.1", + "azure-storage-blob >= 12.1.0", + "requests >= 2.21.0", ] -gcs = ["gcsfs>=0.2.0"] -hdfs = ["pyarrow>=2.0"] -github = ["PyGithub>=1.55"] -black = ["black>=19.3b0"] -all = [ - "boto3", - "azure-datalake-store>=0.0.30,<1.0.0a0", - "azure-storage-blob>=12.1.0", - "requests>=2.21.0", - "azure-identity>=1.3.1", - "gcsfs>=0.2.0", - "pyarrow>=2.0", - "PyGithub>=1.55", - "black>=19.3b0", +binder = [ + "nteract-scrapbook", + "pandas", + "matplotlib", + "ipywidgets", + "ipykernel", ] dev = [ + "attrs>=17.4.0", + "azure-datalake-store >= 0.0.30,<1.0.0a0", + "azure-identity>=1.3.1", + "azure-storage-blob >= 12.1.0", + "black >= 19.3b0", "boto3", "botocore", + "check-manifest", "codecov", "coverage", + "gcsfs>=0.2.0", "google_compute_engine", + "hatch", "ipython>=5.0,<9.0.0", "ipywidgets", + "moto >= 5.0.0,<5.1.0", "notebook", - "moto>=5.0.0,<5.1.0", - "pytest>=4.1", + "pip>=18.1", + "pre-commit", + "pyarrow >= 2.0", + "PyGithub >= 1.55", "pytest-cov>=2.6.1", - "pytest-mock>=1.10", "pytest-env>=0.6.2", - "requests>=2.21.0", - "check-manifest", - "attrs>=17.4.0", - "pre-commit", - "tox", - "bumpversion", + "pytest-mock>=1.10", + "pytest>=4.1", "recommonmark", - "pip>=18.1", - "wheel>=0.31.0", - "setuptools>=38.6.0", + "requests >= 2.21.0", + "tox", "twine>=1.11.0", - "azure-datalake-store>=0.0.30,<1.0.0a0", - "azure-storage-blob>=12.1.0", - "azure-identity>=1.3.1", + "wheel>=0.31.0", +] +docs = [ + "furo>=2023.9.10", + "moto>=4.2.8", + "myst-parser>=2.0.0", + "sphinx-copybutton>=0.5.2", + "Sphinx>=7.2.6", +] +gcs = [ "gcsfs>=0.2.0", - "pyarrow>=2.0", - "PyGithub>=1.55", - "black>=19.3b0", +] +github = [ + "PyGithub >= 1.55", +] +hdfs = [ + "pyarrow >= 2.0", +] +s3 = [ + "boto3", ] test = [ + "attrs>=17.4.0", + "azure-datalake-store >= 0.0.30,<1.0.0a0", + "azure-identity>=1.3.1", + "azure-storage-blob >= 12.1.0", + "black >= 19.3b0", "boto3", "botocore", + "check-manifest", "codecov", "coverage", + "gcsfs>=0.2.0", "google_compute_engine", "ipython>=5.0,<9.0.0", "ipywidgets", + "moto >= 5.0.0,<5.1.0", "notebook", - "moto>=5.0.0,<5.1.0", - "pytest>=4.1", + "pip>=18.1", + "pre-commit", + "pyarrow >= 2.0", + "PyGithub >= 1.55", "pytest-cov>=2.6.1", - "pytest-mock>=1.10", "pytest-env>=0.6.2", - "requests>=2.21.0", - "check-manifest", - "attrs>=17.4.0", - "pre-commit", - "tox", - "bumpversion", + "pytest-mock>=1.10", + "pytest>=4.1", "recommonmark", - "pip>=18.1", - "wheel>=0.31.0", - "setuptools>=38.6.0", + "requests >= 2.21.0", + "tox", "twine>=1.11.0", - "azure-datalake-store>=0.0.30,<1.0.0a0", - "azure-storage-blob>=12.1.0", - "azure-identity>=1.3.1", - "gcsfs>=0.2.0", - "pyarrow>=2.0", - "PyGithub>=1.55", - "black>=19.3b0", -] -docs = [ - "boto3", - "azure-datalake-store>=0.0.30,<1.0.0a0", - "azure-storage-blob>=12.1.0", - "requests>=2.21.0", - "azure-identity>=1.3.1", - "gcsfs>=0.2.0", - "pyarrow>=2.0", - "PyGithub>=1.55", - "black>=19.3b0", - "Sphinx>=7.2.6", - "furo>=2023.9.10", - "myst-parser>=2.0.0", - "moto>=4.2.8", - "sphinx-copybutton>=0.5.2", + "wheel>=0.31.0", ] -[tool.setuptools] -packages = ["papermill"] -include-package-data = true +[project.scripts] +papermill = "papermill.__main__:papermill" + +[project.urls] +Documentation = "https://papermill.readthedocs.io" +Funding = "https://nteract.io" +Homepage = "https://github.com/nteract/papermill" +Source = "https://github.com/nteract/papermill/" +Tracker = "https://github.com/nteract/papermill/issues" + +[tool.hatch] +version.source = "vcs" +build.hooks.vcs.version-file = "papermill/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/papermill", +] +exclude = [ + "binder/*", +] -[tool.setuptools.dynamic] -version = {attr = "papermill.version.version"} -# Example configuration for Black. [tool.black] line-length = 120 target-version = ['py310'] @@ -192,8 +197,22 @@ quiet-level = 3 ignore-words-list = "dne, compiletime" +[tool.pytest.ini_options] +minversion = "9" +testpaths = ["papermill/tests"] +log_level = "INFO" +strict_xfail = true +addopts = ["-ra", "--strict-config", "--strict-markers"] +env = [ + "AWS_SECRET_ACCESS_KEY=foobar_secret", + "AWS_ACCESS_KEY_ID=foobar_key" +] +filterwarnings = [ + "error", # turn all warnings into errors +] + + [tool.ruff] -target-version = "py310" line-length = 120 # Enable Pyflakes `E` and `F` codes by default. lint.select = [ @@ -218,29 +237,11 @@ format.quote-style = "preserve" # Use Google-style docstrings. convention = "google" -[tool.pytest.ini_options] -env = [ - "AWS_SECRET_ACCESS_KEY=foobar_secret", - "AWS_ACCESS_KEY_ID=foobar_key", -] -filterwarnings = [ - "ignore:.*imp module is deprecated.*:DeprecationWarning", -] - -[tool.bumpversion] -current_version = "2.6.0" -commit = true -tag = true -tag_name = "{new_version}" - -[[tool.bumpversion.files]] -filename = "papermill/version.py" - [tool.tox] legacy_tox_ini = """ [tox] skipsdist = true -envlist = py{310,311,312,313}, dist, manifest, docs, binder +envlist = py{310,311,312,313}, dist, docs, binder [gh-actions] python = @@ -249,19 +250,11 @@ python = 3.12: py312 3.13: py313, docs, dist -# Manifest -[testenv:manifest] -skip_install = true -deps = check-manifest -commands = check-manifest -ignore = - .readthedocs.yaml - # Docs [testenv:docs] description = invoke sphinx-build to build the HTML docs deps = - .[docs] + .[docs,azure] extras = docs commands = sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W -bhtml {posargs} @@ -270,9 +263,11 @@ commands = # Binder [testenv:binder] description = ensure /binder/*ipynb are runnable -deps = - -r binder/requirements.txt -commands = python -c "import glob; import papermill as pm; [pm.execute_notebook(input, '{toxworkdir}/out.ipynb', parameters=\\{'binder_dir':'binder'\\}) for input in glob.glob('binder/**/*.ipynb')]" +extras = binder +changedir = binder +commands = + python -m ipykernel install --user --name=python3 + python -c "import papermill as pm; pm.execute_notebook('cli-simple/simple_input.ipynb', '{toxworkdir}/out.ipynb', parameters=\\{'msg': 'test'\\})" # Distro [testenv:dist] @@ -295,7 +290,6 @@ basepython = py311: python3.11 py312: python3.12 py313: python3.13 - manifest: python3.13 dist: python3.13 docs: python3.13 binder: python3.13 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6f8ebb8a..00000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -click -pyyaml -nbformat >= 5.2.0 -nbclient >= 0.2.0 -tqdm >= 4.32.2 -requests -entrypoints -tenacity >= 5.0.2 -aiohttp >=3.9.0; python_version=="3.12" diff --git a/requirements/azure.txt b/requirements/azure.txt deleted file mode 100644 index 59ef020e..00000000 --- a/requirements/azure.txt +++ /dev/null @@ -1,6 +0,0 @@ -# newer versions require changes to authentication -# see https://github.com/nteract/papermill/issues/825 -azure-datalake-store >= 0.0.30,<1.0.0a0 -azure-storage-blob >= 12.1.0 -requests >= 2.21.0 -azure-identity>=1.3.1 diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index d3c7dc51..00000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,25 +0,0 @@ -boto3 -botocore -codecov -coverage -google_compute_engine # Need this because boto has issues with dynamic package loading during tests if other google components are there -# newer versions of ipython have different imports and require refactoring -ipython>=5.0,<9.0.0 -ipywidgets -notebook -moto >= 5.0.0,<5.1.0 -pytest>=4.1 -pytest-cov>=2.6.1 -pytest-mock>=1.10 -pytest-env>=0.6.2 -requests >= 2.21.0 -check-manifest -attrs>=17.4.0 -pre-commit -tox -bumpversion -recommonmark -pip>=18.1 -wheel>=0.31.0 -setuptools>=38.6.0 -twine>=1.11.0 diff --git a/requirements/docs.txt b/requirements/docs.txt deleted file mode 100644 index c4fa860e..00000000 --- a/requirements/docs.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Pin packages for RTD builds -# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html#pinning-dependencies -Sphinx>=7.2.6 -furo>=2023.9.10 -myst-parser>=2.0.0 -moto>=4.2.8 -sphinx-copybutton>=0.5.2 diff --git a/requirements/gcs.txt b/requirements/gcs.txt deleted file mode 100644 index 8db67662..00000000 --- a/requirements/gcs.txt +++ /dev/null @@ -1 +0,0 @@ -gcsfs>=0.2.0 diff --git a/requirements/github.txt b/requirements/github.txt deleted file mode 100644 index 75bfade9..00000000 --- a/requirements/github.txt +++ /dev/null @@ -1 +0,0 @@ -PyGithub >= 1.55 diff --git a/requirements/hdfs.txt b/requirements/hdfs.txt deleted file mode 100644 index fa262689..00000000 --- a/requirements/hdfs.txt +++ /dev/null @@ -1 +0,0 @@ -pyarrow >= 2.0 diff --git a/requirements/s3.txt b/requirements/s3.txt deleted file mode 100644 index 30ddf823..00000000 --- a/requirements/s3.txt +++ /dev/null @@ -1 +0,0 @@ -boto3