diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3477a60d..f54f3074 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,14 +24,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.10', '3.11'] steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install nibabies diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 8cd82acf..20b600fe 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -20,14 +20,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10'] + python-version: ['3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 # Only fetch the latest commit - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/nibabies/_warnings.py b/nibabies/_warnings.py index 3781d671..88bdad5d 100644 --- a/nibabies/_warnings.py +++ b/nibabies/_warnings.py @@ -1,4 +1,5 @@ """Manipulate Python warnings.""" + import logging import warnings diff --git a/nibabies/cli/run.py b/nibabies/cli/run.py index 3d52f6d8..6145d516 100644 --- a/nibabies/cli/run.py +++ b/nibabies/cli/run.py @@ -130,8 +130,6 @@ def main(): _copy_any(dseg_tsv, str(config.execution.nibabies_dir / "desc-aparcaseg_dseg.tsv")) # errno = 0 finally: - from pkg_resources import resource_filename as pkgrf - from ..reports.core import generate_reports # Generate reports phase diff --git a/nibabies/cli/workflow.py b/nibabies/cli/workflow.py index 7f02233f..d8bb2e77 100644 --- a/nibabies/cli/workflow.py +++ b/nibabies/cli/workflow.py @@ -44,8 +44,6 @@ def build_workflow(config_file): # Called with reports only if config.execution.reports_only: - from pkg_resources import resource_filename as pkgrf - build_logger.log( 25, "Running --reports-only on participants %s", @@ -132,14 +130,16 @@ def build_boilerplate(workflow): from shutil import copyfile from subprocess import CalledProcessError, TimeoutExpired, check_call - from pkg_resources import resource_filename as pkgrf + from nibabies.data import load as load_data + + bib = load_data("boilerplate.bib") # Generate HTML file resolving citations cmd = [ "pandoc", "-s", "--bibliography", - pkgrf("nibabies", "data/boilerplate.bib"), + bib, "--citeproc", "--metadata", 'pagetitle="nibabies citation boilerplate"', @@ -159,7 +159,7 @@ def build_boilerplate(workflow): "pandoc", "-s", "--bibliography", - pkgrf("nibabies", "data/boilerplate.bib"), + bib, "--natbib", str(citation_files["md"]), "-o", @@ -171,4 +171,4 @@ def build_boilerplate(workflow): except (FileNotFoundError, CalledProcessError, TimeoutExpired): config.loggers.cli.warning("Could not generate CITATION.tex file:\n%s", " ".join(cmd)) else: - copyfile(pkgrf("nibabies", "data/boilerplate.bib"), citation_files["bib"]) + copyfile(bib, citation_files["bib"]) diff --git a/nibabies/conftest.py b/nibabies/conftest.py index e9d8cf43..8070b17c 100644 --- a/nibabies/conftest.py +++ b/nibabies/conftest.py @@ -1,9 +1,11 @@ """py.test configuration""" + from pathlib import Path from tempfile import TemporaryDirectory import pytest -from pkg_resources import resource_filename + +from nibabies.data import load as load_data FILES = ( "functional.nii", @@ -33,5 +35,5 @@ def data_dir(): @pytest.fixture(autouse=True) def set_namespace(doctest_namespace, data_dir): doctest_namespace["data_dir"] = data_dir - doctest_namespace["test_data"] = Path(resource_filename("nibabies", "tests/data")) + doctest_namespace["test_data"] = load_data.cached('../tests/data') doctest_namespace["Path"] = Path diff --git a/nibabies/data/__init__.py b/nibabies/data/__init__.py index 6578c45d..557c96e2 100644 --- a/nibabies/data/__init__.py +++ b/nibabies/data/__init__.py @@ -1,6 +1,25 @@ +"""Data file retrieval + +.. autofunction:: load + +.. automethod:: load.readable + +.. automethod:: load.as_path + +.. automethod:: load.cached + +.. autoclass:: Loader +""" + +from __future__ import annotations + import atexit -from contextlib import ExitStack +import os +from contextlib import AbstractContextManager, ExitStack +from functools import cached_property from pathlib import Path +from types import ModuleType +from typing import Union try: from functools import cache @@ -10,16 +29,155 @@ try: # Prefer backport to leave consistency to dependency spec from importlib_resources import as_file, files except ImportError: - from importlib.resources import as_file, files + from importlib.resources import as_file, files # type: ignore + +try: # Prefer stdlib so Sphinx can link to authoritative documentation + from importlib.resources.abc import Traversable +except ImportError: + from importlib_resources.abc import Traversable + +__all__ = ["load"] + + +class Loader: + """A loader for package files relative to a module + + This class wraps :mod:`importlib.resources` to provide a getter + function with an interpreter-lifetime scope. For typical packages + it simply passes through filesystem paths as :class:`~pathlib.Path` + objects. For zipped distributions, it will unpack the files into + a temporary directory that is cleaned up on interpreter exit. + + This loader accepts a fully-qualified module name or a module + object. + + Expected usage:: + + '''Data package + + .. autofunction:: load_data + + .. automethod:: load_data.readable + + .. automethod:: load_data.as_path + + .. automethod:: load_data.cached + ''' + + from nibabies.data import Loader + + load_data = Loader(__package__) + + :class:`~Loader` objects implement the :func:`callable` interface + and generate a docstring, and are intended to be treated and documented + as functions. + + For greater flexibility and improved readability over the ``importlib.resources`` + interface, explicit methods are provided to access resources. + + +---------------+----------------+------------------+ + | On-filesystem | Lifetime | Method | + +---------------+----------------+------------------+ + | `True` | Interpreter | :meth:`cached` | + +---------------+----------------+------------------+ + | `True` | `with` context | :meth:`as_path` | + +---------------+----------------+------------------+ + | `False` | n/a | :meth:`readable` | + +---------------+----------------+------------------+ + + It is also possible to use ``Loader`` directly:: + + from nibabies.data import Loader + + Loader(other_package).readable('data/resource.ext').read_text() + + with Loader(other_package).as_path('data') as pkgdata: + # Call function that requires full Path implementation + func(pkgdata) + + # contrast to + + from importlib_resources import files, as_file + + files(other_package).joinpath('data/resource.ext').read_text() + + with as_file(files(other_package) / 'data') as pkgdata: + func(pkgdata) + + .. automethod:: readable + + .. automethod:: as_path + + .. automethod:: cached + """ + + def __init__(self, anchor: Union[str, ModuleType]): + self._anchor = anchor + self.files = files(anchor) + self.exit_stack = ExitStack() + atexit.register(self.exit_stack.close) + # Allow class to have a different docstring from instances + self.__doc__ = self._doc + + @cached_property + def _doc(self): + """Construct docstring for instances + + Lists the public top-level paths inside the location, where + non-public means has a `.` or `_` prefix or is a 'tests' + directory. + """ + top_level = sorted( + os.path.relpath(p, self.files) + "/"[: p.is_dir()] + for p in self.files.iterdir() + if p.name[0] not in (".", "_") and p.name != "tests" + ) + doclines = [ + f"Load package files relative to ``{self._anchor}``.", + "", + "This package contains the following (top-level) files/directories:", + "", + *(f"* ``{path}``" for path in top_level), + ] + + return "\n".join(doclines) + + def readable(self, *segments) -> Traversable: + """Provide read access to a resource through a Path-like interface. + + This file may or may not exist on the filesystem, and may be + efficiently used for read operations, including directory traversal. + + This result is not cached or copied to the filesystem in cases where + that would be necessary. + """ + return self.files.joinpath(*segments) + + def as_path(self, *segments) -> AbstractContextManager[Path]: + """Ensure data is available as a :class:`~pathlib.Path`. + + This method generates a context manager that yields a Path when + entered. + + This result is not cached, and any temporary files that are created + are deleted when the context is exited. + """ + return as_file(self.files.joinpath(*segments)) + + @cache + def cached(self, *segments) -> Path: + """Ensure data is available as a :class:`~pathlib.Path`. -__all__ = ["load_resource"] + Any temporary files that are created remain available throughout + the duration of the program, and are deleted when Python exits. -exit_stack = ExitStack() -atexit.register(exit_stack.close) + Results are cached so that multiple calls do not unpack the same + data multiple times, but the cache is sensitive to the specific + argument(s) passed. + """ + return self.exit_stack.enter_context(as_file(self.files.joinpath(*segments))) -path = files(__package__) + __call__ = cached -@cache -def load_resource(fname: str) -> Path: - return exit_stack.enter_context(as_file(path.joinpath(fname))) +load = Loader(__package__) diff --git a/nibabies/interfaces/confounds.py b/nibabies/interfaces/confounds.py index aa05a894..3a31a466 100644 --- a/nibabies/interfaces/confounds.py +++ b/nibabies/interfaces/confounds.py @@ -114,6 +114,7 @@ class GatherConfounds(SimpleInterface): >>> tmpdir.cleanup() """ + input_spec = GatherConfoundsInputSpec output_spec = GatherConfoundsOutputSpec diff --git a/nibabies/interfaces/maths.py b/nibabies/interfaces/maths.py index e7789eee..bf604f62 100644 --- a/nibabies/interfaces/maths.py +++ b/nibabies/interfaces/maths.py @@ -1,4 +1,5 @@ """A module for interfaces """ + import os import numpy as np diff --git a/nibabies/reports/core.py b/nibabies/reports/core.py index bb175716..58f5661a 100644 --- a/nibabies/reports/core.py +++ b/nibabies/reports/core.py @@ -2,7 +2,7 @@ from nireports.assembler.report import Report -from nibabies.data import load_resource +from nibabies.data import load as load_data def run_reports( @@ -22,7 +22,7 @@ def run_reports( run_uuid, subject=subject, session=session, - bootstrap_file=load_resource('reports-spec.yml'), + bootstrap_file=load_data.readable('reports-spec.yml'), reportlets_dir=reportlets_dir, ).generate_report() diff --git a/nibabies/tests/test_config.py b/nibabies/tests/test_config.py index 2b985521..f6198dd4 100644 --- a/nibabies/tests/test_config.py +++ b/nibabies/tests/test_config.py @@ -27,10 +27,10 @@ import pytest from niworkflows.utils.spaces import format_reference -from pkg_resources import resource_filename as pkgrf from toml import loads -from .. import config +from nibabies import config +from nibabies.data import load as load_data def _reset_config(): @@ -58,8 +58,7 @@ def test_reset_config(): def test_config_spaces(): """Check that all necessary spaces are recorded in the config.""" - filename = Path(pkgrf('nibabies', 'data/tests/config.toml')) - settings = loads(filename.read_text()) + settings = loads(load_data.readable('tests/config.toml').read_text()) for sectionname, configs in settings.items(): if sectionname != 'environment': section = getattr(config, sectionname) diff --git a/nibabies/utils/misc.py b/nibabies/utils/misc.py index c65a788d..98263225 100644 --- a/nibabies/utils/misc.py +++ b/nibabies/utils/misc.py @@ -8,7 +8,7 @@ from typing import Union from nibabies import __version__ -from nibabies.data import load_resource +from nibabies.data import load as load_data def fix_multi_source_name(in_files): @@ -144,7 +144,9 @@ def save_fsLR_mcribs(mcribs_dir: str | Path) -> None: template_dir = Path(mcribs_dir) / 'templates_fsLR' template_dir.mkdir(exist_ok=True) - for src in load_resource('atlases').glob('*sphere.surf.gii'): + atlases = load_data.cached('atlases') + + for src in atlases.glob('*sphere.surf.gii'): if not (dst := (template_dir / src.name)).exists(): try: shutil.copyfile(src, dst) diff --git a/nibabies/workflows/anatomical/base.py b/nibabies/workflows/anatomical/base.py index 3a7c7545..60dfc00d 100644 --- a/nibabies/workflows/anatomical/base.py +++ b/nibabies/workflows/anatomical/base.py @@ -1,4 +1,5 @@ """Base anatomical preprocessing.""" + from __future__ import annotations import typing as ty diff --git a/nibabies/workflows/anatomical/brain_extraction.py b/nibabies/workflows/anatomical/brain_extraction.py index aa3d101d..d8cb8602 100644 --- a/nibabies/workflows/anatomical/brain_extraction.py +++ b/nibabies/workflows/anatomical/brain_extraction.py @@ -3,7 +3,6 @@ """Baby brain extraction from T2w images.""" from nipype.interfaces import utility as niu from nipype.pipeline import engine as pe -from pkg_resources import resource_filename as pkgr_fn def init_infant_brain_extraction_wf( @@ -79,8 +78,6 @@ def init_infant_brain_extraction_wf( from nipype.interfaces.ants import ImageMath, N4BiasFieldCorrection from niworkflows.interfaces.fixes import FixHeaderApplyTransforms as ApplyTransforms from niworkflows.interfaces.fixes import FixHeaderRegistration as Registration - - # niworkflows from niworkflows.interfaces.nibabel import ( ApplyMask, Binarize, @@ -88,7 +85,8 @@ def init_infant_brain_extraction_wf( IntensityClip, ) - from ...utils.misc import cohort_by_months + from nibabies.data import load as load_data + from nibabies.utils.misc import cohort_by_months # handle template specifics template_specs = template_specs or {} @@ -138,9 +136,7 @@ def init_infant_brain_extraction_wf( # Set up initial spatial normalization ants_params = "testing" if sloppy else "precise" norm = pe.Node( - Registration( - from_file=pkgr_fn("nibabies.data", f"antsBrainExtraction_{ants_params}.json") - ), + Registration(from_file=load_data(f"antsBrainExtraction_{ants_params}.json")), name="norm", n_procs=omp_nthreads, mem_gb=mem_gb, diff --git a/nibabies/workflows/anatomical/resampling.py b/nibabies/workflows/anatomical/resampling.py index 0c350660..c687a050 100644 --- a/nibabies/workflows/anatomical/resampling.py +++ b/nibabies/workflows/anatomical/resampling.py @@ -8,7 +8,7 @@ from smriprep.workflows.surfaces import _collate, init_morph_grayords_wf from nibabies.config import DEFAULT_MEMORY_MIN_GB -from nibabies.data import load_resource +from nibabies.data import load as load_data from nibabies.interfaces.utils import CiftiSelect @@ -59,7 +59,7 @@ def init_anat_fsLR_resampling_wf( select_surfaces = pe.Node(CiftiSelect(), name='select_surfaces') if mcribs: - atlases = load_resource('atlases') + atlases = load_data.cached('atlases') # use dHCP 32k fsLR instead select_surfaces.inputs.template_spheres = [ str(atlases / 'tpl-dHCP_space-fsLR_hemi-L_den-32k_desc-week42_sphere.surf.gii'), @@ -240,7 +240,7 @@ def init_mcribs_morph_grayords_wf( ], ) - atlases = load_resource('atlases') + atlases = load_data.cached('atlases') resample.inputs.new_sphere = [ # 32k str(atlases / 'tpl-dHCP_space-fsLR_hemi-L_den-32k_desc-week42_sphere.surf.gii'), str(atlases / 'tpl-dHCP_space-fsLR_hemi-R_den-32k_desc-week42_sphere.surf.gii'), diff --git a/nibabies/workflows/anatomical/segmentation.py b/nibabies/workflows/anatomical/segmentation.py index ce903749..b7607ff0 100644 --- a/nibabies/workflows/anatomical/segmentation.py +++ b/nibabies/workflows/anatomical/segmentation.py @@ -4,10 +4,9 @@ from nipype.interfaces import utility as niu from nipype.interfaces.ants.segmentation import JointFusion from nipype.pipeline import engine as pe +from niworkflows.data import load as load_nwf from niworkflows.interfaces.fixes import FixHeaderApplyTransforms as ApplyTransforms from niworkflows.interfaces.fixes import FixHeaderRegistration as Registration -from niworkflows.interfaces.nibabel import MapLabels -from pkg_resources import resource_filename as pkgr_fn from smriprep.utils.misc import apply_lut as _apply_bids_lut from smriprep.workflows.anatomical import ( _aseg_to_three, @@ -15,7 +14,7 @@ _split_segments, ) -from ...config import DEFAULT_MEMORY_MIN_GB +from nibabies.config import DEFAULT_MEMORY_MIN_GB def init_anat_segmentations_wf( @@ -104,9 +103,7 @@ def init_anat_segmentations_wf( ants_params = "testing" if sloppy else "precise" # Register to each subject space norm = pe.MapNode( - Registration( - from_file=pkgr_fn("niworkflows.data", f"antsBrainExtraction_{ants_params}.json") - ), + Registration(from_file=load_nwf(f"antsBrainExtraction_{ants_params}.json")), name="norm", iterfield=["moving_image"], n_procs=omp_nthreads, diff --git a/nibabies/workflows/anatomical/surfaces.py b/nibabies/workflows/anatomical/surfaces.py index e3e6bd66..fff2d490 100644 --- a/nibabies/workflows/anatomical/surfaces.py +++ b/nibabies/workflows/anatomical/surfaces.py @@ -1,4 +1,5 @@ """Anatomical surface projections""" + from typing import Optional from nipype.interfaces import freesurfer as fs @@ -11,8 +12,8 @@ from niworkflows.utils.connections import pop_file from smriprep.workflows.surfaces import init_gifti_surface_wf -from ...config import DEFAULT_MEMORY_MIN_GB -from ...data import load_resource +from nibabies.config import DEFAULT_MEMORY_MIN_GB +from nibabies.data import load as load_data SURFACE_INPUTS = [ "subjects_dir", @@ -238,7 +239,7 @@ def init_mcribs_sphere_reg_wf(*, name="mcribs_sphere_reg_wf"): fix_meta = pe.MapNode(FixGiftiMetadata(), iterfield="in_file", name="fix_meta") # load template files - atlases = load_resource('atlases') + atlases = load_data.cached('atlases') # SurfaceSphereProjectUnProject # project to 41k dHCP atlas sphere diff --git a/nibabies/workflows/anatomical/template.py b/nibabies/workflows/anatomical/template.py index e74cbe5d..a2125385 100644 --- a/nibabies/workflows/anatomical/template.py +++ b/nibabies/workflows/anatomical/template.py @@ -1,4 +1,5 @@ """Prepare anatomical images for processing.""" + from __future__ import annotations from nipype.interfaces import utility as niu diff --git a/nibabies/workflows/bold/alignment.py b/nibabies/workflows/bold/alignment.py index 6925d410..b43a2825 100644 --- a/nibabies/workflows/bold/alignment.py +++ b/nibabies/workflows/bold/alignment.py @@ -6,9 +6,9 @@ from nipype.interfaces import utility as niu from nipype.pipeline import engine as pe from niworkflows.engine.workflows import LiterateWorkflow as Workflow -from pkg_resources import resource_filename -from ...interfaces.workbench import VolumeLabelImport +from nibabies.data import load as load_data +from nibabies.interfaces.workbench import VolumeLabelImport def init_subcortical_rois_wf(*, name="subcortical_rois_wf"): @@ -70,17 +70,15 @@ def init_subcortical_rois_wf(*, name="subcortical_rois_wf"): # ) map_labels = pe.Node( - MapLabels( - mappings_file=resource_filename("nibabies", "data/FreeSurferLabelRemappings.json") - ), + MapLabels(mappings_file=load_data("FreeSurferLabelRemappings.json")), name='map_labels', ) - subcortical_labels = resource_filename( - "nibabies", "data/FreeSurferSubcorticalLabelTableLut.txt" - ) refine_bold_rois = pe.Node( - VolumeLabelImport(label_list_file=subcortical_labels, discard_others=True), + VolumeLabelImport( + label_list_file=load_data("FreeSurferSubcorticalLabelTableLut.txt"), + discard_others=True, + ), name="refine_bold_rois", ) diff --git a/nibabies/workflows/bold/registration.py b/nibabies/workflows/bold/registration.py index f527396f..8e0b6702 100644 --- a/nibabies/workflows/bold/registration.py +++ b/nibabies/workflows/bold/registration.py @@ -13,12 +13,12 @@ import logging import os -import pkg_resources as pkgr from nipype.interfaces import c3, fsl from nipype.interfaces import utility as niu from nipype.pipeline import engine as pe -from ...config import DEFAULT_MEMORY_MIN_GB +from nibabies.config import DEFAULT_MEMORY_MIN_GB +from nibabies.data import load as load_data LOGGER = logging.getLogger("nipype.workflow") @@ -476,9 +476,11 @@ def init_bbreg_wf(use_bbr, bold2t1w_dof, bold2t1w_init, omp_nthreads, name="bbre Co-registration was configured with {dof} degrees of freedom{reason}. """.format( dof={6: "six", 9: "nine", 12: "twelve"}[bold2t1w_dof], - reason="" - if bold2t1w_dof == 6 - else "to account for distortions remaining in the BOLD reference", + reason=( + "" + if bold2t1w_dof == 6 + else "to account for distortions remaining in the BOLD reference" + ), ) inputnode = pe.Node( @@ -791,7 +793,7 @@ def init_fsl_bbr_wf(use_bbr, bold2t1w_dof, bold2t1w_init, sloppy=False, name="fs else: # Should mostly be hit while building docs LOGGER.warning("FSLDIR unset - using packaged BBR schedule") - flt_bbr.inputs.schedule = pkgr.resource_filename("fmriprep", "data/flirtsch/bbr.sch") + flt_bbr.inputs.schedule = load_data("flirtsch/bbr.sch") # fmt: off workflow.connect([ diff --git a/nibabies/workflows/bold/resampling.py b/nibabies/workflows/bold/resampling.py index aee2a0b5..a536ae47 100644 --- a/nibabies/workflows/bold/resampling.py +++ b/nibabies/workflows/bold/resampling.py @@ -24,7 +24,7 @@ from niworkflows.interfaces.workbench import MetricDilate, MetricMask, MetricResample from nibabies.config import DEFAULT_MEMORY_MIN_GB -from nibabies.data import load_resource +from nibabies.data import load as load_data if ty.TYPE_CHECKING: from niworkflows.utils.spaces import SpatialReferences @@ -589,7 +589,7 @@ def init_bold_fsLR_resampling_wf( # select white, midthickness and pial surfaces based on hemi select_surfaces = pe.Node(CiftiSelect(), name='select_surfaces') if mcribs: - atlases = load_resource('atlases') + atlases = load_data.cached('atlases') # use dHCP 32k fsLR instead select_surfaces.inputs.template_spheres = [ str(atlases / 'tpl-dHCP_space-fsLR_hemi-L_den-32k_desc-week42_sphere.surf.gii'), @@ -1099,12 +1099,14 @@ def init_bold_preproc_trans_wf( These resampled BOLD time-series will be referred to as *preprocessed BOLD in original space*, or just *preprocessed BOLD*. """.format( - transforms="""\ + transforms=( + """\ a single, composite transform to correct for head-motion and susceptibility distortions""" - if use_fieldwarp - else """\ + if use_fieldwarp + else """\ the transforms to correct for head-motion""" + ) ) inputnode = pe.Node( diff --git a/pyproject.toml b/pyproject.toml index 01bbf90e..c173da32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,14 +12,12 @@ classifiers = [ "Intended Audience :: Science/Research", "Topic :: Scientific/Engineering :: Image Recognition", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] license = {file = "LICENSE"} -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ - 'importlib_resources; python_version < "3.9"', "nibabel >= 5.0.0", "nipype >= 1.8.5", "nireports >= 23.2.0",