diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 9791eb3dcc..9ac971d7ba 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -331,6 +331,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'modules_tool_version_check', 'mpi_tests', 'pre_create_installdir', + 'replace_pythonpath', 'show_progress_bar', 'trace', ], diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index c537003de5..03e17b8af2 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -42,6 +42,7 @@ from contextlib import contextmanager from easybuild.tools import LooseVersion from textwrap import wrap +from typing import List, Union, Sequence from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_warning @@ -205,63 +206,79 @@ def get_modules_path(self, fake=False, mod_path_suffix=None): return os.path.join(mod_path, mod_path_suffix) - def _filter_paths(self, key, paths): + def _filter_paths(self, key: str, paths: List[str]) -> List[str]: """Filter out paths already added to key and return the remaining ones""" + assert isinstance(paths, list) + if self.added_paths_per_key is None: # For compatibility this is only a warning for now and we don't filter any paths print_warning('Module creation has not been started. Call start_module_creation first!') return paths added_paths = self.added_paths_per_key.setdefault(key, set()) - # paths can be a string - if isinstance(paths, str): - if paths in added_paths: - filtered_paths = None - else: - added_paths.add(paths) - filtered_paths = paths - else: - # Coerce any iterable/generator into a list - if not isinstance(paths, list): - paths = list(paths) - filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)] + filtered_paths = [x for x in paths if x not in added_paths and not added_paths.add(x)] if filtered_paths != paths: removed_paths = paths if filtered_paths is None else [x for x in paths if x not in filtered_paths] print_warning("Suppressed adding the following path(s) to $%s of the module as they were already added: %s", - key, removed_paths, + key, ', '.join(removed_paths), log=self.log) - if not filtered_paths: - filtered_paths = None return filtered_paths - def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True): + def append_paths(self, key: str, paths: Union[str, Sequence[str]], allow_abs=False, expand_relpaths=True) -> str: """ Generate append-path statements for the given list of paths. :param key: environment variable to append paths to - :param paths: list of paths to append + :param paths: single or list of paths to append :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ - paths = self._filter_paths(key, paths) - if paths is None: - return '' return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths) - def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True): + def prepend_paths(self, key: str, paths: Union[str, Sequence[str]], allow_abs=False, expand_relpaths=True) -> str: """ Generate prepend-path statements for the given list of paths. :param key: environment variable to append paths to - :param paths: list of paths to append + :param paths: single or list of paths to prepend :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ - paths = self._filter_paths(key, paths) - if paths is None: - return '' return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths) + def update_paths(self, key: str, paths: Union[str, Sequence[str]], prepend=True, allow_abs=False, + expand_relpaths=True) -> str: + """ + Generate append/prepend-path statements for the given list of paths. + + :param key: environment variable to append paths to + :param paths: single or list of paths to add + :param allow_abs: allow providing of absolute paths + :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) + """ + if isinstance(paths, str): + self.log.debug("Wrapping %s into a list before using it for %s", paths, key) + paths = [paths] + elif not isinstance(paths, list): + paths = list(paths) # coerce any iterator into a list + + if key == 'PYTHONPATH': + python_paths = [path for path in paths if re.match(r'lib(64)?/python\d+\.\d+/site-packages', path)] + if len(python_paths) > 1: + raise EasyBuildError('Found multiple paths for PYTHONPATH: ' + ', '.join(python_paths)) + + # Special condition for PYTHONPATHs that match the standard pattern, + # replace with EBPYTHONPREFIXES which is added to python sys path at runtime via sitecustomize.py + if python_paths and build_option('replace_pythonpath'): + python_path = python_paths[0] + self.log.info("Replaced PYTHONPATH %s with EBPYTHONPREFIXES", python_path) + ret = self._update_paths('EBPYTHONPREFIXES', [''], prepend, allow_abs, expand_relpaths) + paths = [path for path in paths if path != python_path] + if paths: + ret += self._update_paths(key, paths, prepend, allow_abs, expand_relpaths) + return ret + return self._update_paths(key, paths, prepend, allow_abs, expand_relpaths) + def _modulerc_check_module_version(self, module_version): """ Check value type & contents of specified module-version spec. @@ -551,12 +568,12 @@ def unload_module(self, mod_name): """ raise NotImplementedError - def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True): + def _update_paths(self, key: str, paths: List[str], prepend=True, allow_abs=False, expand_relpaths=True) -> str: """ Generate prepend-path or append-path statements for the given list of paths. :param key: environment variable to prepend/append paths to - :param paths: list of paths to prepend + :param paths: list of paths to add :param prepend: whether to prepend (True) or append (False) paths :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) @@ -955,16 +972,18 @@ def msg_on_unload(self, msg): print_cmd = "puts stderr %s" % quote_str(msg, tcl=True) return '\n'.join(['', self.conditional_statement("module-info mode unload", print_cmd, indent=False)]) - def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True): + def _update_paths(self, key: str, paths: list, prepend=True, allow_abs=False, expand_relpaths=True) -> str: """ Generate prepend-path or append-path statements for the given list of paths. :param key: environment variable to prepend/append paths to - :param paths: list of paths to prepend + :param paths: list of paths to add :param prepend: whether to prepend (True) or append (False) paths :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ + paths = self._filter_paths(key, paths) + if prepend: update_type = 'prepend' else: @@ -974,10 +993,6 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key) return '' - if isinstance(paths, str): - self.log.debug("Wrapping %s into a list before using it to %s path %s", paths, update_type, key) - paths = [paths] - abspaths = [] for path in paths: if os.path.isabs(path) and not allow_abs: @@ -1426,16 +1441,18 @@ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None): return super(ModuleGeneratorLua, self).modulerc(module_version=module_version, filepath=filepath, modulerc_txt=modulerc_txt) - def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True): + def _update_paths(self, key: str, paths: List[str], prepend=True, allow_abs=False, expand_relpaths=True) -> str: """ Generate prepend_path or append_path statements for the given list of paths :param key: environment variable to prepend/append paths to - :param paths: list of paths to prepend/append + :param paths: list of paths to add :param prepend: whether to prepend (True) or append (False) paths :param allow_abs: allow providing of absolute paths :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir) """ + paths = self._filter_paths(key, paths) + if prepend: update_type = 'prepend' else: @@ -1445,10 +1462,6 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath self.log.info("Not including statement to %s environment variable $%s, as specified", update_type, key) return '' - if isinstance(paths, str): - self.log.debug("Wrapping %s into a list before using it to %s path %s", update_type, paths, key) - paths = [paths] - abspaths = [] for path in paths: if os.path.isabs(path): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 54e9b81982..419a1213b0 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -489,6 +489,8 @@ def override_options(self): 'remove-ghost-install-dirs': ("Remove ghost installation directories when --force or --rebuild is used, " "rather than just warning about them", None, 'store_true', False), + 'replace-pythonpath': ("Replaces PYTHONPATH with EBPYTHONPREFIXES in modules when possible", + None, 'store_true', True), 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " "which must be linked in all installed binaries/libraries", 'strlist', 'extend', None), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 075cc3adbd..023b04ed66 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -694,6 +694,9 @@ def append_paths(*args, **kwargs): res = append_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual("append-path\tkey\t\t1234@example.com\n", res) + expected = "append-path\tEBPYTHONPREFIXES\t\t$root\nappend-path\tPYTHONPATH\t\t$root/foo\n" + res = append_paths('PYTHONPATH', ['lib/python3.12/site-packages', 'foo']) + self.assertEqual(expected, res) else: expected = ''.join([ 'append_path("key", pathJoin(root, "path1"))\n', @@ -714,6 +717,10 @@ def append_paths(*args, **kwargs): res = append_paths('key', ['1234@example.com'], expand_relpaths=False) self.assertEqual('append_path("key", "1234@example.com")\n', res) + expected = 'append_path("EBPYTHONPREFIXES", root)\nappend_path("PYTHONPATH", pathJoin(root, "foo"))\n' + res = append_paths('PYTHONPATH', ['lib/python3.12/site-packages', 'foo']) + self.assertEqual(expected, res) + self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to update_paths " "which only expects relative paths." % self.modgen.app.installdir, append_paths, "key2", ["bar", "%s/foo" % self.modgen.app.installdir])