diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 340bb33f60..b0fc9ff51c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -78,7 +78,7 @@ from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES # noqa from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES -from easybuild.tools.config import PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS +from easybuild.tools.config import MOD_SEARCH_PATH_HEADERS, PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env @@ -100,7 +100,7 @@ from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX -from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment +from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment, MODULE_LOAD_ENV_HEADERS from easybuild.tools.modules import curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS @@ -221,7 +221,19 @@ def __init__(self, ec, logfile=None): self.modules_header = read_file(modules_header_path) # environment variables on module load - self.module_load_environment = ModuleLoadEnvironment() + mod_load_aliases = {} + # apply --module-search-path-headers: easyconfig parameter has precedence + mod_load_cpp_headers = self.cfg['module_search_path_headers'] or build_option('module_search_path_headers') + + try: + mod_load_aliases[MODULE_LOAD_ENV_HEADERS] = MOD_SEARCH_PATH_HEADERS[mod_load_cpp_headers] + except KeyError as err: + raise EasyBuildError( + f"Unknown value selected for option module-search-path-headers: {mod_load_cpp_headers}. " + f"Choose one of: {', '.join(MOD_SEARCH_PATH_HEADERS)}" + ) from err + + self.module_load_environment = ModuleLoadEnvironment(aliases=mod_load_aliases) # determine install subdirectory, based on module name self.install_subdir = None diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 4b2c7a0a1d..bca46c3856 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -208,10 +208,11 @@ 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], 'module_depends_on': [None, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' '(implies recursive unloading of modules) [DEPRECATED]', MODULES], + 'module_search_path_headers': [None, "Environment variable set by modules on load " + "with search paths to header files (if None, use $CPATH)", MODULES], 'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module " - "(True/False to hard enable/disable; None implies honoring " - "the --recursive-module-unload EasyBuild configuration setting", - MODULES], + "(True/False to hard enable/disable; None implies honoring the " + "--recursive-module-unload EasyBuild configuration setting", MODULES], # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ecce1a7f0d..3503d5c2f5 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -182,6 +182,15 @@ EBPYTHONPREFIXES = 'EBPYTHONPREFIXES' PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES] +# options to handle header search paths in environment of modules +MOD_SEARCH_PATH_HEADERS_CPATH = 'cpath' +MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS = 'include_paths' +MOD_SEARCH_PATH_HEADERS = { + MOD_SEARCH_PATH_HEADERS_CPATH: ['CPATH'], + MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS: ['C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH'], +} +DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH + class Singleton(ABCMeta): """Serves as metaclass for classes that should implement the Singleton pattern. @@ -389,6 +398,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_MINIMAL_BUILD_ENV: [ 'minimal_build_env', ], + DEFAULT_MOD_SEARCH_PATH_HEADERS: [ + 'module_search_path_headers', + ], DEFAULT_PKG_RELEASE: [ 'package_release', ], diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7d52cfef92..cba8810e74 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -57,6 +57,8 @@ from easybuild.tools.utilities import get_subclasses, nub +MODULE_LOAD_ENV_HEADERS = 'HEADERS' + # software root/version environment variable name prefixes ROOT_ENV_VAR_NAME_PREFIX = "EBROOT" VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION" @@ -241,18 +243,38 @@ def is_path(self): class ModuleLoadEnvironment: - """Changes to environment variables that should be made when environment module is loaded""" + """ + Changes to environment variables that should be made when environment module is loaded. + - Environment variables are defined as ModuleEnvironmentVariables instances + with attribute name equal to environment variable name. + - Aliases are arbitrary names that serve to apply changes to lists of + environment variables + - Only environment variables attributes are public. Other attributes like + aliases are private. + """ - def __init__(self): + def __init__(self, aliases=None): """ Initialize default environment definition Paths are relative to root of installation directory + + :aliases: dict defining environment variables aliases """ + self._aliases = {} + if aliases is not None: + try: + for alias_name, alias_vars in aliases.items(): + self.update_alias(alias_name, alias_vars) + except AttributeError as err: + raise EasyBuildError( + "Wrong format for aliases defitions passed to ModuleLoadEnvironment. " + f"Expected a dictionary but got: {type(aliases)}." + ) from err + self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')] self.CLASSPATH = ['*.jar'] self.CMAKE_LIBRARY_PATH = ['lib64'] # only needed for installations with standalone lib64 self.CMAKE_PREFIX_PATH = [''] - self.CPATH = SEARCH_PATH_HEADER_DIRS self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS] self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS self.LIBRARY_PATH = SEARCH_PATH_LIB_DIRS @@ -261,11 +283,29 @@ def __init__(self): self.PKG_CONFIG_PATH = [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']] self.XDG_DATA_DIRS = ['share'] + # environment variables with known aliases + # e.g. search paths to C/C++ headers + for envar_name in self._aliases.get(MODULE_LOAD_ENV_HEADERS, []): + setattr(self, envar_name, SEARCH_PATH_HEADER_DIRS) + def __setattr__(self, name, value): """ Specific restrictions for ModuleLoadEnvironment attributes: + - public attributes are instances of ModuleEnvironmentVariable with uppercase names + - private attributes are allowed with any name + """ + if name.startswith('_'): + # do not control protected/private attributes + return super().__setattr__(name, value) + + return self.__set_module_environment_variable(name, value) + + def __set_module_environment_variable(self, name, value): + """ + Specific restrictions for ModuleEnvironmentVariable attributes: - attribute names are uppercase - - attributes are instances of ModuleEnvironmentVariable + - dictionaries are unpacked into arguments of ModuleEnvironmentVariable + - controls variables with special types (e.g. PATH, LD_LIBRARY_PATH) """ if name != name.upper(): raise EasyBuildError(f"Names of ModuleLoadEnvironment attributes must be uppercase, got '{name}'") @@ -284,9 +324,15 @@ def __setattr__(self, name, value): return super().__setattr__(name, ModuleEnvironmentVariable(contents, **kwargs)) + @property + def vars(self): + """Return list of public ModuleEnvironmentVariable""" + + return [envar for envar in self.__dict__ if not str(envar).startswith('_')] + def __iter__(self): """Make the class iterable""" - yield from self.__dict__ + yield from self.vars def items(self): """ @@ -294,7 +340,8 @@ def items(self): - key = attribute name - value = its "contents" attribute """ - return self.__dict__.items() + for attr in self.vars: + yield attr, getattr(self, attr) def update(self, new_env): """Update contents of environment from given dictionary""" @@ -304,6 +351,14 @@ def update(self, new_env): except AttributeError as err: raise EasyBuildError("Cannot update ModuleLoadEnvironment from a non-dict variable") from err + def remove(self, var_name): + """ + Remove ModuleEnvironmentVariable attribute from instance + Silently goes through if attribute is already missing + """ + if var_name in self.vars: + delattr(self, var_name) + @property def as_dict(self): """ @@ -319,6 +374,48 @@ def environ(self): """ return {envar_name: str(envar_contents) for envar_name, envar_contents in self.items()} + def alias(self, alias): + """ + Return iterator to search path variables for given alias + """ + try: + yield from [getattr(self, envar) for envar in self._aliases[alias]] + except KeyError as err: + raise EasyBuildError(f"Unknown search path alias: {alias}") from err + except AttributeError as err: + raise EasyBuildError(f"Missing environment variable in '{alias} alias") from err + + def alias_vars(self, alias): + """ + Return list of environment variable names aliased by given alias + """ + try: + return self._aliases[alias] + except KeyError as err: + raise EasyBuildError(f"Unknown search path alias: {alias}") from err + + def update_alias(self, alias, value): + """ + Update existing or non-existing alias with given search paths variables + """ + if isinstance(value, str): + value = [value] + + try: + self._aliases[alias] = [str(envar) for envar in value] + except TypeError as err: + raise TypeError("ModuleLoadEnvironment aliases must be a list of strings") from err + + def set_alias_vars(self, alias, value): + """ + Set value of search paths variables for given alias + """ + try: + for envar_name in self._aliases[alias]: + setattr(self, envar_name, value) + except KeyError as err: + raise EasyBuildError(f"Unknown search path alias: {alias}") from err + class ModulesTool(object): """An abstract interface to a tool that deals with modules.""" diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9c4ad3c734..70671ce20f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -69,6 +69,7 @@ from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL +from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS @@ -615,6 +616,9 @@ def config_options(self): 'module-extensions': ("Include 'extensions' statement in generated module file (Lua syntax only)", None, 'store_true', True), 'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS), + 'module-search-path-headers': ("Environment variable set by modules on load with search paths " + "to header files", 'choice', 'store', DEFAULT_MOD_SEARCH_PATH_HEADERS, + sorted(MOD_SEARCH_PATH_HEADERS.keys())), 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, sorted(avail_module_generators().keys())), 'moduleclasses': (("Extend supported module classes " diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index a235732d23..97d8f31b93 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -532,7 +532,7 @@ def test_make_module_req(self): for env_var in default_mod_load_vars: delattr(eb.module_load_environment, env_var) - self.assertEqual(len(vars(eb.module_load_environment)), 0) + self.assertEqual(len(eb.module_load_environment.vars), 0) # check for behavior when a string value is used as value of module_load_environment eb.module_load_environment.PATH = 'bin' @@ -668,6 +668,99 @@ def test_make_module_req(self): eb.close_log() os.remove(eb.logfile) + def test_module_search_path_headers(self): + """Test functionality of module-search-path-headers option""" + sp_headers_mode = { + "cpath": ["CPATH"], + "include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], + } + + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + self.writeEC() + + for build_opt, sp_headers in sp_headers_mode.items(): + update_build_option('module_search_path_headers', build_opt) + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.installdir = config.install_path() + try: + os.makedirs(os.path.join(eb.installdir, 'include')) + write_file(os.path.join(eb.installdir, 'include', 'header.h'), 'dummy header file') + except FileExistsError: + pass + + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() + + if not sp_headers: + # none option adds nothing to module file + if get_module_syntax() == 'Tcl': + tcl_ref_pattern = r"^prepend-path\s+CPATH\s+\$root/include$" + self.assertFalse(re.search(tcl_ref_pattern, guess, re.M)) + elif get_module_syntax() == 'Lua': + lua_ref_pattern = r'^prepend_path\("CPATH", pathJoin\(root, "include"\)\)$' + self.assertFalse(re.search(lua_ref_pattern, guess, re.M)) + else: + for env_var in sp_headers: + if get_module_syntax() == 'Tcl': + tcl_ref_pattern = rf"^prepend-path\s+{env_var}\s+\$root/include$" + self.assertTrue(re.search(tcl_ref_pattern, guess, re.M)) + elif get_module_syntax() == 'Lua': + lua_ref_pattern = rf'^prepend_path\("{env_var}", pathJoin\(root, "include"\)\)$' + self.assertTrue(re.search(lua_ref_pattern, guess, re.M)) + + # test with easyconfig parameter + for ec_param, sp_headers in sp_headers_mode.items(): + self.contents += f'\nmodule_search_path_headers = "{ec_param}"' + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.installdir = config.install_path() + try: + os.makedirs(os.path.join(eb.installdir, 'include')) + write_file(os.path.join(eb.installdir, 'include', 'header.h'), 'dummy header file') + except FileExistsError: + pass + + for build_opt in sp_headers_mode: + update_build_option('module_search_path_headers', build_opt) + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() + if not sp_headers: + # none option adds nothing to module file + if get_module_syntax() == 'Tcl': + tcl_ref_pattern = r"^prepend-path\s+CPATH\s+\$root/include$" + self.assertFalse(re.search(tcl_ref_pattern, guess, re.M)) + elif get_module_syntax() == 'Lua': + lua_ref_pattern = r'^prepend_path\("CPATH", pathJoin\(root, "include"\)\)$' + self.assertFalse(re.search(lua_ref_pattern, guess, re.M)) + else: + for env_var in sp_headers: + if get_module_syntax() == 'Tcl': + tcl_ref_pattern = rf"^prepend-path\s+{env_var}\s+\$root/include$" + self.assertTrue(re.search(tcl_ref_pattern, guess, re.M)) + elif get_module_syntax() == 'Lua': + lua_ref_pattern = rf'^prepend_path\("{env_var}", pathJoin\(root, "include"\)\)$' + self.assertTrue(re.search(lua_ref_pattern, guess, re.M)) + + # test wrong easyconfig parameter + self.contents += '\nmodule_search_path_headers = "WRONG_OPT"' + self.writeEC() + ec = EasyConfig(self.eb_file) + + error_pattern = "Unknown value selected for option module-search-path-headers" + with eb.module_generator.start_module_creation(): + self.assertErrorRegex(EasyBuildError, error_pattern, EasyBlock, ec) + + # cleanup + eb.close_log() + os.remove(eb.logfile) + def test_make_module_extra(self): """Test for make_module_extra.""" init_config(build_options={'silent': True}) diff --git a/test/framework/modules.py b/test/framework/modules.py index 815d8697b2..dac136f148 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1708,8 +1708,10 @@ def test_module_load_environment(self): self.assertEqual(mod_load_env.TEST_VARTYPE.type, mod.ModEnvVarType.PATH) self.assertRaises(TypeError, setattr, mod_load_env, 'TEST_UNKNONW', (test_contents, {'unkown_param': True})) - # test retrieving environment + # test retrieval of environment + # use copy of public attributes as reference ref_load_env = mod_load_env.__dict__.copy() + ref_load_env = {envar: value for envar, value in ref_load_env.items() if not envar.startswith('_')} self.assertCountEqual(list(mod_load_env), ref_load_env.keys()) ref_load_env_item_list = list(ref_load_env.items()) @@ -1740,6 +1742,61 @@ def test_module_load_environment(self): self.assertTrue(hasattr(mod_load_env, 'TEST_STR')) self.assertEqual(mod_load_env.TEST_STR.contents, ['some/path']) + # test removal of envars + mod_load_env.remove('TEST_VARTYPE') + self.assertFalse(hasattr(mod_load_env, 'TEST_VARTYPE')) + mod_load_env.remove('NONEXISTENT') + + # test aliases + aliases = { + 'ALIAS1': ['ALIAS_VAR11', 'ALIAS_VAR12'], + 'ALIAS2': ['ALIAS_VAR21'], + } + alias_load_env = mod.ModuleLoadEnvironment(aliases=aliases) + self.assertEqual(alias_load_env._aliases, aliases) + self.assertEqual(sorted(alias_load_env.alias_vars('ALIAS1')), ['ALIAS_VAR11', 'ALIAS_VAR12']) + self.assertEqual(alias_load_env.alias_vars('ALIAS2'), ['ALIAS_VAR21']) + # set a known alias + alias_load_env.set_alias_vars('ALIAS1', 'alias1_path') + self.assertTrue(hasattr(alias_load_env, 'ALIAS_VAR11')) + self.assertEqual(alias_load_env.ALIAS_VAR11.contents, ['alias1_path']) + self.assertEqual(alias_load_env.ALIAS_VAR11.type, mod.ModEnvVarType.PATH_WITH_FILES) + self.assertTrue(hasattr(alias_load_env, 'ALIAS_VAR12')) + self.assertEqual(alias_load_env.ALIAS_VAR12.contents, ['alias1_path']) + self.assertEqual(alias_load_env.ALIAS_VAR12.type, mod.ModEnvVarType.PATH_WITH_FILES) + self.assertFalse(hasattr(alias_load_env, 'ALIAS_VAR21')) + for envar in alias_load_env.alias('ALIAS1'): + self.assertEqual(envar.contents, ['alias1_path']) + self.assertEqual(envar.type, mod.ModEnvVarType.PATH_WITH_FILES) + # set a second known alias + alias_load_env.set_alias_vars('ALIAS2', 'alias2_path') + self.assertTrue(hasattr(alias_load_env, 'ALIAS_VAR11')) + self.assertEqual(alias_load_env.ALIAS_VAR11.contents, ['alias1_path']) + self.assertEqual(alias_load_env.ALIAS_VAR11.type, mod.ModEnvVarType.PATH_WITH_FILES) + self.assertTrue(hasattr(alias_load_env, 'ALIAS_VAR21')) + self.assertEqual(alias_load_env.ALIAS_VAR21.contents, ['alias2_path']) + self.assertEqual(alias_load_env.ALIAS_VAR21.type, mod.ModEnvVarType.PATH_WITH_FILES) + # add a new alias + alias_load_env.update_alias('ALIAS3', 'ALIAS_VAR31') + self.assertEqual(alias_load_env.alias_vars('ALIAS3'), ['ALIAS_VAR31']) + alias_load_env.update_alias('ALIAS3', ['ALIAS_VAR31', 'ALIAS_VAR32']) + self.assertEqual(sorted(alias_load_env.alias_vars('ALIAS3')), ['ALIAS_VAR31', 'ALIAS_VAR32']) + alias_load_env.set_alias_vars('ALIAS3', 'alias3_path') + for envar in alias_load_env.alias('ALIAS3'): + self.assertEqual(envar.contents, ['alias3_path']) + self.assertEqual(envar.type, mod.ModEnvVarType.PATH_WITH_FILES) + # append path to existing alias + for envar in alias_load_env.alias('ALIAS3'): + envar.append('new_path') + self.assertEqual(sorted(envar.contents), ['alias3_path', 'new_path']) + self.assertEqual(alias_load_env.ALIAS_VAR31.contents, ['alias3_path', 'new_path']) + self.assertEqual(alias_load_env.ALIAS_VAR32.contents, ['alias3_path', 'new_path']) + + error_pattern = "Wrong format for aliases defitions passed to ModuleLoadEnvironment" + self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases=False) + self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases='wrong') + self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases=['some', 'list']) + def suite(): """ returns all the testcases in this module """