Merge pull request #1 from lexming/granular_exit_code

Add lexming changes about how to manage exit codes

Author: dagonzalezfo

.github/workflows/linting.yml

@@ -14,7 +14,6 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
-        
     steps:
     - uses: actions/checkout@v3
 

easybuild/base/optcomplete.py

@@ -107,6 +107,7 @@
 from optparse import OptionParser, Option
 from pprint import pformat
 
+from easybuild.tools.filetools import get_cwd
 from easybuild.tools.utilities import shell_quote
 
 debugfn = None  # for debugging only
@@ -537,7 +538,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
         # Note: this will get filtered properly below.
 
     completer_kwargs = {
-        'pwd': os.getcwd(),
+        'pwd': get_cwd(),
         'cline': cline,
         'cpoint': cpoint,
         'prefix': prefix,

easybuild/framework/easyblock.py

@@ -59,11 +59,12 @@
 from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
 from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
-from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS
+from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS, REPLACED_PARAMETERS
 from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
-from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict
+from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATES, DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS
+from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_DYNAMIC, template_constant_dict
 from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning, print_msg
 from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
 from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme
@@ -118,11 +119,13 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method):
     def new_ec_method(self, key, *args, **kwargs):
         """Check whether any replace easyconfig parameters are still used"""
         # map deprecated parameters to their replacements, issue deprecation warning(/error)
-        if key in DEPRECATED_PARAMETERS:
+        if key in ALTERNATE_PARAMETERS:
+            key = ALTERNATE_PARAMETERS[key]
+        elif key in DEPRECATED_PARAMETERS:
             depr_key = key
             key, ver = DEPRECATED_PARAMETERS[depr_key]
             _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver)
-        if key in REPLACED_PARAMETERS:
+        elif key in REPLACED_PARAMETERS:
             _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
         return ec_method(self, key, *args, **kwargs)
 
@@ -179,7 +182,7 @@ def triage_easyconfig_params(variables, ec):
 
     for key in variables:
         # validations are skipped, just set in the config
-        if key in ec or key in DEPRECATED_PARAMETERS.keys():
+        if any(key in d for d in (ec, DEPRECATED_PARAMETERS.keys(), ALTERNATE_PARAMETERS.keys())):
             ec_params[key] = variables[key]
             _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key]))
         elif key in REPLACED_PARAMETERS:
@@ -658,7 +661,7 @@ def set_keys(self, params):
         with self.disable_templating():
             for key in sorted(params.keys()):
                 # validations are skipped, just set in the config
-                if key in self._config.keys() or key in DEPRECATED_PARAMETERS.keys():
+                if any(key in x.keys() for x in (self._config, ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS)):
                     self[key] = params[key]
                     self.log.info("setting easyconfig parameter %s: value %s (type: %s)",
                                   key, self[key], type(self[key]))
@@ -827,7 +830,7 @@ def check_deprecated(self, path):
         if depr_msgs:
             depr_msg = ', '.join(depr_msgs)
 
-            depr_maj_ver = int(str(VERSION).split('.')[0]) + 1
+            depr_maj_ver = int(str(VERSION).split('.', maxsplit=1)[0]) + 1
             depr_ver = '%s.0' % depr_maj_ver
 
             more_info_depr_ec = " (see also https://docs.easybuild.io/deprecated-easyconfigs)"
@@ -842,8 +845,8 @@ def validate(self, check_osdeps=True):
         - check license
         """
         self.log.info("Validating easyconfig")
-        for attr in self.validations:
-            self._validate(attr, self.validations[attr])
+        for attr, valid_values in self.validations.items():
+            self._validate(attr, valid_values)
 
         if check_osdeps:
             self.log.info("Checking OS dependencies")
@@ -899,9 +902,12 @@ def validate_os_deps(self):
                 not_found.append(dep)
 
         if not_found:
-            raise EasyBuildError("One or more OS dependencies were not found: %s", not_found, exit_code=8)
-        else:
-            self.log.info("OS dependencies ok: %s" % self['osdependencies'])
+            raise EasyBuildError(
+                "One or more OS dependencies were not found: %s", not_found,
+                exit_code=EasyBuildExit.MISS_SYSTEM_DEPENDENCY
+            )
+
+        self.log.info("OS dependencies ok: %s" % self['osdependencies'])
 
         return True
 
@@ -1207,8 +1213,8 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
         # templated values should be dumped unresolved
         with self.disable_templating():
             # build dict of default values
-            default_values = {key: DEFAULT_CONFIG[key][0] for key in DEFAULT_CONFIG}
-            default_values.update({key: self.extra_options[key][0] for key in self.extra_options})
+            default_values = {key: value[0] for key, value in DEFAULT_CONFIG.items()}
+            default_values.update({key: value[0] for key, value in self.extra_options.items()})
 
             self.generate_template_values()
             templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS}
@@ -1264,7 +1270,10 @@ def _validate(self, attr, values):  # private method
         if values is None:
             values = []
         if self[attr] and self[attr] not in values:
-            raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values, exit_code=12)
+            raise EasyBuildError(
+                "%s provided '%s' is not valid: %s", attr, self[attr], values,
+                exit_code=EasyBuildExit.SYNTAX_ERROR
+            )
 
     def probe_external_module_metadata(self, mod_name, existing_metadata=None):
         """
@@ -1911,14 +1920,17 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
                 modname = modulepath.replace('easybuild.easyblocks.', '')
                 error_re = re.compile(r"No module named '?.*/?%s'?" % modname)
                 _log.debug("error regexp for ImportError on '%s' easyblock: %s", modname, error_re.pattern)
-                if error_re.match(str(err)):
-                    if error_on_missing_easyblock:
-                        raise EasyBuildError(
-                                "No software-specific easyblock '%s' found for %s", class_name, name, exit_code=4)
-                elif error_on_failed_import:
-                    raise EasyBuildError("Failed to import %s easyblock: %s", class_name, err, exit_code=5)
-                else:
-                    _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err))
+                if error_re.match(str(err)) and error_on_missing_easyblock:
+                    raise EasyBuildError(
+                        "No software-specific easyblock '%s' found for %s", class_name, name,
+                        exit_code=EasyBuildExit.MISS_EASYBLOCK
+                    )
+                if error_on_failed_import:
+                    raise EasyBuildError(
+                        "Failed to import %s easyblock: %s", class_name, err,
+                        exit_code=EasyBuildExit.EASYBLOCK_ERROR
+                    )
+                _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err))
 
         if cls is not None:
             _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')",
@@ -1933,7 +1945,9 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
         raise err
     except Exception as err:
         raise EasyBuildError(
-                "Failed to obtain class for %s easyblock (not available?): %s", easyblock, err, exit_code=6)
+            "Failed to obtain class for %s easyblock (not available?): %s", easyblock, err,
+            exit_code=EasyBuildExit.EASYBLOCK_ERROR
+        )
 
 
 def get_module_path(name, generic=None, decode=True):
@@ -1991,12 +2005,41 @@ def resolve_template(value, tmpl_dict):
         # '%(name)s' -> '%(name)s'
         # '%%(name)s' -> '%%(name)s'
         if '%' in value:
+            raw_value = value
             value = re.sub(re.compile(r'(%)(?!%*\(\w+\)s)'), r'\1\1', value)
 
             try:
                 value = value % tmpl_dict
             except KeyError:
-                _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict)
+                # check if any alternate and/or deprecated templates resolve
+                try:
+                    orig_value = value
+                    # map old templates to new values for alternate and deprecated templates
+                    alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in
+                               ALTERNATE_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+                    alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in
+                                ALTERNATE_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()}
+                    depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in
+                                DEPRECATED_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+
+                    # try templating with alternate and deprecated templates included
+                    value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map}
+
+                    for old_tmpl, val in depr_map.items():
+                        # check which deprecated templates were replaced, and issue deprecation warnings
+                        if old_tmpl in orig_value and val in value:
+                            new_tmpl, ver = DEPRECATED_TEMPLATES[old_tmpl]
+                            _log.deprecated(f"Easyconfig template '{old_tmpl}' is deprecated, use '{new_tmpl}' instead",
+                                            ver)
+                except KeyError:
+                    _log.warning(f"Unable to resolve template value {value} with dict {tmpl_dict}")
+                    value = raw_value  # Undo "%"-escaping
+
+                for key in tmpl_dict:
+                    if key in DEPRECATED_TEMPLATES:
+                        new_key, ver = DEPRECATED_TEMPLATES[key]
+                        _log.deprecated(f"Easyconfig template '{key}' is deprecated, use '{new_key}' instead", ver)
+
     else:
         # this block deals with references to objects and returns other references
         # for reading this is ok, but for self['x'] = {}
@@ -2050,10 +2093,10 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False,
             ec = EasyConfig(spec, build_specs=build_specs, validate=validate, hidden=hidden)
         except EasyBuildError as err:
             try:
-                err.exit_code
+                exit_code = err.exit_code
             except AttributeError:
-                err.exit_code = 1
-            raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg, exit_code=err.exit_code)
+                exit_code = EasyBuildExit.EASYCONFIG_ERROR
+            raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg, exit_code=exit_code)
 
         name = ec['name']
 

easybuild/framework/easyconfig/easyconfig.py

@@ -39,6 +39,7 @@
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
 from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
+from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATE_CONSTANTS, DEPRECATED_TEMPLATE_CONSTANTS
 from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.configobj import ConfigObj
@@ -86,6 +87,58 @@ def build_easyconfig_variables_dict():
     return vars_dict
 
 
+def handle_deprecated_constants(method):
+    """Decorator to handle deprecated easyconfig template constants"""
+    def wrapper(self, key, *args, **kwargs):
+        """Check whether any deprecated constants are used"""
+        alternate = ALTERNATE_TEMPLATE_CONSTANTS
+        deprecated = DEPRECATED_TEMPLATE_CONSTANTS
+        if key in alternate:
+            key = alternate[key]
+        elif key in deprecated:
+            depr_key = key
+            key, ver = deprecated[depr_key]
+            _log.deprecated(f"Easyconfig template constant '{depr_key}' is deprecated, use '{key}' instead", ver)
+        return method(self, key, *args, **kwargs)
+    return wrapper
+
+
+class DeprecatedDict(dict):
+    """Custom dictionary that handles deprecated easyconfig template constants gracefully"""
+
+    def __init__(self, *args, **kwargs):
+        self.clear()
+        self.update(*args, **kwargs)
+
+    @handle_deprecated_constants
+    def __contains__(self, key):
+        return super().__contains__(key)
+
+    @handle_deprecated_constants
+    def __delitem__(self, key):
+        return super().__delitem__(key)
+
+    @handle_deprecated_constants
+    def __getitem__(self, key):
+        return super().__getitem__(key)
+
+    @handle_deprecated_constants
+    def __setitem__(self, key, value):
+        return super().__setitem__(key, value)
+
+    def update(self, *args, **kwargs):
+        if args:
+            if isinstance(args[0], dict):
+                for key, value in args[0].items():
+                    self.__setitem__(key, value)
+            else:
+                for key, value in args[0]:
+                    self.__setitem__(key, value)
+
+        for key, value in kwargs.items():
+            self.__setitem__(key, value)
+
+
 class EasyConfigFormatConfigObj(EasyConfigFormat):
     """
     Extended EasyConfig format, with support for a header and sections that are actually parsed (as opposed to exec'ed).
@@ -176,7 +229,7 @@ def parse_header(self, header):
 
     def parse_pyheader(self, pyheader):
         """Parse the python header, assign to docstring and cfg"""
-        global_vars = self.pyheader_env()
+        global_vars = DeprecatedDict(self.pyheader_env())
         self.log.debug("pyheader initial global_vars %s", global_vars)
         self.log.debug("pyheader text being exec'ed: %s", pyheader)
 

easybuild/framework/easyconfig/format/pyheaderconfigobj.py

@@ -42,6 +42,64 @@
 from easybuild.tools.filetools import read_file, write_file
 
 
+# alternate easyconfig parameters, and their non-deprecated equivalents
+ALTERNATE_PARAMETERS = {
+    # <new_param>: <equivalent_param>,
+    'build_deps': 'builddependencies',
+    'build_in_install_dir': 'buildininstalldir',
+    'build_opts': 'buildopts',
+    'build_stats': 'buildstats',
+    'clean_up_old_build': 'cleanupoldbuild',
+    'clean_up_old_install': 'cleanupoldinstall',
+    'configure_opts': 'configopts',
+    'deps': 'dependencies',
+    'doc_paths': 'docpaths',
+    'doc_urls': 'docurls',
+    'do_not_create_install_dir': 'dontcreateinstalldir',
+    'exts_class_map': 'exts_classmap',
+    'exts_default_class': 'exts_defaultclass',
+    'exts_default_opts': 'exts_default_options',
+    'hidden_deps': 'hiddendependencies',
+    'include_modulepath_exts': 'include_modpath_extensions',
+    'install_opts': 'installopts',
+    'keep_previous_install': 'keeppreviousinstall',
+    'keep_symlinks': 'keepsymlinks',
+    'max_parallel': 'maxparallel',
+    'env_mod_aliases': 'modaliases',
+    'env_mod_alt_soft_name': 'modaltsoftname',
+    'modulepath_prepend_paths': 'moddependpaths',
+    'env_mod_extra_paths_append': 'modextrapaths_append',
+    'env_mod_extra_paths': 'modextrapaths',
+    'env_mod_extra_vars': 'modextravars',
+    'env_mod_load_msg': 'modloadmsg',
+    'env_mod_lua_footer': 'modluafooter',
+    'env_mod_tcl_footer': 'modtclfooter',
+    'env_mod_category': 'moduleclass',
+    'env_mod_depends_on': 'module_depends_on',
+    'env_mod_force_unload': 'moduleforceunload',
+    'env_mod_load_no_conflict': 'moduleloadnoconflict',
+    'env_mod_unload_msg': 'modunloadmsg',
+    'only_toolchain_env_mod': 'onlytcmod',
+    'os_deps': 'osdependencies',
+    'post_install_cmds': 'postinstallcmds',
+    'post_install_msgs': 'postinstallmsgs',
+    'post_install_patches': 'postinstallpatches',
+    'pre_build_opts': 'prebuildopts',
+    'pre_configure_opts': 'preconfigopts',
+    'pre_install_opts': 'preinstallopts',
+    'pre_test_opts': 'pretestopts',
+    'recursive_env_mod_unload': 'recursive_module_unload',
+    'run_test': 'runtest',
+    'sanity_check_cmds': 'sanity_check_commands',
+    'skip_fortran_mod_files_sanity_check': 'skip_mod_files_sanity_check',
+    'skip_steps': 'skipsteps',
+    'test_opts': 'testopts',
+    'toolchain_opts': 'toolchainopts',
+    'unpack_opts': 'unpack_options',
+    'version_prefix': 'versionprefix',
+    'version_suffix': 'versionsuffix',
+}
+
 # deprecated easyconfig parameters, and their replacements
 DEPRECATED_PARAMETERS = {
     # <old_param>: (<new_param>, <deprecation_version>),