Merge pull request #4635 from easybuilders/4.9.x

release EasyBuild v4.9.3

Author: boegel

.github/workflows/end2end.yml

@@ -18,6 +18,7 @@ jobs:
       fail-fast: false
     container:
       image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3

.github/workflows/unit_tests.yml

@@ -101,12 +101,12 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
         if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
-            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
+            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
           fi
           echo "GitHub token installed!"
         else
@@ -191,7 +191,17 @@ jobs:
             # run test suite
             python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
             # try and make sure output of running tests is clean (no printed messages/warnings)
-            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS="no GitHub token available"
+            IGNORE_PATTERNS+="|skipping SvnRepository test"
+            IGNORE_PATTERNS+="|requires Lmod as modules tool"
+            IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]"
+            IGNORE_PATTERNS+="|from cryptography.* import "
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2"
+            IGNORE_PATTERNS+="|Blowfish"
+            IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
+            IGNORE_PATTERNS+="|algorithms.TripleDES"
             # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
             PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
             test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)

.github/workflows/unit_tests_python2.yml

@@ -16,6 +16,7 @@ jobs:
       # CentOS 7.9 container that already includes Lmod & co,
       # see https://github.com/easybuilders/easybuild-containers
       image: ghcr.io/easybuilders/centos-7.9-amd64
+    env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
     steps:
     - uses: actions/checkout@v3
 

RELEASE_NOTES

@@ -4,6 +4,43 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.3 (14 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add support for `--extra-source-urls` to fetch sources from additional URLs (#4079)
+  - add definition for gmpflf toolchain (#4566, #4571)
+  - reuse pre-computed checksums (#4569)
+  - add `cuda_cc_space_sep` variant that does not have periods (#4583)
+  - add `--skip-sanity-check` option (#4590)
+  - add `GNU_FTP_SOURCE` template constant (#4597)
+  - improve error messages for empty easyconfigs (#4603)
+  - improve help string for `--dep-graph` (#4610)
+  - only call `_sanity_check_step_extensions` if `--skip-extensions` is not set (#4620)
+  - add support for `--software-commit` and an associated template `%(software_commit)s` (#4628)
+- various bug fixes, including:
+  - correctly evaluate result for `--dep-graph` (#4554)
+  - fix fetch progress bar showing to many files (#4568)
+  - resolve internal for imkl>=2021 version subdir via "latest" symlink (#4570)
+  - fix typo in message about including an easyblock from a commit (#4575)
+  - don't use special flags for `strict`, `precise`, `loose`, `veryloose` toolchain options on RISC-V (#4576)
+  - fix help text for `cuda_compute_capabilities` template (#4589)
+  - fix help message for `--http-headers-fields-urlpat` configuration option (#4594)
+  - fix `test_compiler_cache` in case `gcc` is available multiple times (#4599)
+  - handle post-install patches in check_checksums_for (#4605)
+  - fix `copy_file` with a folder as the target (#4609)
+  - allow for case where `homepage = None` when generating the docs (#4626)
+  - fix test_github_det_commit_status by using more recent commits (#4636)
+- other changes:
+  - clean up code that was only there to support Python 2.6 + avoid syntax warnings when parsing py2vs3/py.p2 with Python 3.x (#3788)
+  - use Intel's oneAPI Fortran compiler by default for version 2024.0.0 and newer (`oneapi_fortran` toolchain option set to `True`) (#4567)
+  - allow using Node 16 actions in CI (#4574)
+  - remove a superflous check in `EasyBlock.run_all_steps` (#4623)
+  - remove trailing dots from backup message produced by --inject-checksums (#4632)
+
+
 v4.9.2 (12 June 2024)
 ---------------------
 

easybuild/base/generaloption.py

@@ -97,11 +97,16 @@ def what_str_list_tuple(name):
     """Given name, return separator, class and helptext wrt separator.
         (Currently supports strlist, strtuple, pathlist, pathtuple)
     """
-    sep = ','
-    helpsep = 'comma'
     if name.startswith('path'):
         sep = os.pathsep
         helpsep = 'pathsep'
+    elif name.startswith('url'):
+        # | is one of the only characters not in the grammar for URIs (RFC3986)
+        sep = '|'
+        helpsep = '|'
+    else:
+        sep = ','
+        helpsep = 'comma'
 
     klass = None
     if name.endswith('list'):
@@ -182,6 +187,7 @@ class ExtOption(CompleterOption):
           - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings
           - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings
               - the path separator is OS-dependent
+          - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings
     """
     EXTEND_SEPARATOR = ','
 
@@ -198,7 +204,7 @@ class ExtOption(CompleterOption):
     TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR
     ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
 
-    TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']]
+    TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path', 'url']]
     TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST}
     TYPE_CHECKER.update(Option.TYPE_CHECKER)
     TYPES = tuple(TYPE_STRLIST + list(Option.TYPES))

easybuild/framework/easyblock.py

@@ -72,6 +72,7 @@
 from easybuild.tools.build_log import print_error, print_msg, print_warning
 from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
+from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
@@ -105,9 +106,6 @@
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
-
-EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
-
 DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64')
 
 MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
@@ -674,14 +672,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         src_fn = os.path.basename(src_path)
 
                         # report both MD5 and SHA256 checksums, since both are valid default checksum types
+                        src_checksums = {}
                         for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
                             src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
+                            src_checksums[checksum_type] = src_checksum
                             self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
 
                         # verify checksum (if provided)
                         self.log.debug('Verifying checksums for extension source...')
                         fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
-                        if verify_checksum(src_path, fn_checksum):
+                        if verify_checksum(src_path, fn_checksum, src_checksums):
                             self.log.info('Checksum for extension source %s verified', src_fn)
                         elif build_option('ignore_checksums'):
                             print_warning("Ignoring failing checksum verification for %s" % src_fn)
@@ -700,12 +700,15 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         ext_src.update({'patches': ext_patches})
 
                         if verify_checksums:
+                            computed_checksums = {}
                             for patch in ext_patches:
                                 patch = patch['path']
+                                computed_checksums[patch] = {}
                                 # report both MD5 and SHA256 checksums,
                                 # since both are valid default checksum types
                                 for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
                                     checksum = compute_checksum(patch, checksum_type=checksum_type)
+                                    computed_checksums[patch][checksum_type] = checksum
                                     self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
 
                             # verify checksum (if provided)
@@ -715,7 +718,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                                 patch_fn = os.path.basename(patch)
 
                                 checksum = self.get_checksum_for(checksums, filename=patch_fn, index=idx+1)
-                                if verify_checksum(patch, checksum):
+                                if verify_checksum(patch, checksum, computed_checksums[patch]):
                                     self.log.info('Checksum for extension patch %s verified', patch_fn)
                                 elif build_option('ignore_checksums'):
                                     print_warning("Ignoring failing checksum verification for %s" % patch_fn)
@@ -754,7 +757,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         """
         srcpaths = source_paths()
 
-        update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
+        # We don't account for the checksums file in the progress bar
+        if filename != 'checksum.json':
+            update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
 
         if alt_location is None:
             location = self.name
@@ -890,8 +895,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     source_urls = []
                 source_urls.extend(self.cfg['source_urls'])
 
-                # add https://sources.easybuild.io as fallback source URL
-                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
+                # Add additional URLs as configured.
+                for url in build_option("extra_source_urls"):
+                    url += "/" + name_letter + "/" + location
+                    source_urls.append(url)
 
                 mkdir(targetdir, parents=True)
 
@@ -2475,7 +2482,7 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
         checksum_issues = []
 
         sources = ent.get('sources', [])
-        patches = ent.get('patches', [])
+        patches = ent.get('patches', []) + ent.get('postinstallpatches', [])
         checksums = ent.get('checksums', [])
         # Single source should be re-wrapped as a list, and checksums with it
         if isinstance(sources, dict):
@@ -3482,10 +3489,7 @@ def _sanity_check_step_extensions(self):
         """Sanity check on extensions (if any)."""
         failed_exts = []
 
-        if build_option('skip_extensions'):
-            self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...")
-            return
-        elif not self.ext_instances:
+        if not self.ext_instances:
             # class instances for extensions may not be initialized yet here,
             # for example when using --module-only or --sanity-check-only
             self.prepare_for_extensions()
@@ -3637,7 +3641,10 @@ def xs2str(xs):
 
         # also run sanity check for extensions (unless we are an extension ourselves)
         if not extension:
-            self._sanity_check_step_extensions()
+            if build_option('skip_extensions'):
+                self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...")
+            else:
+                self._sanity_check_step_extensions()
 
         linked_shared_lib_fails = self.sanity_check_linked_shared_libs()
         if linked_shared_lib_fails:
@@ -3894,12 +3901,13 @@ def update_config_template_run_step(self):
         self.cfg.generate_template_values()
 
     def skip_step(self, step, skippable):
-        """Dedice whether or not to skip the specified step."""
+        """Decide whether or not to skip the specified step."""
         skip = False
         force = build_option('force')
         module_only = build_option('module_only') or self.cfg['module_only']
         sanity_check_only = build_option('sanity_check_only')
         skip_extensions = build_option('skip_extensions')
+        skip_sanity_check = build_option('skip_sanity_check')
         skip_test_step = build_option('skip_test_step')
         skipsteps = self.cfg['skipsteps']
 
@@ -3927,6 +3935,10 @@ def skip_step(self, step, skippable):
             self.log.info("Skipping %s step because of sanity-check-only mode", step)
             skip = True
 
+        elif skip_sanity_check and step == SANITYCHECK_STEP:
+            self.log.info("Skipping %s step as request via skip-sanity-check", step)
+            skip = True
+
         elif skip_extensions and step == EXTENSIONS_STEP:
             self.log.info("Skipping %s step as requested via skip-extensions", step)
             skip = True
@@ -3937,9 +3949,9 @@ def skip_step(self, step, skippable):
 
         else:
             msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, "
-            msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s)"
+            msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s, skip_sanity_check: %s)"
             self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force,
-                           sanity_check_only, skip_extensions, skip_test_step)
+                           sanity_check_only, skip_extensions, skip_test_step, skip_sanity_check)
 
         return skip
 
@@ -4089,7 +4101,7 @@ def run_all_steps(self, run_test_cases):
         Build and install this software.
         run_test_cases (bool): run tests after building (e.g.: make test)
         """
-        if self.cfg['stop'] and self.cfg['stop'] == 'cfg':
+        if self.cfg['stop'] == 'cfg':
             return True
 
         steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
@@ -4708,7 +4720,7 @@ def make_checksum_lines(checksums, indent_level):
 
         # back up easyconfig file before injecting checksums
         ec_backup = back_up_file(ec['spec'])
-        print_msg("backup of easyconfig file saved to %s..." % ec_backup, log=_log)
+        print_msg("backup of easyconfig file saved to %s" % ec_backup, log=_log)
 
         # compute & inject checksums for sources/patches
         print_msg("injecting %s checksums for sources & patches in %s..." % (checksum_type, ec_fn), log=_log)

easybuild/framework/easyconfig/easyconfig.py

@@ -46,6 +46,7 @@
 import functools
 import os
 import re
+from collections import OrderedDict
 from contextlib import contextmanager
 
 import easybuild.tools.filetools as filetools
@@ -74,7 +75,7 @@
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
 from easybuild.tools.modules import modules_tool, NoModulesTool
-from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
+from easybuild.tools.py2vs3 import create_base_metaclass, string_type
 from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
@@ -455,6 +456,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
             self.path = path
             self.rawtxt = read_file(path)
             self.log.debug("Raw contents from supplied easyconfig file %s: %s", path, self.rawtxt)
+            if not self.rawtxt.strip():
+                raise EasyBuildError('Easyconfig file is empty')
         else:
             self.rawtxt = rawtxt
             self.log.debug("Supplied raw easyconfig contents: %s" % self.rawtxt)
@@ -1909,7 +1912,9 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
         else:
             # if no easyblock specified, try to find if one exists
             if name is None:
-                name = "UNKNOWN"
+                if error_on_missing_easyblock:
+                    raise EasyBuildError("No easyblock found as neither name nor easyblock were specified")
+                return None
             # The following is a generic way to calculate unique class names for any funny software title
             class_name = encode_class_name(name)
             # modulepath will be the namespace + encoded modulename (from the classname)