Add an --extra-source-urls CLI option to fetch sources from additional URLs.

Add the --extra-source-urls CLI option, a | separated list of
URLs that EasyBuild will fetch sources from. It replaces the hard-coded
EASYBUILD_SOURCES_URL, but keeps it as a default value. Uses the add_flex
logic to prepend, append, or insert into the default list.

Co-authored-by: Alexander Grund <[email protected]>

Author: joeydumont

easybuild/base/generaloption.py

@@ -103,6 +103,11 @@ def what_str_list_tuple(name):
         sep = os.pathsep
         helpsep = 'pathsep'
 
+    elif name.startswith('url'):
+        # | is one of the only characters not in the grammar for URIs (RFC3986)
+        sep = '|'
+        helpsep = '|'
+
     klass = None
     if name.endswith('list'):
         klass = list
@@ -182,6 +187,7 @@ class ExtOption(CompleterOption):
           - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings
           - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings
               - the path separator is OS-dependent
+          - urllist, urltuple: using '|', convert urlsep separated string in a list resp. tuple of strings
     """
     EXTEND_SEPARATOR = ','
 
@@ -198,7 +204,7 @@ class ExtOption(CompleterOption):
     TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR
     ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
 
-    TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']]
+    TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path', 'url']]
     TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST}
     TYPE_CHECKER.update(Option.TYPE_CHECKER)
     TYPES = tuple(TYPE_STRLIST + list(Option.TYPES))

easybuild/framework/easyblock.py

@@ -106,8 +106,6 @@
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
 
-EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
-
 DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64')
 
 MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
@@ -897,8 +895,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     source_urls = []
                 source_urls.extend(self.cfg['source_urls'])
 
-                # add https://sources.easybuild.io as fallback source URL
-                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
+                # add extra-source-urls CLI as either a first check, or a fallback.
+                self.log.warning("[extra_source_urls] %s", build_option('extra_source_urls'))
+                for url in build_option("extra_source_urls"):
+                    url += "/" + name_letter + "/" + location
+                    self.log.warning("[extra_source_urls] url is %s", url)
+                    source_urls.extend([url])
+                    self.log.warning("[extra_source_urls] %s", source_urls)
 
                 mkdir(targetdir, parents=True)
 

easybuild/tools/config.py

@@ -120,6 +120,7 @@
 DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders'
 DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild")
 DEFAULT_REPOSITORY = 'FileRepository'
+DEFAULT_EXTRA_SOURCE_URLS = ('https://sources.easybuild.io/',)
 # Filter these CUDA libraries by default from the RPATH sanity check.
 # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build
 # against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should
@@ -215,6 +216,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'easyblock',
         'envvars_user_modules',
         'extra_modules',
+        'extra_source_urls',
         'filter_deps',
         'filter_ecs',
         'filter_env_vars',

easybuild/tools/options.py

@@ -69,10 +69,12 @@
 from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
 from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
 from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
-from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT
+from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
+from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT
 from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
 from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS
-from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
+from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR
+from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
 from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS
 from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN
@@ -407,6 +409,8 @@ def override_options(self):
                              None, 'store_true', False),
             'extra-modules': ("List of extra modules to load after setting up the build environment",
                               'strlist', 'extend', None),
+            "extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig",
+                                  "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[%sURL]' % '|'}),
             'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, "
                       "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False),
             'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "