Merge pull request #4079 from joeydumont/source_url_cli_option

add support for `--extra-source-urls` to fetch sources from additional URLs

Author: boegel

easybuild/base/generaloption.py

@@ -97,11 +97,16 @@ def what_str_list_tuple(name):
     """Given name, return separator, class and helptext wrt separator.
         (Currently supports strlist, strtuple, pathlist, pathtuple)
     """
-    sep = ','
-    helpsep = 'comma'
     if name.startswith('path'):
         sep = os.pathsep
         helpsep = 'pathsep'
+    elif name.startswith('url'):
+        # | is one of the only characters not in the grammar for URIs (RFC3986)
+        sep = '|'
+        helpsep = '|'
+    else:
+        sep = ','
+        helpsep = 'comma'
 
     klass = None
     if name.endswith('list'):
@@ -182,6 +187,7 @@ class ExtOption(CompleterOption):
           - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings
           - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings
               - the path separator is OS-dependent
+          - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings
     """
     EXTEND_SEPARATOR = ','
 
@@ -198,7 +204,7 @@ class ExtOption(CompleterOption):
     TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR
     ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
 
-    TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']]
+    TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path', 'url']]
     TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST}
     TYPE_CHECKER.update(Option.TYPE_CHECKER)
     TYPES = tuple(TYPE_STRLIST + list(Option.TYPES))

easybuild/framework/easyblock.py

@@ -72,6 +72,7 @@
 from easybuild.tools.build_log import print_error, print_msg, print_warning
 from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
+from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
@@ -105,9 +106,6 @@
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
-
-EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
-
 DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64')
 
 MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
@@ -897,8 +895,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     source_urls = []
                 source_urls.extend(self.cfg['source_urls'])
 
-                # add https://sources.easybuild.io as fallback source URL
-                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
+                # Add additional URLs as configured.
+                for url in build_option("extra_source_urls"):
+                    url += "/" + name_letter + "/" + location
+                    source_urls.append(url)
 
                 mkdir(targetdir, parents=True)
 

easybuild/tools/config.py

@@ -120,6 +120,8 @@
 DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders'
 DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild")
 DEFAULT_REPOSITORY = 'FileRepository'
+EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
+DEFAULT_EXTRA_SOURCE_URLS = (EASYBUILD_SOURCES_URL,)
 # Filter these CUDA libraries by default from the RPATH sanity check.
 # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build
 # against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should
@@ -393,6 +395,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     'defaultopt': [
         'default_opt_level',
     ],
+    DEFAULT_EXTRA_SOURCE_URLS: [
+        'extra_source_urls',
+    ],
     DEFAULT_ALLOW_LOADED_MODULES: [
         'allow_loaded_modules',
     ],

easybuild/tools/options.py

@@ -70,9 +70,9 @@
 from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
 from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
 from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
-from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT
+from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
 from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
-from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS
+from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS
 from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
 from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS
@@ -408,6 +408,8 @@ def override_options(self):
                              None, 'store_true', False),
             'extra-modules': ("List of extra modules to load after setting up the build environment",
                               'strlist', 'extend', None),
+            "extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig",
+                                  "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[|URL]'}),
             'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, "
                       "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False),
             'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "