diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ae9b78b5ed..4fd4486a1a 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -612,6 +612,13 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, if github_repo is None: github_repo = GITHUB_EASYCONFIGS_REPO + if github_repo == GITHUB_EASYCONFIGS_REPO: + easybuild_subdir = os.path.join('easybuild', 'easyconfigs') + elif github_repo == GITHUB_EASYBLOCKS_REPO: + easybuild_subdir = os.path.join('easybuild', 'easyblocks') + else: + raise EasyBuildError("Unknown repo: %s", github_repo) + if path is None: if github_repo == GITHUB_EASYCONFIGS_REPO: extra_ec_paths = build_option('extra_ec_paths') @@ -655,6 +662,12 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, else: raise EasyBuildError("Unknown repo: %s" % github_repo) + # symlink subdirectories of 'easybuild/easy{blocks,configs}' into path that gets added to robot search path + mkdir(path, parents=True) + dirpath = os.path.join(repo_commit, easybuild_subdir) + for subdir in os.listdir(dirpath): + symlink(os.path.join(dirpath, subdir), os.path.join(path, subdir)) + # copy specified files to directory where they're expected to be found file_paths = [] for file in files: diff --git a/test/framework/options.py b/test/framework/options.py index 3deaf8cf9c..6263d8186f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1908,7 +1908,7 @@ def test_github_from_pr(self): # make sure that *only* these modules are listed, no others regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) - self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr6424') regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) @@ -1943,12 +1943,12 @@ def test_github_from_pr(self): # make sure that *only* these modules are listed, no others regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) - self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) for pr in ('12150', '12366'): pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr%s' % pr) regex = re.compile(r"Extended list of robot search paths with .*%s.*:" % pr_tmpdir, re.M) - self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + self.assertTrue(regex.search(outtxt), "Found pattern '%s' in: %s" % (regex.pattern, outtxt)) except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) @@ -2125,6 +2125,50 @@ def test_from_commit(self): print("Ignoring URLError '%s' in test_from_commit" % err) shutil.rmtree(tmpdir) + easyblock_template = '\n'.join([ + "from easybuild.framework.easyblock import EasyBlock", + "class %s(EasyBlock):", + " pass", + ]) + + # create fake custom easyblock for CMake that is required by easyconfig used in test below + easyblock_file = os.path.join(self.test_prefix, 'easyblocks', 'cmake.py') + write_file(easyblock_file, easyblock_template % 'EB_CMake') + + # also test with an easyconfig that requires additional easyconfigs to resolve dependencies, + # cfr. https://github.com/easybuilders/easybuild-framework/issues/4540; + # using commit that adds CMake-3.18.4.eb (which requires ncurses-6.2.eb), + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/13156 + test_commit = '41eee3fe2e5102f52319481ca8dde16204dab590' + args = [ + '--from-commit=%s' % test_commit, + '--dry-run', + '--tmpdir=%s' % tmpdir, + '--include-easyblocks=' + os.path.join(self.test_prefix, 'easyblocks', '*.py'), + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'ncurses/6.2'), + (tmpdir, 'CMake/3.18.4'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + path = '.*%s' % os.path.dirname(path_prefix) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) + + pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_commit_%s' % test_commit) + regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError as err: + print("Ignoring URLError '%s' in test_from_commit" % err) + shutil.rmtree(tmpdir) + # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... def test_xxx_include_easyblocks_from_commit(self):