Merge pull request #3446 from migueldiascosta/upload_test_report_easyblocks

take into account --include-easyblocks-from-pr when uploading test reports

Author: boegel

easybuild/main.py

@@ -137,10 +137,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
             parent_dir = os.path.dirname(test_report_fp)
             # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used
             if os.stat(parent_dir).st_mode & 0o200:
-                write_file(test_report_fp, test_report_txt)
+                write_file(test_report_fp, test_report_txt['full'])
             else:
                 adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False)
-                write_file(test_report_fp, test_report_txt)
+                write_file(test_report_fp, test_report_txt['full'])
                 adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
 
         if not ec_res['success'] and exit_on_failure:

easybuild/tools/config.py

@@ -184,6 +184,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'group',
         'hooks',
         'ignore_dirs',
+        'include_easyblocks_from_pr',
         'job_backend_config',
         'job_cores',
         'job_deps_type',

easybuild/tools/testing.py

@@ -46,7 +46,7 @@
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file
-from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, create_gist, post_comment_in_issue
+from easybuild.tools.github import GITHUB_EASYBLOCKS_REPO, GITHUB_EASYCONFIGS_REPO, create_gist, post_comment_in_issue
 from easybuild.tools.jenkins import aggregate_xml_in_dirs
 from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel
 from easybuild.tools.robot import resolve_dependencies
@@ -160,7 +160,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l
         "",
     ])
 
-    build_overview = []
+    build_overview = ["#### Overview of tested easyconfigs (in order)"]
     for (ec, ec_res) in ecs_with_res:
         test_log = ''
         if ec_res.get('success', False):
@@ -189,7 +189,8 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l
                 test_log = "(partial log available at %s)" % gist_url
 
         build_overview.append(" * **%s** _%s_ %s" % (test_result, os.path.basename(ec['spec']), test_log))
-    test_report.extend(["#### Overview of tested easyconfigs (in order)"] + build_overview + [""])
+    build_overview.append("")
+    test_report.extend(build_overview)
 
     time_format = "%a, %d %b %Y %H:%M:%S +0000 (UTC)"
     start_time = strftime(time_format, init_session_state['time'])
@@ -232,7 +233,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l
 
     test_report.extend(["#### Environment", "```"] + environment + ["```"])
 
-    return '\n'.join(test_report)
+    return {'full': '\n'.join(test_report), 'overview': '\n'.join(build_overview)}
 
 
 def upload_test_report_as_gist(test_report, descr=None, fn=None):
@@ -248,18 +249,18 @@ def upload_test_report_as_gist(test_report, descr=None, fn=None):
     return gist_url
 
 
-def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, success):
-    """Post test report in a gist, and submit comment in easyconfigs PR."""
+def post_pr_test_report(pr_nr, repo_type, test_report, msg, init_session_state, success):
+    """Post test report in a gist, and submit comment in easyconfigs or easyblocks PR."""
 
     github_user = build_option('github_user')
     pr_target_account = build_option('pr_target_account')
-    pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
+    pr_target_repo = build_option('pr_target_repo') or repo_type
 
     # create gist with test report
     descr = "EasyBuild test report for %s/%s PR #%s" % (pr_target_account, pr_target_repo, pr_nr)
     timestamp = strftime("%Y%M%d-UTC-%H-%M-%S", gmtime())
     fn = 'easybuild_test_report_%s_%s_pr%s_%s.md' % (pr_nr, pr_target_account, pr_target_repo, timestamp)
-    gist_url = upload_test_report_as_gist(test_report, descr=descr, fn=fn)
+    gist_url = upload_test_report_as_gist(test_report['full'], descr=descr, fn=fn)
 
     # post comment to report test result
     system_info = init_session_state['system_info']
@@ -276,18 +277,31 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state,
         'pyver': system_info['python_version'].split(' ')[0],
     }
 
-    comment_lines = [
-        "Test report by @%s" % github_user,
-        ('**FAILED**', '**SUCCESS**')[success],
+    comment_lines = ["Test report by @%s" % github_user]
+
+    easyblocks_pr_nr = build_option('include_easyblocks_from_pr')
+    if easyblocks_pr_nr:
+        if repo_type == GITHUB_EASYCONFIGS_REPO:
+            comment_lines.append("Using easyblocks from https://github.com/%s/%s/pull/%s" % (
+                pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_pr_nr))
+        elif repo_type == GITHUB_EASYBLOCKS_REPO:
+            comment_lines.append(test_report['overview'])
+        else:
+            raise EasyBuildError("Don't know how to submit test reports to repo %s.", repo_type)
+
+    if repo_type == GITHUB_EASYCONFIGS_REPO:
+        comment_lines.append(('**FAILED**', '**SUCCESS**')[success])
+
+    comment_lines.extend([
         msg,
         short_system_info,
         "See %s for a full test report." % gist_url,
-    ]
+    ])
     comment = '\n'.join(comment_lines)
 
     post_comment_in_issue(pr_nr, comment, account=pr_target_account, repo=pr_target_repo, github_user=github_user)
 
-    msg = "Test report uploaded to %s and mentioned in a comment in easyconfigs PR#%s" % (gist_url, pr_nr)
+    msg = "Test report uploaded to %s and mentioned in a comment in %s PR#%s" % (gist_url, pr_target_repo, pr_nr)
     return msg
 
 
@@ -302,25 +316,29 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state
     """
     dump_path = build_option('dump_test_report')
     pr_nr = build_option('from_pr')
+    eb_pr_nr = build_option('include_easyblocks_from_pr')
     upload = build_option('upload_test_report')
 
     if upload:
-        msg = msg + " (%d easyconfigs in this PR)" % orig_cnt
+        msg = msg + " (%d easyconfigs in total)" % orig_cnt
         test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True)
         if pr_nr:
             # upload test report to gist and issue a comment in the PR to notify
-            txt = post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, success)
+            txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, success)
+        elif eb_pr_nr:
+            # upload test report to gist and issue a comment in the easyblocks PR to notify
+            txt = post_pr_test_report(eb_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, init_session_state, success)
         else:
             # only upload test report as a gist
-            gist_url = upload_test_report_as_gist(test_report)
+            gist_url = upload_test_report_as_gist(test_report['full'])
             txt = "Test report uploaded to %s" % gist_url
     else:
         test_report = create_test_report(msg, ecs_with_res, init_session_state)
         txt = None
-    _log.debug("Test report: %s" % test_report)
+    _log.debug("Test report: %s" % test_report['full'])
 
     if dump_path is not None:
-        write_file(dump_path, test_report)
+        write_file(dump_path, test_report['full'])
         _log.info("Test report dumped to %s" % dump_path)
 
     return txt

test/framework/github.py

@@ -43,7 +43,7 @@
 from easybuild.tools.configobj import ConfigObj
 from easybuild.tools.filetools import read_file, write_file
 from easybuild.tools.github import VALID_CLOSE_PR_REASONS
-from easybuild.tools.testing import post_easyconfigs_pr_test_report, session_state
+from easybuild.tools.testing import post_pr_test_report, session_state
 from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters
 import easybuild.tools.github as gh
 
@@ -828,25 +828,24 @@ def test_push_branch_to_github(self):
         regex = re.compile(pattern)
         self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout))
 
-    def test_post_easyconfigs_pr_test_report(self):
-        """Test for post_easyconfigs_pr_test_report function."""
+    def test_pr_test_report(self):
+        """Test for post_pr_test_report function."""
         if self.skip_github_tests:
-            print("Skipping test_post_easyconfigs_pr_test_report, no GitHub token available?")
+            print("Skipping test_post_pr_test_report, no GitHub token available?")
             return
 
         init_config(build_options={
             'dry_run': True,
             'github_user': GITHUB_TEST_ACCOUNT,
         })
 
-        test_report = os.path.join(self.test_prefix, 'test_report.txt')
-        write_file(test_report, "This is a test report!")
+        test_report = {'full': "This is a test report!"}
 
         init_session_state = session_state()
 
         self.mock_stderr(True)
         self.mock_stdout(True)
-        post_easyconfigs_pr_test_report('1234', test_report, "OK!", init_session_state, True)
+        post_pr_test_report('1234', gh.GITHUB_EASYCONFIGS_REPO, test_report, "OK!", init_session_state, True)
         stderr, stdout = self.get_stderr(), self.get_stdout()
         self.mock_stderr(False)
         self.mock_stdout(False)
@@ -861,6 +860,23 @@ def test_post_easyconfigs_pr_test_report(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
 
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        post_pr_test_report('1234', gh.GITHUB_EASYBLOCKS_REPO, test_report, "OK!", init_session_state, True)
+        stderr, stdout = self.get_stderr(), self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+
+        self.assertEqual(stderr, '')
+
+        patterns = [
+            r"^\[DRY RUN\] Adding comment to easybuild-easyblocks issue #1234: 'Test report by @easybuild_test",
+            r"^See https://gist.github.com/DRY_RUN for a full test report.'",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
 
 def suite():
     """ returns all the testcases in this module """