diff --git a/src/sinol_make/commands/chkwer/__init__.py b/src/sinol_make/commands/chkwer/__init__.py index 55d3e1ca..7d54e4b1 100644 --- a/src/sinol_make/commands/chkwer/__init__.py +++ b/src/sinol_make/commands/chkwer/__init__.py @@ -34,6 +34,7 @@ def configure_subparser(self, subparser): ) parser.add_argument('-t', '--tests', type=str, nargs='+', help='test to run, for example in/abc{0,1}*') + parser.add_argument('--cerr', action='store_true', help='capture cerr output') parsers.add_cpus_argument(parser, 'number of cpus to use when verifying tests') parsers.add_compilation_arguments(parser) return parser @@ -59,11 +60,11 @@ def run_test(self, execution: ChkwerExecution) -> RunResult: with open(execution.in_test_path, 'r') as inf, open(output_file, 'w') as outf: process = subprocess.Popen([execution.model_exe], stdin=inf, stdout=outf) process.wait() - ok, points, comment = self.task_type.check_output(execution.in_test_path, output_file, execution.out_test_path) + ok, points, comment, stderr = self.task_type.check_output(execution.in_test_path, output_file, execution.out_test_path) - return RunResult(execution.in_test_path, ok, int(points), comment) + return RunResult(execution.in_test_path, ok, int(points), comment, stderr) - def run_and_print_table(self) -> Dict[str, TestResult]: + def run_and_print_table(self, args) -> Dict[str, TestResult]: results = {} sorted_tests = sorted(self.tests, key=lambda test: package_util.get_group(test, self.task_id)) executions: List[ChkwerExecution] = [] @@ -77,14 +78,14 @@ def run_and_print_table(self) -> Dict[str, TestResult]: if has_terminal: run_event = threading.Event() run_event.set() - thr = threading.Thread(target=printer.printer_thread, args=(run_event, chkwer_util.print_view, table_data)) + thr = threading.Thread(target=printer.printer_thread, args=(run_event, chkwer_util.print_view, table_data, args)) thr.start() keyboard_interrupt = False try: with mp.Pool(self.cpus) as pool: for i, result in enumerate(pool.imap(self.run_test, executions)): - table_data.results[result.test_path].set_results(result.points, result.ok, result.comment) + table_data.results[result.test_path].set_results(result.points, result.ok, result.comment, result.stderr) table_data.i = i except KeyboardInterrupt: keyboard_interrupt = True @@ -93,7 +94,7 @@ def run_and_print_table(self) -> Dict[str, TestResult]: run_event.clear() thr.join() - print("\n".join(chkwer_util.print_view(terminal_width, terminal_height, table_data)[0])) + print("\n".join(chkwer_util.print_view(terminal_width, terminal_height, table_data, args)[0])) if keyboard_interrupt: util.exit_with_error("Keyboard interrupt.") return results @@ -129,7 +130,7 @@ def run(self, args): "model solution", args.compile_mode) print() - results = self.run_and_print_table() + results = self.run_and_print_table(args) for result in results.values(): if not result.ok or result.points != self.contest_type.max_score_per_test(): util.exit_with_error("Model solution didn't score maximum points.") diff --git a/src/sinol_make/commands/chkwer/chkwer_util.py b/src/sinol_make/commands/chkwer/chkwer_util.py index c4708eb1..c482d8cb 100644 --- a/src/sinol_make/commands/chkwer/chkwer_util.py +++ b/src/sinol_make/commands/chkwer/chkwer_util.py @@ -5,8 +5,7 @@ from sinol_make.commands.inwer.inwer_util import sort_tests from sinol_make.structs.chkwer_structs import TableData - -def print_view(term_width, term_height, table_data: TableData): +def print_view(term_width, term_height, table_data: TableData, args): """ Prints current results of test verification. """ @@ -65,6 +64,11 @@ def print_line_separator(): print(util.color_gray("No comment")) print_line_separator() + + if args.cerr: + for test_path in tests: + result = results[test_path] + print(util.bold(f"Stderr on {result.test_name}: ") + result.stderr) print() print() diff --git a/src/sinol_make/structs/chkwer_structs.py b/src/sinol_make/structs/chkwer_structs.py index 2f339511..2daad36a 100644 --- a/src/sinol_make/structs/chkwer_structs.py +++ b/src/sinol_make/structs/chkwer_structs.py @@ -14,6 +14,7 @@ class TestResult: points: int ok: bool comment: str + stderr: str def __init__(self, test_path, task_id): self.test_path = test_path @@ -24,12 +25,14 @@ def __init__(self, test_path, task_id): self.points = 0 self.ok = False self.run = False + self.stderr = "" - def set_results(self, points, ok, output): + def set_results(self, points, ok, output, stderr): self.run = True self.points = points self.ok = ok self.comment = output + self.stderr = stderr @dataclass @@ -66,3 +69,4 @@ class RunResult: ok: bool points: int comment: str + stderr: str diff --git a/src/sinol_make/task_type/__init__.py b/src/sinol_make/task_type/__init__.py index 428be58c..3222302e 100644 --- a/src/sinol_make/task_type/__init__.py +++ b/src/sinol_make/task_type/__init__.py @@ -135,17 +135,17 @@ def _output_to_fraction(self, output_str): except TypeError: raise CheckerException(f'Invalid checker output "{output_str}"') - def _parse_checker_output(self, output: List[str]) -> Tuple[bool, Fraction, str]: + def _parse_checker_output(self, output: List[str], stderr: str) -> Tuple[bool, Fraction, str, str]: while len(output) < 3: output.append('') if output[0].strip() == "OK": points = self._output_to_fraction(output[2]) - return True, points, output[1].strip() + return True, points, output[1].strip(), stderr else: - return False, Fraction(0, 1), output[1].strip() + return False, Fraction(0, 1), output[1].strip(), stderr - def _run_checker(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]: + def _run_checker(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str, str]: proc = subprocess.Popen([self.checker_path, input_file_path, output_file_path, answer_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc.wait() @@ -153,25 +153,25 @@ def _run_checker(self, input_file_path, output_file_path, answer_file_path) -> T if proc.returncode > 2: return False, Fraction(0, 1), (f"Checker returned with code {proc.returncode}, " f"stderr: '{stderr.decode('utf-8')}'") - return self._parse_checker_output(output.decode('utf-8').split('\n')) + return self._parse_checker_output(output.decode('utf-8').split('\n'), stderr.decode('utf-8')) - def _run_diff(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]: + def _run_diff(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str, str]: same = oicompare.compare(output_file_path, answer_file_path) if same: - return True, Fraction(100, 1), "" + return True, Fraction(100, 1), "", "" else: - return False, Fraction(0, 1), "" + return False, Fraction(0, 1), "", "" - def _run_oicompare(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]: + def _run_oicompare(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str, str]: path = oicompare.get_path() proc = subprocess.Popen([path, output_file_path, answer_file_path, 'english_abbreviated'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc.wait() output, stderr = proc.communicate() if proc.returncode == 0: - return True, Fraction(100, 1), "" + return True, Fraction(100, 1), "", "" elif proc.returncode == 1: - return False, Fraction(0, 1), output.decode('utf-8').strip() + return False, Fraction(0, 1), output.decode('utf-8').strip(), stderr.decode('utf-8') else: raise CheckerException(f"!!! oicompare failed with code {proc.returncode}. This is a huge bug, please report" f" it here https://github.com/sio2project/sinol-make/issues/new/choose and provide " @@ -179,7 +179,7 @@ def _run_oicompare(self, output_file_path, answer_file_path) -> Tuple[bool, Frac f"Output: {output.decode('utf-8').strip()}\n" f"Stderr: {stderr.decode('utf-8').strip()}") - def check_output(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]: + def check_output(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str, str]: """ Runs the checker (or runs diff) and returns a tuple of three values: - bool: whether the solution is correct diff --git a/src/sinol_make/task_type/interactive.py b/src/sinol_make/task_type/interactive.py index 5e746cd1..fb8bcef3 100644 --- a/src/sinol_make/task_type/interactive.py +++ b/src/sinol_make/task_type/interactive.py @@ -116,7 +116,7 @@ def _fill_result(self, result: ExecutionResult, iresult: ExecutionResult, intera result.Error = None else: try: - ok, points, comment = self._parse_checker_output(interactor_output) + ok, points, comment, _ = self._parse_checker_output(interactor_output, '') except CheckerException as e: result.Status = Status.RE result.Error = str(e) diff --git a/src/sinol_make/task_type/normal.py b/src/sinol_make/task_type/normal.py index 0a6c1f12..1b55978e 100644 --- a/src/sinol_make/task_type/normal.py +++ b/src/sinol_make/task_type/normal.py @@ -25,7 +25,7 @@ def run(self, time_limit, hard_time_limit, memory_limit, input_file_path, output result.Status = Status.ML elif result.Status == Status.OK: try: - correct, points, comment = self.check_output(input_file_path, output_file_path, answer_file_path) + correct, points, comment, _ = self.check_output(input_file_path, output_file_path, answer_file_path) result.Points = float(points) result.Comment = comment if not correct: