diff --git a/common.py b/common.py index 53195ee..40f562e 100644 --- a/common.py +++ b/common.py @@ -1021,11 +1021,11 @@ TestResult = enum.Enum('TestResult', ['PASS', 'FAIL']) class Test: def __init__( - self, - test_id: str, - result : TestResult =None, - ellapsed_seconds : float =None - ): + self, + test_id: str, + result : TestResult =None, + ellapsed_seconds : float =None + ): self.test_id = test_id self.result = result self.ellapsed_seconds = ellapsed_seconds @@ -1042,16 +1042,33 @@ class TestCliFunction(LkmcCliFunction): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.tests = [] + self.add_argument( + '--fail-early', + default=True, + help='''\ +Stop running at the first failed test. +''' + ) def run_test(self, run_obj, run_args, extra_params): + test_id_string = self.test_setup(extra_params) + exit_status = run_obj(**run_args) + self.test_teardown(run_obj, exit_status, test_id_string) + + def test_setup(self, extra_params): test_id_string = '{} {} {}'.format(self.env['emulator'], self.env['arch'], extra_params) self.log_info('test_id {}'.format(test_id_string), flush=True) - exit_status = run_obj(**run_args) + return test_id_string + + def test_teardown(self, run_obj, exit_status, test_id_string): if not self.env['dry_run']: if exit_status == 0: test_result = TestResult.PASS else: test_result = TestResult.FAIL + if self.env['fail_early']: + self.log_error('Test failed') + sys.exit(1) self.log_info('test_result {}'.format(test_result.name)) ellapsed_seconds = run_obj.ellapsed_seconds else: @@ -1059,11 +1076,21 @@ class TestCliFunction(LkmcCliFunction): ellapsed_seconds = None self.log_info() self.tests.append(Test(test_id_string, test_result, ellapsed_seconds)) - if exit_status != 0: - self.log_error('test failed, program exit status: {} test id: {}'.format(exit_status, test_id_string)) - sys.exit(1) def teardown(self): self.log_info('Test result summary') + passes = [] + fails = [] for test in self.tests: - self.log_info(test) + if test.result == TestResult.PASS: + passes.append(test) + else: + fails.append(test) + if passes: + for test in passes: + self.log_info(test) + if fails: + for test in fails: + self.log_info(test) + self.log_error('A test failed') + sys.exit(1) diff --git a/test-gdb b/test-gdb index 42d40f3..0362cca 100755 --- a/test-gdb +++ b/test-gdb @@ -5,7 +5,7 @@ import os import common -class Main(common.LkmcCliFunction): +class Main(common.TestCliFunction): def __init__(self): super().__init__( defaults={ @@ -43,9 +43,7 @@ found by searching for the Python test files. for test_script_noext in test_scripts_noext: common_args = self.get_common_args() common_args['baremetal'] = test_script_noext - - test_id_string = self.test_setup(common_args, test_script_noext) - + test_id_string = self.test_setup(test_script_noext) run_args = common_args.copy() run_args['wait_gdb'] = True run_args['background'] = True @@ -55,8 +53,7 @@ found by searching for the Python test files. gdb_args['test'] = True run_gdb(**gdb_args) run_thread.join() - - self.test_teardown(run) + self.test_teardown(run, 0, test_id_string) if __name__ == '__main__': Main().cli()