diff --git a/README.adoc b/README.adoc index 71b220d..06eabc9 100644 --- a/README.adoc +++ b/README.adoc @@ -11000,7 +11000,6 @@ cat "$(./getvar test_boot_benchmark_file)" Sample results at 8fb9db39316d43a6dbd571e04dd46ae73915027f: .... - cmd ./run --arch x86_64 --eval '/poweroff.out' time 8.25 exit_status 0 diff --git a/baremetal/exit.c b/baremetal/exit.c index 7260f78..98bcda2 100644 --- a/baremetal/exit.c +++ b/baremetal/exit.c @@ -4,4 +4,3 @@ int main(void) { exit(0); } - diff --git a/common.py b/common.py index f26a86c..53195ee 100644 --- a/common.py +++ b/common.py @@ -5,6 +5,7 @@ import base64 import collections import copy import datetime +import enum import glob import imp import inspect @@ -822,9 +823,11 @@ Valid emulators: {} start_time = time.time() env['arch'] = arch env['archs'] = [arch] + env['_args_given']['archs'] = True env['all_archs'] = False env['emulator'] = emulator env['emulators'] = [emulator] + env['_args_given']['emulators'] = True env['all_emulators'] = False self.env = env.copy() self._init_env(self.env) @@ -841,6 +844,7 @@ Valid emulators: {} return ret elif not real_all_archs: raise Exception('Unsupported arch for this action: ' + arch) + self.teardown() return 0 def make_build_dirs(self): @@ -866,7 +870,8 @@ Valid emulators: {} return True return False - def seconds_to_hms(self, seconds): + @staticmethod + def seconds_to_hms(seconds): ''' Seconds to hour:minute:seconds @@ -951,25 +956,11 @@ Valid emulators: {} self.env['userland_build_ext'], ) - def test_setup(self, test_env, source): - if not self.env['verbose']: - test_env['quiet'] = True - test_id_string = '{} {} {}'.format(self.env['emulator'], self.env['arch'], source) - if self.env['verbose']: - end = '\n' - elif not self.env['dry_run']: - end = ' ' - else: - end = '' - self.log_info(test_id_string, flush=True, end=end) - return test_id_string - - def test_teardown(self, run_object): - if self.env['dry_run']: - if not self.env['verbose']: - self.log_info() - else: - self.log_info(self.seconds_to_hms(run_object.ellapsed_seconds)) + def teardown(self): + ''' + Gets run just once after looping over all archs and emulators. + ''' + pass def timed_main(self): ''' @@ -998,6 +989,7 @@ class BuildCliFunction(LkmcCliFunction): type=int, help='Number of processors to use for the build.', ) + self.test_results = [] def clean(self): build_dir = self.get_build_dir() @@ -1023,3 +1015,55 @@ class BuildCliFunction(LkmcCliFunction): return self.clean() else: return self.build() + +# from aenum import Enum # for the aenum version +TestResult = enum.Enum('TestResult', ['PASS', 'FAIL']) + +class Test: + def __init__( + self, + test_id: str, + result : TestResult =None, + ellapsed_seconds : float =None + ): + self.test_id = test_id + self.result = result + self.ellapsed_seconds = ellapsed_seconds + def __str__(self): + out = [] + if self.result is not None: + out.append(self.result.name) + if self.ellapsed_seconds is not None: + out.append(LkmcCliFunction.seconds_to_hms(self.ellapsed_seconds)) + out.append(self.test_id) + return ' '.join(out) + +class TestCliFunction(LkmcCliFunction): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.tests = [] + + def run_test(self, run_obj, run_args, extra_params): + test_id_string = '{} {} {}'.format(self.env['emulator'], self.env['arch'], extra_params) + self.log_info('test_id {}'.format(test_id_string), flush=True) + exit_status = run_obj(**run_args) + if not self.env['dry_run']: + if exit_status == 0: + test_result = TestResult.PASS + else: + test_result = TestResult.FAIL + self.log_info('test_result {}'.format(test_result.name)) + ellapsed_seconds = run_obj.ellapsed_seconds + else: + test_result = None + ellapsed_seconds = None + self.log_info() + self.tests.append(Test(test_id_string, test_result, ellapsed_seconds)) + if exit_status != 0: + self.log_error('test failed, program exit status: {} test id: {}'.format(exit_status, test_id_string)) + sys.exit(1) + + def teardown(self): + self.log_info('Test result summary') + for test in self.tests: + self.log_info(test) diff --git a/run b/run index 1990229..8d12c0c 100755 --- a/run +++ b/run @@ -32,6 +32,12 @@ https://superuser.com/questions/1373226/how-to-redirect-qemu-serial-output-to-bo '-c', '--cpus', default=1, type=int, help='Number of guest CPUs to emulate. Default: %(default)s' ) + self.add_argument( + '--ctrl-c-host', default=False, + help='''\ +Ctrl +C kills the QEMU simulator instead of being passed to the guest. +''' + ) self.add_argument( '-D', '--debug-vm', default=False, help='Run GDB on the emulator itself.' @@ -446,7 +452,11 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: if self.env['quiet']: show_stdout = False else: - serial_monitor = ['-serial', 'mon:stdio', LF] + if self.env['ctrl_c_host']: + serial = 'stdio' + else: + serial = 'mon:stdio' + serial_monitor = ['-serial', serial, LF] if self.env['kvm']: extra_emulator_args.extend(['-enable-kvm', LF]) extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(self.env['extra_serial_port']), LF]) diff --git a/test-baremetal b/test-baremetal index 698e74c..8d6869e 100755 --- a/test-baremetal +++ b/test-baremetal @@ -5,7 +5,7 @@ import sys import common -class Main(common.LkmcCliFunction): +class Main(common.TestCliFunction): def __init__(self): super().__init__( defaults={ @@ -50,15 +50,10 @@ If given, run only the given tests. Otherwise, run all tests. sources = self.env['tests'] for source in sources: run_args['baremetal'] = source - run_args['background'] = True - test_id_string = self.test_setup(run_args, source) + run_args['ctrl_c_host'] = True if os.path.splitext(os.path.basename(source))[0] == 'multicore': run_args['cpus'] = 2 - exit_status = run(**run_args) - self.test_teardown(run) - if exit_status != 0: - self.log_error('test failed, program exit status: {} test id: {}'.format(exit_status, test_id_string)) - sys.exit(1) + self.run_test(run, run_args, source) if __name__ == '__main__': Main().cli() diff --git a/test-boot b/test-boot index 5776b36..89b4b96 100755 --- a/test-boot +++ b/test-boot @@ -1,9 +1,10 @@ #!/usr/bin/env python3 import common +import shell_helpers from shell_helpers import LF -class Main(common.LkmcCliFunction): +class Main(common.TestCliFunction): def __init__(self): super().__init__( defaults={ @@ -27,19 +28,16 @@ Size of the tests to run. Scale: ) def _bench(self, **kwargs): - self.run(**kwargs) + words = [] + for line in self.run.get_cli(**kwargs): + words.extend(line) + extra_params = shell_helpers.ShellHelpers().cmd_to_string(words + [LF]) + run_args = kwargs.copy() + run_args.update(self.common_args) + self.run_test(self.run, run_args, extra_params) def timed_main(self): # TODO bring this benchmark code back to life. Likely should go inside run with an option - # - #bench() ( - # "${root_dir}/bench-cmd" "./run --arch ${1}${extra_args}" "$self.env['test_boot_benchmark_file']" - #) - # - #newline() ( - # echo >> "$self.env['test_boot_benchmark_file']" - #) - # #gem5_insts() ( # printf "instructions $(./gem5-stat --arch "$1" sim_insts)\n" >> "$self.env['test_boot_benchmark_file']" # newline @@ -55,20 +53,21 @@ Size of the tests to run. Scale: # #rm -f "${self.env['test_boot_benchmark_file']}" self.run = self.import_path_main('run') - run_args = self.get_common_args() + self.common_args = self.get_common_args() + self.common_args['ctrl_c_host'] = True if self.env['emulator'] == 'gem5': - run_args['eval'] = 'm5 exit' + self.common_args['eval'] = 'm5 exit' elif self.env['emulator'] == 'qemu': - run_args['eval'] = '/poweroff.out' + self.common_args['eval'] = '/poweroff.out' if (self.env['emulator'] == 'qemu' or (self.env['emulator'] == 'gem5' and self.env['size'] >= 2)): - self._bench(**run_args) + self._bench() if self.env['host_arch'] == self.env['arch']: # TODO: find out why it fails. if self.env['emulator'] != 'gem5': - self._bench(kvm=True, **run_args) + self._bench(kvm=True) if self.env['emulator'] == 'qemu' and self.env['size'] >= 2: - self._bench(trace='exec_tb', **run_args) + self._bench(trace='exec_tb') if self.env['emulator'] == 'gem5' and self.env['size'] >= 3: if self.env['arch'] == 'x86_64': cpu_types = ['DerivO3CPU'] @@ -83,18 +82,17 @@ Size of the tests to run. Scale: '--cpu-type', cpu_type, '--caches', '--l2cache', - '--l1d_size=1024kB', - '--l1i_size=1024kB', - '--l2_size=1024kB', - '--l3_size=1024kB', + '--l1d_size', '1024kB', + '--l1i_size', '1024kB', + '--l2_size', '1024kB', + '--l3_size', '1024kB', ], - **run_args ) if self.env['arch'] == 'aarch64': # Do a fuller testing for aarch64. for build_type in ['debug', 'fast']: - self._bench(gem5_build_type=build_type, **run_args) - self._bench(gem5_script='biglittle', **run_args) + self._bench(gem5_build_type=build_type) + self._bench(gem5_script='biglittle') if __name__ == '__main__': Main().cli_exit() diff --git a/test-user-mode b/test-user-mode index 0f01d15..e4c5b97 100755 --- a/test-user-mode +++ b/test-user-mode @@ -5,7 +5,7 @@ import sys import common -class Main(common.LkmcCliFunction): +class Main(common.TestCliFunction): def __init__(self): super().__init__( defaults={ @@ -27,6 +27,7 @@ If given, run only the given tests. Otherwise, run all tests. def timed_main(self): run = self.import_path_main('run') run_args = self.get_common_args() + run_args['ctrl_c_host'] = True if self.env['emulator'] == 'gem5': run_args['userland_build_id'] = 'static' if self.env['tests'] == []: @@ -52,13 +53,7 @@ If given, run only the given tests. Otherwise, run all tests. sources = self.env['tests'] for source in sources: run_args['userland'] = source - run_args['background'] = True - test_id_string = self.test_setup(run_args, source) - exit_status = run(**run_args) - self.test_teardown(run) - if exit_status != 0: - self.log_error('test failed, program exit status: {} test id: {}'.format(exit_status, test_id_string)) - sys.exit(1) + self.run_test(run, run_args, source) if __name__ == '__main__': Main().cli()