diff --git a/README.adoc b/README.adoc index 5486d0a..3969d0d 100644 --- a/README.adoc +++ b/README.adoc @@ -13774,6 +13774,8 @@ Failure is detected by looking for the <> Most userland programs that don't rely on kernel modules can also be tested in user mode simulation as explained at: <>. +TODO: we really need a mechanism to automatically generate the test list much like user-mode-tests, currently there are many tests missing, and we have to add everything manually which is very annoying. + ===== Test GDB We have some link:https://github.com/pexpect/pexpect[pexpect] automated tests for the baremetal programs! diff --git a/build-userland b/build-userland index d7545ea..97f2dc7 100755 --- a/build-userland +++ b/build-userland @@ -125,36 +125,6 @@ Default: build all examples that have their package dependencies met, e.g.: ) return ret - def _walk_targets(self, exts): - ''' - Resolve userland input targets, and walk them if directories. - - Ignore the input extension of targets, and select only files - with the given extensions exts. - - An empty extension indicates that directories will also be chosen. - ''' - if self.env['targets']: - targets = self.env['targets'] - else: - targets = [self.env['userland_source_dir']] - for target in targets: - resolved_targets = self.resolve_source_tree( - target, - exts + [''], - self.env['userland_source_dir'] - ) - for resolved_target in resolved_targets: - for path, dirnames, filenames in self.sh.walk(resolved_target): - dirnames.sort() - filenames = [ - filename for filename in filenames - if os.path.splitext(filename)[1] in exts - ] - filenames.sort() - for filename in filenames: - yield path, dirnames, filenames - def build(self): build_dir = self.get_build_dir() has_packages = set(self.env['has_package']) @@ -231,9 +201,9 @@ Default: build all examples that have their package dependencies met, e.g.: self._build_one, nthreads=self.env['nproc'], ) as thread_pool: - class ExitLoop(Exception): pass try: - for path, in_dirnames, in_filenames in self._walk_targets( + for path, in_dirnames, in_filenames in self.walk_source_targets( + self.env['targets'], self.env['userland_in_exts'] ): path_abs = os.path.abspath(path) @@ -347,8 +317,8 @@ Default: build all examples that have their package dependencies met, e.g.: 'ccflags_after': ccflags_after, }) if error is not None: - raise ExitLoop() - except ExitLoop: + raise common.ExitLoop() + except common.ExitLoop: pass error = thread_pool.get_error() if error is not None: @@ -364,8 +334,10 @@ Default: build all examples that have their package dependencies met, e.g.: def clean(self): if self.env['in_tree']: - for path, dirnames, filenames in self._walk_targets( - self.env['userland_out_exts'] + for path, dirnames, filenames in self.walk_source_targets( + self.env['targets'], + self.env['userland_out_exts'], + empty_ok=True ): for filename in filenames: self.sh.rmrf(os.path.join(path, filename)) diff --git a/common.py b/common.py index 3c561ba..b33785a 100644 --- a/common.py +++ b/common.py @@ -122,6 +122,9 @@ for key in consts['emulator_short_to_long_dict']: consts['emulator_choices'].add(consts['emulator_short_to_long_dict'][key]) consts['host_arch'] = platform.processor() +class ExitLoop(Exception): + pass + class LkmcCliFunction(cli_function.CliFunction): ''' Common functionality shared across our CLI functions: @@ -1010,12 +1013,11 @@ lunch aosp_{}-eng else: real_emulators = env['emulators'] return_value = 0 - class GetOutOfLoop(Exception): pass try: ret = self.setup() if ret is not None and ret != 0: return_value = ret - raise GetOutOfLoop() + raise ExitLoop() for emulator in real_emulators: for arch in real_archs: if arch in env['arch_short_to_long_dict']: @@ -1045,11 +1047,10 @@ lunch aosp_{}-eng if ret is not None and ret != 0: return_value = ret if self.env['quit_on_fail']: - raise GetOutOfLoop() + raise ExitLoop() elif not real_all_archs: raise Exception('Unsupported arch for this action: ' + arch) - - except GetOutOfLoop: + except ExitLoop: pass ret = self.teardown() if ret is not None and ret != 0: @@ -1122,7 +1123,7 @@ lunch aosp_{}-eng ] ) - def resolve_source_tree(self, in_path, exts, source_tree_root): + def resolve_source_tree(self, in_path, exts, source_tree_root, empty_ok=False): ''' Convert a convenient shorthand user input string to paths of existing files in the source tree. @@ -1178,7 +1179,7 @@ lunch aosp_{}-eng try_path = name + try_ext if os.path.exists(try_path): result.append(try_path) - if not result: + if not result and not empty_ok: raise Exception('No file not found for input: ' + in_path) return result @@ -1237,6 +1238,34 @@ lunch aosp_{}-eng ''' pass + def walk_source_targets(self, targets, exts, empty_ok=False): + ''' + Resolve userland or baremetal source tree targets, and walk them. + + Ignore the input extension of targets, and select only files + with the given extensions exts. + ''' + if targets: + targets = targets + else: + targets = [self.env['userland_source_dir']] + for target in targets: + resolved_targets = self.resolve_source_tree( + target, + exts + [''], + self.env['userland_source_dir'], + empty_ok=empty_ok, + ) + for resolved_target in resolved_targets: + for path, dirnames, filenames in self.sh.walk(resolved_target): + dirnames.sort() + filenames = [ + filename for filename in filenames + if os.path.splitext(filename)[1] in exts + ] + filenames.sort() + yield path, dirnames, filenames + class BuildCliFunction(LkmcCliFunction): ''' A CLI function with common facilities to build stuff, e.g.: @@ -1364,7 +1393,13 @@ class TestCliFunction(LkmcCliFunction): super().__init__(*args, **kwargs) self.tests = [] - def run_test(self, run_obj, run_args=None, test_id=None): + def run_test( + self, + run_obj, + run_args=None, + test_id=None, + expected_exit_status=None + ): ''' This is a setup / run / teardown setup for simple tests that just do a single run. @@ -1380,7 +1415,12 @@ class TestCliFunction(LkmcCliFunction): run_args = {} test_id_string = self.test_setup(test_id) exit_status = run_obj(**run_args) - self.test_teardown(run_obj, exit_status, test_id_string) + return self.test_teardown( + run_obj, + exit_status, + test_id_string, + expected_exit_status=expected_exit_status + ) def test_setup(self, test_id): test_id_string = '{} {}'.format(self.env['emulator'], self.env['arch']) @@ -1389,15 +1429,20 @@ class TestCliFunction(LkmcCliFunction): self.log_info('test_id {}'.format(test_id_string), flush=True) return test_id_string - def test_teardown(self, run_obj, exit_status, test_id_string): + def test_teardown( + self, + run_obj, + exit_status, + test_id_string, + expected_exit_status=None + ): + if expected_exit_status is None: + expected_exit_status = 0 if not self.env['dry_run']: - if exit_status == 0: + if exit_status == expected_exit_status: test_result = TestResult.PASS else: test_result = TestResult.FAIL - if self.env['quit_on_fail']: - self.log_error('Test failed') - sys.exit(1) self.log_info('test_result {}'.format(test_result.name)) ellapsed_seconds = run_obj.ellapsed_seconds else: @@ -1405,6 +1450,7 @@ class TestCliFunction(LkmcCliFunction): ellapsed_seconds = None self.log_info() self.tests.append(Test(test_id_string, test_result, ellapsed_seconds)) + return test_result def teardown(self): ''' diff --git a/example_properties.py b/example_properties.py index 4a0d85b..6e78e67 100644 --- a/example_properties.py +++ b/example_properties.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -class ExampleProperties: +class ExecutableProperties: ''' Encodes properties of userland and baremetal examples. For directories, it applies to all files under the directory. Used to determine how to build and test the examples. ''' def __init__( + self, exit_status=0, interactive=False, more_than_1s=False, @@ -21,5 +22,12 @@ class ExampleProperties: not self.more_than_1s executable_properties = { - 'userland/arch/x86_64/c/ring0.c': ExecutableProperties(exits_nonzero=True), + 'c/assert_fail.c': ExecutableProperties(exit_status=0), + 'c/false.c': ExecutableProperties(exit_status=0), } + +def get(test_path): + if test_path in executable_properties: + return executable_properties[test_path] + else: + return ExecutableProperties() diff --git a/test-user-mode b/test-user-mode index a334a5a..d021e7b 100755 --- a/test-user-mode +++ b/test-user-mode @@ -4,6 +4,7 @@ import os import sys import common +import example_properties class Main(common.TestCliFunction): def __init__(self): @@ -27,37 +28,36 @@ If given, run only the given tests. Otherwise, run all tests. if self.env['emulator'] == 'gem5': run_args['userland_build_id'] = 'static' if self.env['tests'] == []: - tests = [ - 'add.c', - 'hello.c', - 'hello_cpp.cpp', - 'print_argv.c', + test_paths = [ + 'c/add.c', + 'c/false.c', + 'c/hello.c', + 'c/print_argv.c', + 'cpp/hello.cpp', ] - if self.env['arch'] == 'x86_64': - arch_sources = [ - 'asm_hello' - ] - elif self.env['arch'] == 'aarch64': - arch_sources = [ - 'asm_hello' - ] - else: - arch_sources = [] - arch_sources[:] = [ - os.path.join('arch', self.env['arch'], arch_source) - for arch_source - in arch_sources - ] - tests.extend(arch_sources) else: - tests = self.env['tests'] - for test_dir_or_file in tests: - for test in self.sh.walk(self.resolve_userland_source(test_dir_or_file)): - - consts['userland_in_exts'] = [ - - run_args['userland'] = test - self.run_test(run, run_args) + test_paths = self.env['tests'] + had_failure = False + for test_path in test_paths: + test = example_properties.get(test_path) + if test.should_be_tested(): + # for test in self.sh.walk(self.resolve_userland_source(test_dir_or_file)): + run_args['userland'] = test_path + test_result = self.run_test( + run, + run_args, + test_id=test_path, + expected_exit_status=test.exit_status + ) + if test_result != common.TestResult.PASS: + if self.env['quit_on_fail']: + return 1 + else: + had_failure = True + if had_failure: + return 1 + else: + return 0 if __name__ == '__main__': Main().cli()