diff --git a/build-baremetal b/build-baremetal index f8548fa..6388bcd 100755 --- a/build-baremetal +++ b/build-baremetal @@ -135,28 +135,11 @@ Build the baremetal examples with crosstool-NG. os.path.isfile(in_path) and in_ext in (self.env['c_ext'], self.env['asm_ext']) ): - main_obj = os.path.join( - out_dir, - '{}{}'.format( - in_name, - self.env['obj_ext'] - ) - ) - src = os.path.join(self.env['baremetal_source_dir'], in_path) - if self.need_rebuild([src, self.env['common_h']], main_obj): - self.sh.run_cmd( - [self.env['gcc'], LF] + - cflags + - [ - '-c', LF, - '-o', main_obj, LF, - src, LF, - ] + - cflags_after - ) - objs = common_objs_bootloader + [main_obj] out = os.path.join(out_dir, in_name + self.env['baremetal_build_ext']) - if self.need_rebuild(objs + [self.env['baremetal_link_script']], out): + if self.need_rebuild( + common_objs_bootloader + [self.env['baremetal_link_script'] + self.env['common_h']], + out + ): self.sh.run_cmd( [self.env['gcc'], LF] + cflags + @@ -165,11 +148,13 @@ Build the baremetal examples with crosstool-NG. '-o', out, LF, '-T', self.env['baremetal_link_script'], LF, ] + - self.sh.add_newlines(objs) + + [ + os.path.join(self.env['baremetal_source_dir'], in_path), LF, + ] + + self.sh.add_newlines(common_objs_bootloader) + cflags_after ) - def get_build_dir(self): return self.env['baremetal_build_dir'] diff --git a/build-userland b/build-userland index e3dec25..6251c61 100755 --- a/build-userland +++ b/build-userland @@ -8,7 +8,6 @@ import threading from shell_helpers import LF import common from thread_pool import ThreadPool -import path_properties class Main(common.BuildCliFunction): def __init__(self, *args, **kwargs): @@ -42,133 +41,6 @@ Default: build all examples that have their package dependencies met, e.g.: self._add_argument('--force-rebuild') self._add_argument('--optimization-level') - def _build_one( - self, - in_path, - out_path, - build_exts=None, - cc_flags=None, - cc_flags_after=None, - extra_objs_userland_asm=None, - extra_objs_lkmc_common=None, - extra_deps=None, - link=True, - ): - if cc_flags is None: - cc_flags = [] - else: - cc_flags = cc_flags.copy() - if cc_flags_after is None: - cc_flags_after = [] - else: - cc_flags_after = cc_flags_after.copy() - if extra_deps is None: - extra_deps = [] - ret = 0 - in_dir, in_basename = os.path.split(in_path) - in_dir_abs = os.path.abspath(in_dir) - dirpath_relative_root = in_dir_abs[len(self.env['userland_source_dir']) + 1:] - dirpath_relative_root_components = dirpath_relative_root.split(os.sep) - dirpath_relative_root_components_len = len(dirpath_relative_root_components) - my_path_properties = path_properties.get(os.path.join( - self.env['userland_subdir'], - dirpath_relative_root, - in_basename - )) - if my_path_properties.should_be_built(self.env, link): - extra_objs= [] - if my_path_properties['extra_objs_lkmc_common']: - extra_objs.extend(extra_objs_lkmc_common) - if my_path_properties['extra_objs_userland_asm']: - extra_objs.extend(extra_objs_userland_asm) - if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path): - cc_flags.extend(my_path_properties['cc_flags']) - cc_flags_after.extend(my_path_properties['cc_flags_after']) - if my_path_properties['cc_pedantic']: - cc_flags.extend(['-pedantic', LF]) - if not link: - cc_flags.extend(['-c', LF]) - in_ext = os.path.splitext(in_path)[1] - if in_ext in (self.env['c_ext'], self.env['asm_ext']): - cc = self.env['gcc'] - std = my_path_properties['c_std'] - elif in_ext == self.env['cxx_ext']: - cc = self.env['gxx'] - std = my_path_properties['cxx_std'] - if dirpath_relative_root_components_len > 0: - if dirpath_relative_root_components[0] == 'arch': - cc_flags.extend([ - '-I', os.path.join(self.env['userland_source_arch_arch_dir']), LF, - '-I', os.path.join(self.env['userland_source_arch_dir']), LF, - ]) - elif dirpath_relative_root_components[0] == 'libs': - if dirpath_relative_root_components_len > 1: - if self.env['gcc_which'] == 'host': - eigen_root = '/' - else: - eigen_root = self.env['buildroot_staging_dir'] - packages = { - 'eigen': { - # TODO: was failing with: - # fatal error: Eigen/Dense: No such file or directory as of - # 975ce0723ee3fa1fea1766e6683e2f3acb8558d6 - # http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html - 'cc_flags': [ - '-I', - os.path.join( - eigen_root, - 'usr', - 'include', - 'eigen3' - ), - LF - ], - # Header only. - 'cc_flags_after': [], - }, - } - package_key = dirpath_relative_root_components[1] - if package_key in packages: - package = packages[package_key] - else: - package = {} - if 'cc_flags' in package: - cc_flags.extend(package['cc_flags']) - else: - pkg_config_output = subprocess.check_output([ - self.env['pkg_config'], - '--cflags', - package_key - ]).decode() - cc_flags.extend(self.sh.shlex_split(pkg_config_output)) - if 'cc_flags_after' in package: - cc_flags.extend(package['cc_flags_after']) - else: - pkg_config_output = subprocess.check_output([ - self.env['pkg_config'], - '--libs', - package_key - ]).decode() - cc_flags_after.extend(self.sh.shlex_split(pkg_config_output)) - os.makedirs(os.path.dirname(out_path), exist_ok=True) - ret = self.sh.run_cmd( - ( - [ - cc, LF, - ] + - cc_flags + - [ - '-std={}'.format(std), LF, - '-o', out_path, LF, - in_path, LF, - ] + - self.sh.add_newlines(extra_objs) + - cc_flags_after - ), - extra_paths=[self.env['ccache_dir']], - ) - return ret - def build(self): build_dir = self.get_build_dir() cc_flags = [ diff --git a/common.py b/common.py index a06a448..d654632 100644 --- a/common.py +++ b/common.py @@ -27,9 +27,10 @@ import time import urllib import urllib.request -import cli_function -import shell_helpers from shell_helpers import LF +import cli_function +import path_properties +import shell_helpers common = sys.modules[__name__] @@ -1352,6 +1353,138 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-debug-build **self._build_arguments[argument_name] ) + + def _build_one( + self, + in_path, + out_path, + build_exts=None, + cc_flags=None, + cc_flags_after=None, + extra_objs_userland_asm=None, + extra_objs_lkmc_common=None, + extra_deps=None, + link=True, + ): + ''' + Build one userland or baremetal executable. + ''' + if cc_flags is None: + cc_flags = [] + else: + cc_flags = cc_flags.copy() + if cc_flags_after is None: + cc_flags_after = [] + else: + cc_flags_after = cc_flags_after.copy() + if extra_deps is None: + extra_deps = [] + ret = 0 + in_dir, in_basename = os.path.split(in_path) + in_dir_abs = os.path.abspath(in_dir) + dirpath_relative_root = in_dir_abs[len(self.env['root_dir']) + 1:] + dirpath_relative_root_components = dirpath_relative_root.split(os.sep) + dirpath_relative_root_components_len = len(dirpath_relative_root_components) + my_path_properties = path_properties.get(os.path.join( + dirpath_relative_root, + in_basename + )) + if my_path_properties.should_be_built(self.env, link): + extra_objs= [] + if my_path_properties['extra_objs_lkmc_common']: + extra_objs.extend(extra_objs_lkmc_common) + if my_path_properties['extra_objs_userland_asm']: + extra_objs.extend(extra_objs_userland_asm) + if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path): + cc_flags.extend(my_path_properties['cc_flags']) + cc_flags_after.extend(my_path_properties['cc_flags_after']) + if my_path_properties['cc_pedantic']: + cc_flags.extend(['-pedantic', LF]) + if not link: + cc_flags.extend(['-c', LF]) + in_ext = os.path.splitext(in_path)[1] + if in_ext in (self.env['c_ext'], self.env['asm_ext']): + cc = self.env['gcc'] + std = my_path_properties['c_std'] + elif in_ext == self.env['cxx_ext']: + cc = self.env['gxx'] + std = my_path_properties['cxx_std'] + if dirpath_relative_root_components_len > 0: + if dirpath_relative_root_components[0] == 'userland': + if dirpath_relative_root_components_len > 1: + if dirpath_relative_root_components[1] == 'arch': + cc_flags.extend([ + '-I', os.path.join(self.env['userland_source_arch_arch_dir']), LF, + '-I', os.path.join(self.env['userland_source_arch_dir']), LF, + ]) + elif dirpath_relative_root_components[1] == 'libs': + if dirpath_relative_root_components_len > 1: + if self.env['gcc_which'] == 'host': + eigen_root = '/' + else: + eigen_root = self.env['buildroot_staging_dir'] + packages = { + 'eigen': { + # TODO: was failing with: + # fatal error: Eigen/Dense: No such file or directory as of + # 975ce0723ee3fa1fea1766e6683e2f3acb8558d6 + # http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html + 'cc_flags': [ + '-I', + os.path.join( + eigen_root, + 'usr', + 'include', + 'eigen3' + ), + LF + ], + # Header only. + 'cc_flags_after': [], + }, + } + package_key = dirpath_relative_root_components[1] + if package_key in packages: + package = packages[package_key] + else: + package = {} + if 'cc_flags' in package: + cc_flags.extend(package['cc_flags']) + else: + pkg_config_output = subprocess.check_output([ + self.env['pkg_config'], + '--cflags', + package_key + ]).decode() + cc_flags.extend(self.sh.shlex_split(pkg_config_output)) + if 'cc_flags_after' in package: + cc_flags.extend(package['cc_flags_after']) + else: + pkg_config_output = subprocess.check_output([ + self.env['pkg_config'], + '--libs', + package_key + ]).decode() + cc_flags_after.extend(self.sh.shlex_split(pkg_config_output)) + os.makedirs(os.path.dirname(out_path), exist_ok=True) + ret = self.sh.run_cmd( + ( + [ + cc, LF, + ] + + cc_flags + + [ + '-std={}'.format(std), LF, + '-o', out_path, LF, + in_path, LF, + ] + + self.sh.add_newlines(extra_objs) + + cc_flags_after + ), + extra_paths=[self.env['ccache_dir']], + ) + return ret + def clean(self): build_dir = self.get_build_dir() if build_dir is not None: diff --git a/path_properties.py b/path_properties.py index e35007b..3584304 100644 --- a/path_properties.py +++ b/path_properties.py @@ -9,6 +9,7 @@ class PathProperties: default_cxx_std = 'c++17' default_properties = { 'allowed_archs': None, + 'allowed_emulators': None, 'c_std': default_c_std, 'cc_flags': [ '-Wall', LF, @@ -50,6 +51,14 @@ class PathProperties: # We were lazy to properly classify why we are skipping these tests. # TODO get it done. 'skip_run_unclassified': False, + # Aruments added automatically to run when running tests, + # but not on manual running. + 'test_run_args': { + 'ctrl_c_host': True, + 'show_stdout': False, + 'show_time': False, + 'background': True, + }, 'uses_dynamic_library': False, } @@ -88,6 +97,10 @@ class PathProperties: self['allowed_archs'] is None or env['arch'] in self['allowed_archs'] ) and \ + ( + self['allowed_emulators'] is None or + env['emulator'] in self['allowed_emulators'] + ) and \ not ( link and self['no_executable'] @@ -109,7 +122,14 @@ class PathProperties: def update(self, other): other_tmp_properties = other.properties.copy() if 'cc_flags' in self.properties and 'cc_flags' in other_tmp_properties: - other_tmp_properties['cc_flags'] = self.properties['cc_flags'] + other_tmp_properties['cc_flags'] + other_tmp_properties['cc_flags'] = \ + self.properties['cc_flags'] + \ + other_tmp_properties['cc_flags'] + if 'test_run_args' in self.properties and 'test_run_args' in other_tmp_properties: + other_tmp_properties['test_run_args'] = { + **self.properties['test_run_args'], + **other_tmp_properties['test_run_args'] + } return self.properties.update(other_tmp_properties) class PrefixTree: @@ -178,13 +198,31 @@ path_properties_tuples = ( 'arm': ( {'allowed_archs': {'arm'}}, { - 'no_bootloader': {'extra_objs_baremetal_bootloader': False}, + 'gem5_assert.S': {'allowed_emulators': {'gem5'}}, + 'multicore.S': {'test_run_args': {'cpus': 2}}, + 'no_bootloader': ( + {'extra_objs_baremetal_bootloader': False}, + { + 'gem5_exit.S': {'allowed_emulators': {'gem5'}}, + 'semihost_exit.S': {'allowed_emulators': {'qemu'}}, + } + ), + 'semihost_exit.S': {'allowed_emulators': {'qemu'}}, }, + ), 'aarch64': ( {'allowed_archs': {'aarch64'}}, { - 'no_bootloader': {'extra_objs_baremetal_bootloader': False}, + 'multicore.S': {'test_run_args': {'cpus': 2}}, + 'no_bootloader': ( + {'extra_objs_baremetal_bootloader': False}, + { + 'gem5_exit.S': {'allowed_emulators': {'gem5'}}, + 'semihost_exit.S': {'allowed_emulators': {'qemu'}}, + } + ), + 'semihost_exit.S': {'allowed_emulators': {'qemu'}}, }, ) } diff --git a/test-baremetal b/test-baremetal index 17e3e9f..5fc55b9 100755 --- a/test-baremetal +++ b/test-baremetal @@ -18,40 +18,52 @@ If given, run only the given tests. Otherwise, run all tests. ''' ) + def setup_one(self): + self.env['tests'] = self.resolve_targets( + self.env['baremetal_source_dir'], + self.env['tests'] + ) + def timed_main(self): - run = self.import_path_main('run') run_args = self.get_common_args() - if self.env['emulator'] == 'gem5': - run_args['userland_build_id'] = 'static' - if self.env['tests'] == []: - baremetal_source_exts = (self.env['c_ext'], self.env['asm_ext']) - paths = [] - for f in sorted(os.listdir(self.env['baremetal_source_dir'])): - path = os.path.join(self.env['baremetal_source_dir'], f) - if os.path.isfile(path) and os.path.splitext(path)[1] in baremetal_source_exts: - paths.append(path) - for root, dirnames, filenames in os.walk(self.env['baremetal_source_arch_dir'], topdown=True): - dirnames[:] = [d for d in dirnames if d != 'interactive'] - dirnames.sort() - for filename in filenames: - path = os.path.join(root, filename) - if os.path.splitext(path)[1] in baremetal_source_exts: - paths.append(path) - sources = [] - for path in paths: - if not ( - self.env['emulator'] == 'gem5' and os.path.basename(path).startswith('semihost_') or - self.env['emulator'] == 'qemu' and os.path.basename(path).startswith('gem5_') - ): - sources.append(os.path.relpath(path, self.env['root_dir'])) + with ThreadPool( + self.run_test, + nthreads=self.env['nproc'], + thread_id_arg='thread_id', + ) as thread_pool: + try: + for test in self.env['tests']: + for path, in_dirnames, in_filenames in self.sh.walk(test): + path_abs = os.path.abspath(path) + dirpath_relative_root = path_abs[rootdir_abs_len + 1:] + for in_filename in in_filenames: + if os.path.splitext(in_filename)[1] in (self.env['c_ext'], self.env['asm_ext']): + path_relative_root = os.path.join(dirpath_relative_root, in_filename) + my_path_properties = path_properties.get(path_relative_root) + if my_path_properties.should_be_tested(self.env): + cur_run_args = run_args.copy() + cur_run_args.update({ + 'baremetal': os.path.relpath(os.path.join(path_abs, in_filename), os.getcwd()), + }) + cur_run_args.update(my_path_properties['test_run_args']) + error = thread_pool.submit({ + 'expected_exit_status': my_path_properties['exit_status'], + 'run_args': cur_run_args, + 'run_obj': self.import_path_main('run'), + 'test_id': path_relative_root, + }) + if error is not None: + if self.env['quit_on_fail']: + raise common.ExitLoop() + + except common.ExitLoop: + pass + error = thread_pool.get_error() + if error is not None: + print(error) + return 1 else: - sources = self.env['tests'] - for source in sources: - run_args['baremetal'] = source - run_args['ctrl_c_host'] = True - if os.path.splitext(os.path.basename(source))[0] == 'multicore': - run_args['cpus'] = 2 - self.run_test(run, run_args, source) + return 0 if __name__ == '__main__': Main().cli() diff --git a/test-user-mode b/test-user-mode index ab34a57..c2bb19c 100755 --- a/test-user-mode +++ b/test-user-mode @@ -35,9 +35,6 @@ If given, run only the given tests. Otherwise, run all tests. def timed_main(self): run_args = self.get_common_args() - run_args['ctrl_c_host'] = True - run_args['show_stdout'] = False - run_args['show_time'] = False if self.env['emulator'] == 'gem5': run_args['userland_build_id'] = 'static' had_failure = False @@ -59,9 +56,9 @@ If given, run only the given tests. Otherwise, run all tests. if my_path_properties.should_be_tested(self.env): cur_run_args = run_args.copy() cur_run_args.update({ - 'background': True, 'userland': os.path.relpath(os.path.join(path_abs, in_filename), os.getcwd()), }) + cur_run_args.update(my_path_properties['test_run_args']) error = thread_pool.submit({ 'expected_exit_status': my_path_properties['exit_status'], 'run_args': cur_run_args,