From 42ce64409b6743b8e3ecd41fd23a7d807b69f6ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Tue, 22 Jan 2019 00:00:00 +0000 Subject: [PATCH] polish testing a bit further --- README.adoc | 14 +++++++++++-- build | 55 ++++++++++++++++++++++++++++++++++++++++++------- build-buildroot | 3 --- build-test | 4 +++- build-test-boot | 3 ++- common.py | 16 +++++++++----- test | 55 +++++++++++++++++++++++++++++++++++-------------- test-boot | 16 +++++++------- 8 files changed, 123 insertions(+), 43 deletions(-) diff --git a/README.adoc b/README.adoc index 890b9bc..ad8d010 100644 --- a/README.adoc +++ b/README.adoc @@ -3172,11 +3172,17 @@ Result on <> at bad30f513c46c1b0995d3a10c0d9bc2a33dc4fa0: Automatically run non-interactive userland tests that can be run in user mode simulation: .... -./build-userland --all-archs --all-emulators -./build-userland --all-archs --all-emulators --static --userland-build-id static +./build --all-archs test-user-mode ./test-user-mode --all-archs --all-emulators .... +Or just for QEMU: + +.... +./build --all-archs test-user-mode-qemu +./test-user-mode --all-archs --emulator qemu +.... + Source: link:test-user-mode[] This testing excludes notably kernel module tests which depend on a full running kernel. @@ -6695,6 +6701,8 @@ Bibliography: ==== Count boot instructions +TODO: didn't port during refactor after 3b0a343647bed577586989fb702b760bd280844a. Reimplementing should not be hard. + * https://www.quora.com/How-many-instructions-does-a-typical-Linux-kernel-boot-take * https://github.com/cirosantilli/chat/issues/31 * https://rwmj.wordpress.com/2016/03/17/tracing-qemu-guest-execution/ @@ -10975,6 +10983,8 @@ make CROSS_COMPILE_DIR=/usr/bin == Benchmark this repo +TODO: didn't fully port during refactor after 3b0a343647bed577586989fb702b760bd280844a. Reimplementing should not be hard. + In this section document how benchmark builds and runs of this repo, and how to investigate what the bottleneck is. Ideally, we should setup an automated build server that benchmarks those things continuously for us, but our <> attempt failed. diff --git a/build b/build index 268a831..962b196 100755 --- a/build +++ b/build @@ -73,6 +73,36 @@ This is equivalent to: .... ./%(prog)s --arch x86_64 qemu-buildroot .... + +Another important target is `all`: + +.... +./%(prog)s all +.... + +This does not trully build ALL configurations: that would be impractical. +But more precisely: build the reference configuration of each major component. + +So e.g.: one config of Linux kenrel, Buildroot, gem5 and qemu. +Don't do for example all possible gem5 configs: debug, opt and fast, +as that would be huge. This ensures that every piece of software +builds in at least one config. + +TODO looping over emulators is not currently supported by this script, e.g.: + +.... +./%(prog)s --arch x86_64 --arch aarch64 all +.... + +Instead, for the targets that are emulator dependent, you must select the +taret version for the desired emulatore, e.g.: + +.... +./build --arch aarch64 baremetal-qemu baremetal-gem5 +.... + +The reason is that some targets depend on emulator, while others don't, +so looping over all of them would waste time. ''', ) buildroot_component = _Component( @@ -131,8 +161,9 @@ This is equivalent to: self.name_to_component_map = { 'all': _Component(dependencies=[ - 'all-linux', + 'qemu-gem5-buildroot', 'all-baremetal', + 'user-mode-qemu', ]), 'all-baremetal': _Component(dependencies=[ 'qemu-baremetal', @@ -141,12 +172,6 @@ This is equivalent to: ], supported_archs=common.consts['crosstool_ng_supported_archs'], ), - 'all-linux': _Component(dependencies=[ - 'qemu-gem5-buildroot', - 'gem5-debug', - 'gem5-fast', - 'qemu-user', - ]), 'baremetal': _Component(dependencies=[ 'baremetal-gem5', 'baremetal-qemu', @@ -282,6 +307,18 @@ This is equivalent to: ], supported_archs=common.consts['crosstool_ng_supported_archs'], ), + 'test-user-mode': _Component(dependencies=[ + 'test-user-mode-qemu', + 'test-user-mode-gem5', + ]), + 'test-user-mode-qemu': _Component(dependencies=[ + 'user-mode-qemu', + 'userland', + ]), + 'test-user-mode-gem5': _Component(dependencies=[ + 'gem5', + 'userland-gem5', + ]), 'user-mode-qemu': _Component( dependencies=['qemu-user', 'userland'], ), @@ -289,6 +326,10 @@ This is equivalent to: self._build_file('build-userland'), dependencies=['buildroot'], ), + 'userland-gem5': _Component( + self._build_file('build-userland', static=True, userland_build_id='static'), + dependencies=['buildroot'], + ), } self.component_to_name_map = {self.name_to_component_map[key]:key for key in self.name_to_component_map} diff --git a/build-buildroot b/build-buildroot index f3b0011..cf3f4da 100755 --- a/build-buildroot +++ b/build-buildroot @@ -76,8 +76,6 @@ usually extra Buildroot targets. extra_make_args = self.sh.add_newlines(self.env['extra_make_args']) if self.env['build_linux']: extra_make_args.extend(['linux-reconfigure', LF]) - if self.env['emulator'] == 'gem5': - extra_make_args.extend(['gem5-reconfigure', LF]) if self.env['arch'] == 'x86_64': defconfig = 'qemu_x86_64_defconfig' elif self.env['arch'] == 'arm': @@ -151,7 +149,6 @@ usually extra Buildroot targets. self.sh.run_cmd( [ 'make', LF, - 'LKMC_GEM5_SRCDIR="{}"'.format(self.env['gem5_source_dir']), LF, 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(self.env['parsec_benchmark_source_dir']), LF, 'O={}'.format(self.env['buildroot_build_dir']), LF, 'V={}'.format(int(self.env['verbose'])), LF, diff --git a/build-test b/build-test index 87a6ad0..5a8eafb 100755 --- a/build-test +++ b/build-test @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# Build just enough to run ./test: +# https://github.com/cirosantilli/linux-kernel-module-cheat#automated-tests set -eu test_size=1 while [ $# -gt 0 ]; do @@ -10,4 +12,4 @@ while [ $# -gt 0 ]; do esac done ./build-test-boot --size "$test_size" -./build --all-archs test-gdb +./build --all-archs test-gdb test-user-mode diff --git a/build-test-boot b/build-test-boot index 34efc11..59aae6f 100755 --- a/build-test-boot +++ b/build-test-boot @@ -11,5 +11,6 @@ while [ $# -gt 0 ]; do done ./build --all-archs qemu-gem5-buildroot if [ "$test_size" -ge 3 ]; then - ./build --arch aarch64 all-linux + ./build-gem5 --arch aarch64 --gem5-build-type debug + ./build-gem5 --arch aarch64 --gem5-build-type fast fi diff --git a/common.py b/common.py index d1b30c7..2b0b598 100644 --- a/common.py +++ b/common.py @@ -802,6 +802,9 @@ Valid emulators: {} ''' return self.import_path(path).Main() + def is_arch_supported(self, arch): + return self.supported_archs is None or arch in self.supported_archs + def log_error(self, msg): print('error: {}'.format(msg), file=sys.stdout) @@ -831,7 +834,7 @@ Valid emulators: {} for arch in real_archs: if arch in env['arch_short_to_long_dict']: arch = env['arch_short_to_long_dict'][arch] - if self.supported_archs is None or arch in self.supported_archs: + if self.is_arch_supported(arch): if not env['dry_run']: start_time = time.time() env['arch'] = arch @@ -1075,7 +1078,7 @@ Stop running at the first failed test. ''' ) - def run_test(self, run_obj, run_args, test_id=None): + def run_test(self, run_obj, run_args=None, test_id=None): ''' This is a setup / run / teardown setup for simple tests that just do a single run. @@ -1086,9 +1089,12 @@ Stop running at the first failed test. :param run_args: arguments to be passed to the runnable object :param test_id: test identifier, to be added in addition to of arch and emulator ids ''' - test_id_string = self.test_setup(test_id) - exit_status = run_obj(**run_args) - self.test_teardown(run_obj, exit_status, test_id_string) + if run_obj.is_arch_supported(self.env['arch']): + if run_args is None: + run_args = {} + test_id_string = self.test_setup(test_id) + exit_status = run_obj(**run_args) + self.test_teardown(run_obj, exit_status, test_id_string) def test_setup(self, test_id): test_id_string = '{} {}'.format(self.env['emulator'], self.env['arch']) diff --git a/test b/test index a26bc1d..ee9fdda 100755 --- a/test +++ b/test @@ -1,16 +1,39 @@ -#!/usr/bin/env bash -set -eu -test_size=1 -while [ $# -gt 0 ]; do - case "$1" in - --size) - test_size="$2" - shift 2 - ;; - esac -done -./test-boot --size "$test_size" -./test-userland --all-archs --all-emulators -./test-gdb --all-archs --all-emulators -./test-baremetal --all-archs --all-emulators -./test-user-mode --all-archs --all-emulators +#!/usr/bin/env python3 + +import common +import shell_helpers +from shell_helpers import LF + +class Main(common.TestCliFunction): + def __init__(self): + super().__init__( + description='''\ +Run all tests in one go. +''' + ) + self.add_argument( + '--size', + default=1, + type=int, + help='''\ +Size of the tests to run. Scale: + +* 1: a few seconds and important +* 2: < 5 minutes and important or a few seconds and not too important +* 3: all +''' + ) + + def timed_main(self): + run_args = self.get_common_args() + test_boot_args = run_args.copy() + test_boot_args['size'] = self.env['size'] + self.run_test(self.import_path_main('test-boot'), test_boot_args, 'test-boot') + self.run_test(self.import_path_main('test-userland-full-system'), run_args, 'test-userland') + self.run_test(self.import_path_main('test-baremetal'), run_args, 'test-baremetal') + self.run_test(self.import_path_main('test-user-mode'), run_args, 'test-user-mode') + self.run_test(self.import_path_main('test-gdb'), run_args, 'test-gdb') + +if __name__ == '__main__': + Main().cli_exit() + diff --git a/test-boot b/test-boot index cc179f4..3bed86b 100755 --- a/test-boot +++ b/test-boot @@ -8,7 +8,7 @@ class Main(common.TestCliFunction): def __init__(self): super().__init__( description='''\ -Run Linux kernel boot tests and benchmarks. +Test and benchmark the Linux kernel boot. Use inits that exit immediately. ''' ) self.add_argument( @@ -16,11 +16,7 @@ Run Linux kernel boot tests and benchmarks. default=1, type=int, help='''\ -Size of the tests to run. Scale: - -* 1: a few seconds and important -* 2: < 5 minutes and important or a few seconds and not too important -* 3: all +See ./test --help for --size. ''' ) @@ -64,7 +60,10 @@ Size of the tests to run. Scale: self._bench(trace='exec_tb') if self.env['emulator'] == 'gem5' and self.env['size'] >= 3: if self.env['arch'] == 'x86_64': - cpu_types = ['DerivO3CPU'] + cpu_types = [ + # TODO segfault + #'DerivO3CPU' + ] elif self.env['is_arm']: cpu_types = [ 'DerivO3CPU', @@ -86,7 +85,8 @@ Size of the tests to run. Scale: # Do a fuller testing for aarch64. for build_type in ['debug', 'fast']: self._bench(gem5_build_type=build_type) - self._bench(gem5_script='biglittle') + # Requires patching the executable. + # self._bench(gem5_script='biglittle') if __name__ == '__main__': Main().cli_exit()