polish testing a bit further

README.adoc

@@ -3172,11 +3172,17 @@ Result on <<p51>> at bad30f513c46c1b0995d3a10c0d9bc2a33dc4fa0:
 Automatically run non-interactive userland tests that can be run in user mode simulation:
 
 ....
-./build-userland --all-archs --all-emulators
-./build-userland --all-archs --all-emulators --static --userland-build-id static
+./build --all-archs test-user-mode
 ./test-user-mode --all-archs --all-emulators
 ....
 
+Or just for QEMU:
+
+....
+./build --all-archs test-user-mode-qemu
+./test-user-mode --all-archs --emulator qemu
+....
+
 Source: link:test-user-mode[]
 
 This testing excludes notably kernel module tests which depend on a full running kernel.
@@ -6695,6 +6701,8 @@ Bibliography:
 
 ==== Count boot instructions
 
+TODO: didn't port during refactor after 3b0a343647bed577586989fb702b760bd280844a. Reimplementing should not be hard.
+
 * https://www.quora.com/How-many-instructions-does-a-typical-Linux-kernel-boot-take
 * https://github.com/cirosantilli/chat/issues/31
 * https://rwmj.wordpress.com/2016/03/17/tracing-qemu-guest-execution/
@@ -10975,6 +10983,8 @@ make CROSS_COMPILE_DIR=/usr/bin
 
 == Benchmark this repo
 
+TODO: didn't fully port during refactor after 3b0a343647bed577586989fb702b760bd280844a. Reimplementing should not be hard.
+
 In this section document how benchmark builds and runs of this repo, and how to investigate what the bottleneck is.
 
 Ideally, we should setup an automated build server that benchmarks those things continuously for us, but our <<travis>> attempt failed.

build

@@ -73,6 +73,36 @@ This is equivalent to:
 ....
 ./%(prog)s --arch x86_64 qemu-buildroot
 ....
+
+Another important target is `all`:
+
+....
+./%(prog)s all
+....
+
+This does not trully build ALL configurations: that would be impractical.
+But more precisely: build the reference configuration of each major component.
+
+So e.g.: one config of Linux kenrel, Buildroot, gem5 and qemu.
+Don't do for example all possible gem5 configs: debug, opt and fast,
+as that would be huge. This ensures that every piece of software
+builds in at least one config.
+
+TODO looping over emulators is not currently supported by this script, e.g.:
+
+....
+./%(prog)s --arch x86_64 --arch aarch64 all
+....
+
+Instead, for the targets that are emulator dependent, you must select the
+taret version for the desired emulatore, e.g.:
+
+....
+./build --arch aarch64 baremetal-qemu baremetal-gem5
+....
+
+The reason is that some targets depend on emulator, while others don't,
+so looping over all of them would waste time.
 ''',
         )
         buildroot_component = _Component(
@@ -131,8 +161,9 @@ This is equivalent to:
 
         self.name_to_component_map = {
             'all': _Component(dependencies=[
-                'all-linux',
+                'qemu-gem5-buildroot',
                 'all-baremetal',
+                'user-mode-qemu',
             ]),
             'all-baremetal': _Component(dependencies=[
                     'qemu-baremetal',
@@ -141,12 +172,6 @@ This is equivalent to:
                 ],
                 supported_archs=common.consts['crosstool_ng_supported_archs'],
             ),
-            'all-linux': _Component(dependencies=[
-                'qemu-gem5-buildroot',
-                'gem5-debug',
-                'gem5-fast',
-                'qemu-user',
-            ]),
             'baremetal': _Component(dependencies=[
                 'baremetal-gem5',
                 'baremetal-qemu',
@@ -282,6 +307,18 @@ This is equivalent to:
                 ],
                 supported_archs=common.consts['crosstool_ng_supported_archs'],
             ),
+            'test-user-mode': _Component(dependencies=[
+                'test-user-mode-qemu',
+                'test-user-mode-gem5',
+            ]),
+            'test-user-mode-qemu': _Component(dependencies=[
+                'user-mode-qemu',
+                'userland',
+            ]),
+            'test-user-mode-gem5': _Component(dependencies=[
+                'gem5',
+                'userland-gem5',
+            ]),
             'user-mode-qemu': _Component(
                 dependencies=['qemu-user', 'userland'],
             ),
@@ -289,6 +326,10 @@ This is equivalent to:
                 self._build_file('build-userland'),
                 dependencies=['buildroot'],
             ),
+            'userland-gem5': _Component(
+                self._build_file('build-userland', static=True, userland_build_id='static'),
+                dependencies=['buildroot'],
+            ),
         }
         self.component_to_name_map = {self.name_to_component_map[key]:key for key in self.name_to_component_map}
 

build-buildroot

@@ -76,8 +76,6 @@ usually extra Buildroot targets.
         extra_make_args = self.sh.add_newlines(self.env['extra_make_args'])
         if self.env['build_linux']:
             extra_make_args.extend(['linux-reconfigure', LF])
-        if self.env['emulator'] == 'gem5':
-            extra_make_args.extend(['gem5-reconfigure', LF])
         if self.env['arch'] == 'x86_64':
             defconfig = 'qemu_x86_64_defconfig'
         elif self.env['arch'] == 'arm':
@@ -151,7 +149,6 @@ usually extra Buildroot targets.
         self.sh.run_cmd(
             [
                 'make', LF,
-                'LKMC_GEM5_SRCDIR="{}"'.format(self.env['gem5_source_dir']), LF,
                 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(self.env['parsec_benchmark_source_dir']), LF,
                 'O={}'.format(self.env['buildroot_build_dir']), LF,
                 'V={}'.format(int(self.env['verbose'])), LF,

build-test

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Build just enough to run ./test:
+# https://github.com/cirosantilli/linux-kernel-module-cheat#automated-tests
 set -eu
 test_size=1
 while [ $# -gt 0 ]; do
@@ -10,4 +12,4 @@ while [ $# -gt 0 ]; do
   esac
 done
 ./build-test-boot --size "$test_size"
-./build --all-archs test-gdb
+./build --all-archs test-gdb test-user-mode

build-test-boot

@@ -11,5 +11,6 @@ while [ $# -gt 0 ]; do
 done
 ./build --all-archs qemu-gem5-buildroot
 if [ "$test_size" -ge 3 ]; then
-  ./build --arch aarch64 all-linux
+  ./build-gem5 --arch aarch64 --gem5-build-type debug
+  ./build-gem5 --arch aarch64 --gem5-build-type fast
 fi

common.py

@@ -802,6 +802,9 @@ Valid emulators: {}
         '''
         return self.import_path(path).Main()
 
+    def is_arch_supported(self, arch):
+        return self.supported_archs is None or arch in self.supported_archs
+
     def log_error(self, msg):
         print('error: {}'.format(msg), file=sys.stdout)
 
@@ -831,7 +834,7 @@ Valid emulators: {}
             for arch in real_archs:
                 if arch in env['arch_short_to_long_dict']:
                     arch = env['arch_short_to_long_dict'][arch]
-                if self.supported_archs is None or arch in self.supported_archs:
+                if self.is_arch_supported(arch):
                     if not env['dry_run']:
                         start_time = time.time()
                     env['arch'] = arch
@@ -1075,7 +1078,7 @@ Stop running at the first failed test.
 '''
         )
 
-    def run_test(self, run_obj, run_args, test_id=None):
+    def run_test(self, run_obj, run_args=None, test_id=None):
         '''
         This is a setup / run / teardown setup for simple tests that just do a single run.
 
@@ -1086,9 +1089,12 @@ Stop running at the first failed test.
         :param run_args: arguments to be passed to the runnable object
         :param test_id: test identifier, to be added in addition to of arch and emulator ids
         '''
-        test_id_string = self.test_setup(test_id)
-        exit_status = run_obj(**run_args)
-        self.test_teardown(run_obj, exit_status, test_id_string)
+        if run_obj.is_arch_supported(self.env['arch']):
+            if run_args is None:
+                run_args = {}
+            test_id_string = self.test_setup(test_id)
+            exit_status = run_obj(**run_args)
+            self.test_teardown(run_obj, exit_status, test_id_string)
 
     def test_setup(self, test_id):
         test_id_string = '{} {}'.format(self.env['emulator'], self.env['arch'])

test

@@ -1,16 +1,39 @@
-#!/usr/bin/env bash
-set -eu
-test_size=1
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --size)
-      test_size="$2"
-      shift 2
-      ;;
-  esac
-done
-./test-boot --size "$test_size"
-./test-userland --all-archs --all-emulators
-./test-gdb --all-archs --all-emulators
-./test-baremetal --all-archs --all-emulators
-./test-user-mode --all-archs --all-emulators
+#!/usr/bin/env python3
+
+import common
+import shell_helpers
+from shell_helpers import LF
+
+class Main(common.TestCliFunction):
+    def __init__(self):
+        super().__init__(
+            description='''\
+Run all tests in one go.
+'''
+        )
+        self.add_argument(
+            '--size',
+            default=1,
+            type=int,
+            help='''\
+Size of the tests to run. Scale:
+
+* 1: a few seconds and important
+* 2: < 5 minutes and important or a few seconds and not too important
+* 3: all
+'''
+        )
+
+    def timed_main(self):
+        run_args = self.get_common_args()
+        test_boot_args = run_args.copy()
+        test_boot_args['size'] = self.env['size']
+        self.run_test(self.import_path_main('test-boot'), test_boot_args, 'test-boot')
+        self.run_test(self.import_path_main('test-userland-full-system'), run_args, 'test-userland')
+        self.run_test(self.import_path_main('test-baremetal'), run_args, 'test-baremetal')
+        self.run_test(self.import_path_main('test-user-mode'), run_args, 'test-user-mode')
+        self.run_test(self.import_path_main('test-gdb'), run_args, 'test-gdb')
+
+if __name__ == '__main__':
+    Main().cli_exit()
+

test-boot

@@ -8,7 +8,7 @@ class Main(common.TestCliFunction):
     def __init__(self):
         super().__init__(
             description='''\
-Run Linux kernel boot tests and benchmarks.
+Test and benchmark the Linux kernel boot. Use inits that exit immediately.
 '''
         )
         self.add_argument(
@@ -16,11 +16,7 @@ Run Linux kernel boot tests and benchmarks.
             default=1,
             type=int,
             help='''\
-Size of the tests to run. Scale:
-
-* 1: a few seconds and important
-* 2: < 5 minutes and important or a few seconds and not too important
-* 3: all
+See ./test --help for --size.
 '''
         )
 
@@ -64,7 +60,10 @@ Size of the tests to run. Scale:
             self._bench(trace='exec_tb')
         if self.env['emulator'] == 'gem5' and self.env['size'] >= 3:
             if self.env['arch'] == 'x86_64':
-                cpu_types = ['DerivO3CPU']
+                cpu_types = [
+                    # TODO segfault
+                    #'DerivO3CPU'
+                ]
             elif self.env['is_arm']:
                 cpu_types = [
                     'DerivO3CPU',
@@ -86,7 +85,8 @@ Size of the tests to run. Scale:
                 # Do a fuller testing for aarch64.
                 for build_type in ['debug', 'fast']:
                     self._bench(gem5_build_type=build_type)
-                self._bench(gem5_script='biglittle')
+                # Requires patching the executable.
+                # self._bench(gem5_script='biglittle')
 
 if __name__ == '__main__':
     Main().cli_exit()