test-userland: rename to test-userland-full-system and port to LkmcCliFunction

README.adoc

@@ -2467,7 +2467,7 @@ asdf=qwer
 
 Source: link:userland/init_env_poweroff.c[].
 
-==== init environment args
+==== init arguments
 
 The annoying dash `-` gets passed as a parameter to `init`, which makes it impossible to use this method for most non custom executables.
 
@@ -2488,6 +2488,8 @@ ab
 
 so see how `a.b` is gone.
 
+The simple workaround is to just create a shell script that does it, e.g. as we've done at: link:rootfs_overlay/gem5_exit.sh[].
+
 ==== init environment env
 
 Wait, where do `HOME` and `TERM` come from? (greps the kernel). Ah, OK, the kernel sets those by default: https://github.com/torvalds/linux/blob/94710cac0ef4ee177a63b5227664b38c95bbf703/init/main.c#L173
@@ -11886,11 +11888,16 @@ This directory is copied into the target filesystem by:
 
 ....
 ./copy-overlay
+./build-buildroot
 ....
 
-Source link:copy-overlay[]
+Source: link:copy-overlay[]
 
-Since this directory does not require compilation, we also make it <<9p>> available to the guest directly even without `copy-overlay` at:
+`copy-overlay` by itself, only places the files into our intermediate `./getenv out_rootfs_overlay_dir` directory.
+
+This directory combines files from several sources, including for example link:build-userland[], which the final `./build-buildroot` puts into the root filesystem.
+
+Since the link:rootfs_overlay[] directory does not require compilation, unlike say link:userland[] we also make it <<9p>> available to the guest directly even without `copy-overlay` at:
 
 ....
 ls /mnt/9p/rootfs_overlay
@@ -11900,10 +11907,6 @@ This way you can just hack away the scripts and try them out immediately without
 
 To add those scripts to the Buildroot root filesystem, you will need to run:
 
-....
-./build-buildroot
-....
-
 We could add that directory to `BR2_ROOTFS_OVERLAY` but we don't do this because this mechanism:
 
 * also works for non Buildroot root filesystesms
@@ -11915,7 +11918,7 @@ and maintaining `BR2_ROOTFS_OVERLAY` in addition to our mechanism would duplicat
 
 ==== Automated tests
 
-Run absolutely all tests:
+Run almost all tests:
 
 ....
 ./build-test --size 3 && \
@@ -11929,15 +11932,20 @@ Sources:
 
 * link:build-test[]
 * link:test[]
-* link:test-userland[]
 * <<user-mode-tests>>
 * <<baremetal-tests>>
 
+This is not all tests, because there are too many possible variations and that would take forever.
+
+Instead, we currently select on magic arch, currently `aarch64`, and for that arch run more stuff than on others.
+
+We could in the future we will add an option to select the large arch, or do something smarter.
+
 This full testing takes too much time to be feasible for every patch, but it should be done for every release.
 
 See the sources of those test scripts to learn how to run more specialized tests.
 
-One important tip is that you can select multiple archs and emulators of interest with a command such as:
+One important tip is that you can select multiple archs and emulators of interest with any command as in:
 
 ....
 ./test-user-mode \
@@ -11958,7 +11966,21 @@ Test that the Internet works:
 ./run --arch x86_64 --kernel-cli '- lkmc_eval="ifup -a;wget -S google.com;poweroff;"'
 ....
 
-Source: link:rootfs_overlay/test_all.sh[].
+===== Test userland in full system
+
+Run all userland tests from inside full system simulation (i.e. not <<user-mode-simulation>>):
+
+....
+./test-userland-full-system
+....
+
+This includes, in particular, userland programs that test the kernel modules, which cannot be tested in user mode simulation.
+
+Basically just boots and runs: link:rootfs_overlay/test_all.sh[]
+
+Failure is detected by looking for the <<magic-failure-string>>
+
+Most userland programs that don't rely on kernel modules can also be tested in user mode simulation as explained at: <<user-mode-tests>>.
 
 ===== Test GDB
 
@@ -12037,7 +12059,7 @@ to the terminal, then our run scripts detect that and exit with status `1`.
 This magic output string is notably used by:
 
 * the `common_assert_fail()` function, which is used by <<baremetal-tests>>
-* link:rootfs_overlay/test_fail.sh[], which is used by the link:test-userland[]
+* link:rootfs_overlay/test_fail.sh[], which is used by <<test-userland-in-full-system>>
 
 === Bisection
 

common.py

@@ -439,11 +439,15 @@ Valid emulators: {}
                         env['machine'] = 'VExpress_GEM5_V1'
             else:
                 if not env['_args_given']['machine']:
-                    # highmem=off needed since v3.0.0 due to:
+                    env['machine'] = 'virt'
-                    # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
+                    if env['arch'] == 'arm':
-                    env['machine'] = 'virt,highmem=off'
+                        # highmem=off needed since v3.0.0 due to:
-                    if env['arch'] == 'aarch64':
+                        # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
-                        env['machine'] += ',gic_version=3'
+                        env['machine2'] = 'highmem=off'
+                    elif env['arch'] == 'aarch64':
+                        env['machine2'] = 'gic_version=3'
+        else:
+            env['machine2'] = None
 
         # Buildroot
         env['buildroot_build_dir'] = join(env['buildroot_out_dir'], 'build', env['buildroot_build_id'], env['arch'])
@@ -556,7 +560,7 @@ Valid emulators: {}
             env['trace_txt_file'] = env['qemu_trace_txt_file']
         env['run_cmd_file'] = join(env['run_dir'], 'run.sh')
 
-        # Linux kernl.
+        # Linux kernel.
         if not env['_args_given']['linux_build_dir']:
             env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch'])
         env['lkmc_vmlinux'] = join(env['linux_build_dir'], 'vmlinux')
@@ -577,6 +581,11 @@ Valid emulators: {}
         else:
             env['vmlinux'] = env['lkmc_vmlinux']
             env['linux_image'] = env['lkmc_linux_image']
+        if env['emulator']== 'gem5':
+            env['userland_quit_cmd'] = '/gem5_exit.sh'
+        else:
+            env['userland_quit_cmd'] = '/poweroff.out'
+        env['quit_init'] = 'init={}'.format(env['userland_quit_cmd'])
 
         # Kernel modules.
         env['kernel_modules_build_dir'] = join(env['kernel_modules_build_base_dir'], env['arch'])
@@ -1043,7 +1052,19 @@ class Test:
         return ' '.join(out)
 
 class TestCliFunction(LkmcCliFunction):
+    '''
+    Represents a CLI command that runs tests.
+
+    Automates test reporting boilerplate for those commands.
+    '''
+
     def __init__(self, *args, **kwargs):
+        defaults = {
+            'print_time': False,
+        }
+        if 'defaults' in kwargs:
+            defaults.update(kwargs['defaults'])
+        kwargs['defaults'] = defaults
         super().__init__(*args, **kwargs)
         self.tests = []
         self.add_argument(
@@ -1054,13 +1075,25 @@ Stop running at the first failed test.
 '''
         )
 
-    def run_test(self, run_obj, run_args, extra_params):
+    def run_test(self, run_obj, run_args, test_id=None):
-        test_id_string = self.test_setup(extra_params)
+        '''
+        This is a setup / run / teardown setup for simple tests that just do a single run.
+
+        More complex tests might need to run the steps separately, e.g. gdb tests
+        must run multiple commands: one for the run and one GDB.
+
+        :param run_obj: callable object
+        :param run_args: arguments to be passed to the runnable object
+        :param test_id: test identifier, to be added in addition to of arch and emulator ids
+        '''
+        test_id_string = self.test_setup(test_id)
         exit_status = run_obj(**run_args)
         self.test_teardown(run_obj, exit_status, test_id_string)
 
-    def test_setup(self, extra_params):
+    def test_setup(self, test_id):
-        test_id_string = '{} {} {}'.format(self.env['emulator'], self.env['arch'], extra_params)
+        test_id_string = '{} {}'.format(self.env['emulator'], self.env['arch'])
+        if test_id is not None:
+            test_id_string += ' {}'.format(test_id)
         self.log_info('test_id {}'.format(test_id_string), flush=True)
         return test_id_string
 

copy-overlay

@@ -13,7 +13,6 @@ class Main(common.BuildCliFunction):
             description='''\
 https://github.com/cirosantilli/linux-kernel-module-cheat#rootfs_overlay
 ''')
-
     def build(self):
         # TODO: print rsync equivalent, move into shell_helpers.
         distutils.dir_util.copy_tree(

rootfs_overlay/gem5_exit.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+# To be able to do init=/gem5_exit.sh, since kernel CLI argument passing is too messy:
+# https://github.com/cirosantilli/linux-kernel-module-cheat#init-arguments
+m5 exit

run

@@ -472,10 +472,20 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
                                 .format(virtfs_dir=virtfs_dir, virtfs_tag=virtfs_tag),
                             LF,
                         ])
+                if self.env['machine2'] is not None:
+                    # Multiple -machine options can also be given comma separated in one -machine.
+                    # We use multiple because the machine is used as an identifier on baremetal tests
+                    # build paths, so better keep them clean.
+                    machine2 = ['-machine', self.env['machine2'], LF]
+                else:
+                    machine2 = []
                 cmd.extend(
                     [
                         qemu_executable, LF,
                         '-machine', self.env['machine'], LF,
+                    ] +
+                    machine2 +
+                    [
                         '-device', 'rtl8139,netdev=net0', LF,
                         '-gdb', 'tcp::{}'.format(self.env['gdb_port']), LF,
                         '-kernel', self.env['image'], LF,

test

@@ -10,7 +10,7 @@ while [ $# -gt 0 ]; do
   esac
 done
 ./test-boot --size "$test_size"
-./test-modules --all-archs --all-emulators
+./test-userland --all-archs --all-emulators
 ./test-gdb --all-archs --all-emulators
 ./test-baremetal --all-archs --all-emulators
 ./test-user-mode --all-archs --all-emulators

test-baremetal

@@ -8,9 +8,6 @@ import common
 class Main(common.TestCliFunction):
     def __init__(self):
         super().__init__(
-            defaults={
-                'print_time': False,
-            },
             supported_archs=common.consts['crosstool_ng_supported_archs'],
         )
         self.add_argument(

test-boot

@@ -7,9 +7,6 @@ from shell_helpers import LF
 class Main(common.TestCliFunction):
     def __init__(self):
         super().__init__(
-            defaults={
-                'print_time': False,
-            },
             description='''\
 Run Linux kernel boot tests and benchmarks.
 '''
@@ -55,10 +52,7 @@ Size of the tests to run. Scale:
         self.run = self.import_path_main('run')
         self.common_args = self.get_common_args()
         self.common_args['ctrl_c_host'] = True
-        if self.env['emulator'] == 'gem5':
+        self.common_args['kernel_cli'] = self.env['quit_init']
-            self.common_args['eval'] = 'm5 exit'
-        elif self.env['emulator'] == 'qemu':
-            self.common_args['eval'] = '/poweroff.out'
         if (self.env['emulator'] == 'qemu' or
                 (self.env['emulator'] == 'gem5' and self.env['size'] >= 2)):
             self._bench()

test-gdb

@@ -7,11 +7,7 @@ import common
 
 class Main(common.TestCliFunction):
     def __init__(self):
-        super().__init__(
+        super().__init__()
-            defaults={
-                'print_time': False,
-            },
-        )
         self.add_argument(
             'tests',
             nargs='*',

test-user-mode

@@ -8,9 +8,6 @@ import common
 class Main(common.TestCliFunction):
     def __init__(self):
         super().__init__(
-            defaults={
-                'print_time': False,
-            },
             description='''\
 https://github.com/cirosantilli/linux-kernel-module-cheat#user-mode-tests
 '''

test-userland

@@ -1,29 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-
-import common
-
-class Main(common.LkmcCliFunction):
-    def __init__(self):
-        super().__init__(
-            defaults={
-                'print_time': False,
-            },
-            description='''\
-Boot Linux and run all non-interactive userland tests, including those
-for kernel modules.
-
-Detect failure based on the magic terminal failure string.
-'''
-        )
-
-    def timed_main(self):
-        run = self.import_path_main('run')
-        run_args = self.get_common_args()
-        run_args['eval_after'] = '/test_all.sh;/poweroff.out'
-        run(**run_args)
-
-if __name__ == '__main__':
-    Main().cli()

test-userland-full-system

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+import common
+
+class Main(common.TestCliFunction):
+    def __init__(self):
+        super().__init__(
+            description='''\
+https://github.com/cirosantilli/linux-kernel-module-cheat#test-userland-in-full-system
+'''
+        )
+    def timed_main(self):
+        run = self.import_path_main('run')
+        run_args = self.get_common_args()
+        run_args['eval_after'] = '/test_all.sh;{};'.format(self.env['userland_quit_cmd'])
+        self.run_test(run, run_args)
+
+if __name__ == '__main__':
+    Main().cli()

trace-boot

@@ -15,18 +15,13 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#t
     def timed_main(self):
         args = self.get_common_args()
         run = self.import_path_main('run')
+        self.common_args['kernel_cli'] = self.env['quit_init']
         if self.env['emulator'] == 'gem5':
-            args.update({
+            args['trace'] = 'Exec,-ExecSymbol,-ExecMicro'
-                'eval': 'm5 exit',
-                'trace': 'Exec,-ExecSymbol,-ExecMicro',
-            })
             run.main(**args)
         elif self.env['emulator'] == 'qemu':
             run_args = args.copy()
-            run_args.update({
+            args['trace'] = 'exec_tb'
-                'kernel_cli': 'init=/poweroff.out',
-                'trace': 'exec_tb',
-            })
             run.main(**run_args)
             qemu_trace2txt = self.import_path_main('qemu-trace2txt')
             qemu_trace2txt.main(**args)