gdb: create some automated tests with pytest

gem5 baremetal: use m5exit m5op in exit() so as to not force users to
apply a patch for almost all examples

README.adoc

@@ -767,24 +767,22 @@ Every `.c` file inside link:baremetal/[] and `.S` file inside `baremetal/arch/<a
 
 ....
 ./run --arch arm --baremetal exit
-./run --arch arm --baremetal arch/arm/semihost_exit
+./run --arch arm --baremetal arch/arm/add
 ....
 
 which will run respectively:
 
 * link:baremetal/exit.c[]
-* link:baremetal/arch/arm/m5exit.S[]
-
-which just make the emulator quit via <<semihosting>>.
+* link:baremetal/arch/arm/add.S[]
 
 Alternatively, for the sake of tab completion, we also accept relative paths inside `baremetal/`:
 
 ....
 ./run --arch arm --baremetal baremetal/exit.c
-./run --arch arm --baremetal baremetal/arch/arm/semihost_exit.c
+./run --arch arm --baremetal baremetal/arch/arm/add.S
 ....
 
-Absolute paths however as used as is an must point to the actual executable:
+Absolute paths however are used as is and must point to the actual executable:
 
 ....
 ./run --arch arm --baremetal "$(./getvar --arch arm baremetal_build_dir)/exit.elf"
@@ -793,7 +791,6 @@ Absolute paths however as used as is an must point to the actual executable:
 To use gem5 instead of QEMU do:
 
 ....
-patch -d "$(./getvar gem5_src_dir)" -p 1 < patches/manual/gem5-semihost.patch
 ./download-dependencies --baremetal --gem5
 ./build-gem5 --arch arm
 ./build-crosstool-ng --arch arm
@@ -809,8 +806,6 @@ and then <<qemu-buildroot-setup,as usual>> open a shell with:
 
 TODO: the carriage returns are a bit different than in QEMU, see: <<gem5-baremetal-carriage-return>>.
 
-The semihosting patch is required to enable <<semihosting>>, on which base functionality such as `exit()` depends, see also: https://stackoverflow.com/questions/52475268/how-to-enable-arm-semihosting-in-gem5/52475269#52475269
-
 Note that `./build-baremetal` requires the `--gem5` option, and generates separate executable images for both, as can be seen from:
 
 ....
@@ -10200,18 +10195,16 @@ and on another shell:
 
 Semihosting is a publicly documented interface specified by ARM Holdings that allows us to do some magic operations very useful in development.
 
-Semihosting is implemented both on some real devices and on simulators such as QEMU and gem5.
+Semihosting is implemented both on some real devices and on simulators such as QEMU and <<gem5-semihosting>>.
 
 It is documented at: https://developer.arm.com/docs/100863/latest/introduction
 
-Example:
+For example, the following code makes QEMU exit:
 
 ....
 ./run --arch arm --baremetal arch/arm/semihost_exit
 ....
 
-makes both the QEMU and gem5 host executables exit.
-
 Source: link:baremetal/arch/arm/no_bootloader/semihost_exit.S[]
 
 That program program contains the code:
@@ -10224,11 +10217,10 @@ svc 0x00123456
 
 and we can see from the docs that `0x18` stands for the `SYS_EXIT` command.
 
-This is also how we implement the `exit(0)` system call in C for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/common.c[].
+This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/common.c[].
 
 Other magic operations we can do with semihosting besides exiting the on the host include:
 
-* exit
 * read and write to host stdin and stdout
 * read and write to host files
 
@@ -10260,6 +10252,16 @@ Bibliography:
 * https://stackoverflow.com/questions/31990487/how-to-cleanly-exit-qemu-after-executing-bare-metal-program-without-user-interve/40957928#40957928
 * https://balau82.wordpress.com/2010/11/04/qemu-arm-semihosting/
 
+==== gem5 semihosting
+
+For gem5, you need:
+
+....
+patch -d "$(./getvar gem5_src_dir)" -p 1 < patches/manual/gem5-semihost.patch
+....
+
+https://stackoverflow.com/questions/52475268/how-to-enable-arm-semihosting-in-gem5/52475269#52475269
+
 === gem5 baremetal carriage return
 
 TODO: our example is printing newlines without automatic carriage return `\r` as in:

arm

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+build-crosstool-ng \
+;

baremetal/add.c

@@ -0,0 +1,13 @@
+#include <common.h>
+
+void main(void) {
+	int i, j, k;
+	i = 1;
+    /* test-gdb-op1 */
+	j = 2;
+    /* test-gdb-op2 */
+	k = i + j;
+    /* test-gdb-result */
+	if (k != 3)
+		assert_fail();
+}

baremetal/add.py

@@ -0,0 +1,9 @@
+def test(self):
+    self.sendline('tbreak main')
+    self.sendline('continue')
+    self.continue_to('op1')
+    assert self.get_int('i') == 1
+    self.continue_to('op2')
+    assert self.get_int('j') == 2
+    self.continue_to('result')
+    assert self.get_int('k') == 3

baremetal/arch/aarch64/add.S

@@ -0,0 +1,12 @@
+.global main
+main:
+    /* 1 + 2 == 3 */
+    mov x0, #1
+    /* test-gdb-op1 */
+    add x1, x0, #2
+    /* test-gdb-result */
+    cmp x1, #3
+    beq 1f
+    bl assert_fail
+1:
+    ret

baremetal/arch/aarch64/add.py

@@ -0,0 +1,7 @@
+def test(self):
+    self.sendline('tbreak main')
+    self.sendline('continue')
+    self.continue_to('op1')
+    assert self.get_int('$x0') == 1
+    self.continue_to('result')
+    assert self.get_int('$x1') == 3

baremetal/arch/arm/add.S

@@ -0,0 +1,12 @@
+.global main
+main:
+    /* 1 + 2 == 3 */
+    mov r0, #1
+    /* test-gdb-op1 */
+    add r1, r0, #2
+    /* test-gdb-result */
+    cmp r1, #3
+    beq 1f
+    bl assert_fail
+1:
+    bx lr

baremetal/arch/arm/add.py

@@ -0,0 +1,7 @@
+def test(self):
+    self.sendline('tbreak main')
+    self.sendline('continue')
+    self.continue_to('op1')
+    assert self.get_int('$r0') == 1
+    self.continue_to('result')
+    assert self.get_int('$r1') == 3

baremetal/interactive/hello.c

@@ -2,5 +2,4 @@
 
 void main(void) {
 	puts("hello");
-	return;
 }

baremetal/lib/common.c

@@ -61,6 +61,13 @@ int _write(int file, char *ptr, int len) {
 
 /* Only 0 is supported for now, arm semihosting cannot handle other values. */
 void _exit(int status) {
+#if defined(GEM5)
+#if defined(__arm__)
+	__asm__ __volatile__ ("mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);");
+#elif defined(__aarch64__)
+	__asm__ __volatile__ ("mov x0, #0; .inst 0XFF000110 | (0x21 << 16);");
+#endif
+#else
 #if defined(__arm__)
     __asm__ __volatile__ ("mov r0, #0x18; ldr r1, =#0x20026; svc 0x00123456");
 #elif defined(__aarch64__)
@@ -76,6 +83,7 @@ void _exit(int status) {
 		"hlt 0xf000\n"
     );
 #endif
+#endif
 }
 
 void assert_fail() {

baremetal/return.c

@@ -1,3 +1 @@
-void main(void) {
-	return;
-}
+void main(void) {}

build

@@ -22,15 +22,15 @@ class Component:
             self.dependencies = []
         else:
             self.dependencies = dependencies
-    def build(self, arch, dry_run):
+    def build(self, arch):
         if self.build_callback is not None:
             self.build_callback(arch)
 
-def build_baremetal(arch, dry_run):
-    common.run_cmd(['build-crosstool-ng'], arch)
-    common.run_cmd(['build-baremetal'], arch)
-    common.run_cmd(['build-baremetal', '--gem5'], arch)
-    common.run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch)
+def build_baremetal(arch):
+    run_cmd(['build-crosstool-ng'], arch)
+    run_cmd(['build-baremetal'], arch)
+    run_cmd(['build-baremetal', '--gem5'], arch)
+    run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch)
 
 def run_cmd(cmd, arch):
     global args
@@ -220,4 +220,4 @@ for component_name in components:
 # Do the build.
 for arch in archs:
     for component in selected_components:
-        component.build(arch, args.dry_run)
+        component.build(arch)

build-baremetal

@@ -11,11 +11,11 @@ class BaremetalComponent(common.Component):
         bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext))
         common_obj = os.path.join(common.baremetal_build_lib_dir, 'common{}'.format(common.obj_ext))
         cflags = [
+            '-I', common.baremetal_src_lib_dir, common.Newline,
+            '-O0', common.Newline,
             '-ggdb3', common.Newline,
             '-mcpu={}'.format(common.mcpu), common.Newline,
             '-nostartfiles', common.Newline,
-            '-O0', common.Newline,
-            '-I', common.baremetal_src_lib_dir, common.Newline,
         ]
         if args.prebuilt:
             gcc = 'arm-none-eabi-gcc'
@@ -31,6 +31,7 @@ class BaremetalComponent(common.Component):
                 uart_address = 0x10009000
             else:
                 raise Exception('unknown machine: ' + common.machine)
+            cflags.extend(['-D', 'GEM5'.format(uart_address), common.Newline])
         else:
             entry_address = 0x40000000
             uart_address = 0x09000000
@@ -50,8 +51,7 @@ class BaremetalComponent(common.Component):
             cflags +
             [
                 '-c', common.Newline,
-                '-D', common.Newline,
-                'UART0_ADDR={:#x}'.format(uart_address), common.Newline,
+                '-D', 'UART0_ADDR={:#x}'.format(uart_address), common.Newline,
                 '-o', common_obj, common.Newline,
                 os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext), common.Newline,
             ]

common.py

@@ -657,7 +657,7 @@ def run_cmd(
     #sigpipe_old = signal.getsignal(signal.SIGPIPE)
     #signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
-    cmd = [x for x in cmd if x != this_module.Newline]
+    cmd = this_module.strip_newlines(cmd)
     if not dry_run and not this_module.dry_run:
         # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
         with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
@@ -927,20 +927,13 @@ def setup(parser):
                 this_module.baremetal_build_dir,
                 this_module.baremetal_build_ext,
             )
+            this_module.source_path = glob.glob(os.path.splitext(os.path.join(
+                this_module.baremetal_src_dir,
+                os.path.relpath(path, this_module.baremetal_build_dir)
+            ))[0] + '.*')[0]
         this_module.image = path
     return args
 
-def setup_dry_run_arguments(args):
-    this_module.dry_run = args.dry_run
-
-def shlex_split(string):
-    '''
-    shlex_split, but also add Newline after every word.
-
-    Not perfect since it does not group arguments, but I don't see a solution.
-    '''
-    return this_module.add_newlines(shlex.split(string))
-
 def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext):
     if os.path.isabs(in_path):
         return in_path
@@ -967,6 +960,20 @@ def resolve_userland(path):
         this_module.userland_build_ext,
     )
 
+def setup_dry_run_arguments(args):
+    this_module.dry_run = args.dry_run
+
+def shlex_split(string):
+    '''
+    shlex_split, but also add Newline after every word.
+
+    Not perfect since it does not group arguments, but I don't see a solution.
+    '''
+    return this_module.add_newlines(shlex.split(string))
+
+def strip_newlines(cmd):
+    return [x for x in cmd if x != this_module.Newline]
+
 def write_configs(config_path, configs, config_fragments=None):
     """
     Write extra configs into the Buildroot config file.

requirements.txt

@@ -0,0 +1 @@
+pexpect==4.6.0

run

@@ -152,12 +152,11 @@ def main(args, extra_args=None):
         cmd.extend(
             [
                 common.executable, common.Newline,
-                '--debug-file=trace.txt', common.Newline
+                '--debug-file=trace.txt', common.Newline,
+                '--listener-mode', 'on', common.Newline,
+                '--outdir', common.m5out_dir, common.Newline,
             ] +
-            gem5_exe_args +
-            [
-                '-d', common.m5out_dir, common.Newline
-            ]
+            gem5_exe_args
         )
         if args.userland is not None:
             cmd.extend([

run-gdb

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
+import imp
 import os
-import sys
 import signal
 import subprocess
+import sys
 
 import common
 
@@ -14,10 +15,70 @@ defaults = {
     'kgdb': False,
     'no_continue': False,
     'no_lxsymbols': False,
+    'test': False,
     'sim': False,
     'userland': None,
 }
 
+class GdbTestcase:
+    def __init__(
+        self,
+        source_path,
+        test_script_path,
+        cmd,
+        debug=False
+    ):
+        '''
+        :param debug: if True, print extra debug information to help understand
+                    why a test is not working
+        '''
+        self.prompt = '\(gdb\) '
+        self.source_path = source_path
+        common.print_cmd(cmd)
+        cmd = common.strip_newlines(cmd)
+        import pexpect
+        self.child = pexpect.spawn(
+            cmd[0],
+            cmd[1:],
+            encoding='utf-8'
+        )
+        if debug:
+            self.child.logfile = sys.stdout
+        self.child.setecho(False)
+        self.child.expect(self.prompt)
+        test = imp.load_source('test', test_script_path)
+        test.test(self)
+        self.child.sendcontrol('d')
+        self.child.close()
+
+    def before(self):
+        return self.child.before.rstrip()
+
+    def continue_to(self, lineid):
+        line_number = self.find_line(lineid)
+        self.sendline('tbreak {}'.format(line_number))
+        self.sendline('continue')
+
+    def get_int(self, int_id):
+        self.sendline('printf "%d\\n", {}'.format(int_id))
+        return int(self.before())
+
+    def find_line(self, lineid):
+        '''
+        Search for the first line that contains a comment line
+        that ends in /* test-gdb-<lineid> */ and return the line number.
+        '''
+        lineend = '/* test-gdb-' + lineid + ' */'
+        with open(self.source_path, 'r') as f:
+            for i, line in enumerate(f):
+                if line.rstrip().endswith(lineend):
+                    return i + 1
+        return -1
+
+    def sendline(self, line):
+        self.child.sendline(line)
+        self.child.expect(self.prompt)
+
 def main(args, extra_args=None):
     '''
     :param args: argparse parse_argument() output. Must contain all the common options,
@@ -34,6 +95,14 @@ def main(args, extra_args=None):
     args = common.resolve_args(defaults, args, extra_args)
     after = common.shlex_split(args.after)
     before = common.shlex_split(args.before)
+    no_continue = args.no_continue
+    if args.test:
+        no_continue = True
+        before.extend([
+            '-q', common.Newline,
+            '-nh', common.Newline,
+            '-ex', 'set confirm off', common.Newline
+        ])
     if args.break_at is not None:
         break_at = ['-ex', 'break {}'.format(args.break_at), common.Newline]
     else:
@@ -43,6 +112,7 @@ def main(args, extra_args=None):
         image = common.resolve_userland(args.userland)
     elif args.baremetal:
         image = common.image
+        test_script_path = os.path.splitext(common.source_path)[0] + '.py'
     else:
         image = common.vmlinux
     if args.baremetal:
@@ -70,7 +140,7 @@ def main(args, extra_args=None):
     ])
     if not args.kgdb:
         cmd.extend(break_at)
-    if not args.no_continue:
+    if not no_continue:
         # ## lx-symbols
         #
         # ### lx-symbols after continue
@@ -94,14 +164,21 @@ def main(args, extra_args=None):
         if not args.no_lxsymbols and linux_full_system:
             cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), common.Newline])
     cmd.extend(after)
-    # I would rather have cwd be out_rootfs_overlay_dir,
-    # but then lx-symbols cannot fine the vmlinux and fails with:
-    # vmlinux: No such file or directory.
-    return common.run_cmd(
-        cmd,
-        cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'),
-        cwd=common.linux_build_dir
-    )
+    if args.test:
+        GdbTestcase(
+            common.source_path,
+            test_script_path,
+            cmd
+        )
+    else:
+        # I would rather have cwd be out_rootfs_overlay_dir,
+        # but then lx-symbols cannot fine the vmlinux and fails with:
+        # vmlinux: No such file or directory.
+        return common.run_cmd(
+            cmd,
+            cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'),
+            cwd=common.linux_build_dir
+        )
 
 if __name__ == '__main__':
     parser = common.get_argparse(argparse_args={'description': 'Connect with GDB to an emulator to debug Linux itself'})
@@ -129,6 +206,13 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-s
     parser.add_argument(
         '-X', '--no-lxsymbols', default=defaults['no_lxsymbols'], action='store_true'
     )
+    parser.add_argument(
+        '--test', default=defaults['test'], action='store_true',
+        help='''\
+Run an expect test case instead of interactive usage. For baremetal and userland,
+the script is a .py file next to the source code.
+'''
+    )
     parser.add_argument(
         '--userland', default=defaults['userland'],
     )

test-gdb

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -eux
+
+# QEMU
+./run --arch arm --background --baremetal add --wait-gdb &
+./run-gdb --arch arm --baremetal add --test
+wait
+./run --arch arm --background --baremetal arch/arm/add --wait-gdb &
+./run-gdb --arch arm --baremetal arch/arm/add --test
+wait
+./run --arch aarch64 --background --baremetal arch/aarch64/add --wait-gdb &
+./run-gdb --arch aarch64 --baremetal arch/aarch64/add --test
+wait
+
+# gem5
+./run --arch arm --background --baremetal add --gem5 --wait-gdb &
+./run-gdb --arch arm --baremetal add --gem5 --test
+wait
+./run --arch arm --background --baremetal arch/arm/add --gem5 --wait-gdb &
+./run-gdb --arch arm --baremetal arch/arm/add --gem5 --test
+wait
+./run --arch aarch64 --background --baremetal arch/aarch64/add --gem5 --wait-gdb &
+./run-gdb --arch aarch64 --baremetal arch/aarch64/add --gem5 --test
+wait