From 7597834f317662842d140932708e13e6bc67c814 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?=
 =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= <ciro.santilli@gmail.com>
Date: Tue, 22 Jan 2019 00:00:00 +0000
Subject: [PATCH] test-baremetal: create

---
 README.adoc    | 21 +++++++++++++++--
 test           |  5 ++--
 test-baremetal | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 86 insertions(+), 4 deletions(-)
 create mode 100755 test-baremetal

diff --git a/README.adoc b/README.adoc
index 9c80b00..9b0f09e 100644
--- a/README.adoc
+++ b/README.adoc
@@ -3157,9 +3157,9 @@ Result on <<p51>> at bad30f513c46c1b0995d3a10c0d9bc2a33dc4fa0:
 * QEMU user: 45 seconds
 * QEMU full system: 223 seconds
 
-=== User mode testing
+=== User mode tests
 
-Automatically run non-interactive userland tests that don't depend on nay kernel modules:
+Automatically run non-interactive userland tests that don't depend on any kernel modules:
 
 ....
 ./build-userland --all-archs --all-emulators
@@ -10897,6 +10897,23 @@ We then found out that QEMU starts in EL1, and so we kept just the EL1 part, and
 * https://stackoverflow.com/questions/42824706/qemu-system-aarch64-entering-el1-when-emulating-a53-power-up
 * https://stackoverflow.com/questions/37299524/neon-support-in-armv8-system-mode-qemu
 
+=== Baremetal tests
+
+Automatically run non-interactive baremetal tests:
+
+....
+./test-baremetal
+....
+
+Source: link:test-baremetal[]
+
+Since there is no standardized exit status concept that works across all emulators, we detect if tests failed by parsing logs for the string output by our fail function: `common_assert_fail()`.
+
+We also skip tests that cannot work on certain conditions based on their basenames, e.g.:
+
+* tests that start with `gem5_` only run in `gem5`
+* tests that start with `semihost_` only run in QEMU, until we find a better way to automate <<gem5-semihosting>>
+
 === Baremetal bibliography
 
 https://stackoverflow.com/questions/43682311/uart-communication-in-gem5-with-arm-bare-metal
diff --git a/test b/test
index 0819339..96c423c 100755
--- a/test
+++ b/test
@@ -11,5 +11,6 @@ while [ $# -gt 0 ]; do
 done
 ./bench-boot --size "$test_size"
 ./test-modules
-./test-gdb
-./test-userland
+./test-gdb --all-archs --all-emulators
+./test-baremetal --all-archs --all-emulators
+./test-userland --all-archs --all-emulators
diff --git a/test-baremetal b/test-baremetal
new file mode 100755
index 0000000..698e74c
--- /dev/null
+++ b/test-baremetal
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+import common
+
+class Main(common.LkmcCliFunction):
+    def __init__(self):
+        super().__init__(
+            defaults={
+                'print_time': False,
+            },
+            supported_archs=common.consts['crosstool_ng_supported_archs'],
+        )
+        self.add_argument(
+            'tests',
+            nargs='*',
+            help='''\
+If given, run only the given tests. Otherwise, run all tests.
+'''
+        )
+
+    def timed_main(self):
+        run = self.import_path_main('run')
+        run_args = self.get_common_args()
+        if self.env['emulator'] == 'gem5':
+            run_args['userland_build_id'] = 'static'
+        if self.env['tests'] == []:
+            baremetal_source_exts = (self.env['c_ext'], self.env['asm_ext'])
+            paths = []
+            for f in os.listdir(self.env['baremetal_source_dir']):
+                path = os.path.join(self.env['baremetal_source_dir'], f)
+                if os.path.isfile(path) and os.path.splitext(path)[1] in baremetal_source_exts:
+                    paths.append(path)
+            for root, dirs, files in os.walk(self.env['baremetal_source_arch_dir'], topdown=True):
+                dirs[:] = [d for d in dirs if d != 'interactive']
+                for file in files:
+                    path = os.path.join(root, file)
+                    if os.path.splitext(path)[1] in baremetal_source_exts:
+                        paths.append(path)
+            sources = []
+            for path in paths:
+                if not (
+                        self.env['emulator'] == 'gem5' and os.path.basename(path).startswith('semihost_') or
+                        self.env['emulator'] == 'qemu' and os.path.basename(path).startswith('gem5_')
+                    ):
+                        sources.append(os.path.relpath(path, self.env['baremetal_source_dir']))
+        else:
+            sources = self.env['tests']
+        for source in sources:
+            run_args['baremetal'] = source
+            run_args['background'] = True
+            test_id_string = self.test_setup(run_args, source)
+            if os.path.splitext(os.path.basename(source))[0] == 'multicore':
+                run_args['cpus'] = 2
+            exit_status = run(**run_args)
+            self.test_teardown(run)
+            if exit_status != 0:
+                self.log_error('test failed, program exit status: {} test id: {}'.format(exit_status, test_id_string))
+                sys.exit(1)
+
+if __name__ == '__main__':
+    Main().cli()