build-baremetal: enable parallel build and target selection just like build-userland

The factoring out also led to some small bugs being found and solved ;-)

baremetal/arch/aarch64/multicore.S

@@ -36,7 +36,7 @@ cpu1_sleep_forever:
 cpu0_only:
     /* Only CPU 0 reaches this point. */
 
-#if !defined(GEM5)
+#if !LKMC_GEM5
     /* Wake up CPU 1 from initial sleep!
      * See:https://github.com/cirosantilli/linux-kernel-module-cheat#psci
      */

baremetal/arch/aarch64/svc.c

@@ -1,5 +1,6 @@
 /* https://github.com/cirosantilli/linux-kernel-module-cheat#svc */
 
+#include <assert.h>
 #include <inttypes.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -57,8 +58,8 @@ int main(void) {
     printf("daif 0x%" PRIx32 "\n", lkmc_sysreg_daif_read());
     printf("spsel 0x%" PRIx32 "\n", lkmc_sysreg_spsel_read());
     printf("vbar_el1 0x%" PRIx64 "\n", lkmc_sysreg_vbar_el1_read());
-    lkmc_assert(myvar == 0);
+    assert(myvar == 0);
     LKMC_SVC(0x42);
-    lkmc_assert(myvar == 1);
+    assert(myvar == 1);
     return 0;
 }

baremetal/arch/arm/dump_regs.c

@@ -5,7 +5,7 @@
 
 int main(void) {
     uint32_t cpsr;
-    uint32_t mvfr1;
+    /*uint32_t mvfr1;*/
     __asm__ ("mrs %0, cpsr" : "=r" (cpsr) : :);
     /* TODO this is blowing up an exception, how to I read from it? */
     /*__asm__ ("vmrs %0, mvfr1" : "=r" (mvfr1) : :);*/

baremetal/arch/arm/multicore.S

@@ -19,7 +19,7 @@ cpu1_sleep_forever:
     wfe
     b cpu1_sleep_forever
 cpu0_only:
-#if !defined(GEM5)
+#if !LKMC_GEM5
     /* PSCI CPU_ON. */
     ldr r0, =0x84000003
     mov r1, #1

baremetal/getchar.c

@@ -16,7 +16,7 @@ int main(void) {
             puts("out of memory");
             break;
         } else {
-            printf("new alloc of %d bytes at address 0x%p\n", alloc_size, ptr);
+            printf("new alloc of %zu bytes at address 0x%p\n", alloc_size, ptr);
             alloc_size <<= 1;
         }
     }

baremetal/lib/syscalls.c

@@ -2,6 +2,7 @@
 #include <stdlib.h>
 #include <sys/stat.h>
 
+#include <lkmc.h>
 #include <lkmc/m5ops.h>
 
 enum {
@@ -11,10 +12,14 @@ enum {
 #define UART_DR(baseaddr) (*(unsigned int *)(baseaddr))
 #define UART_FR(baseaddr) (*(((unsigned int *)(baseaddr))+6))
 
-int _close(int file) { return -1; }
+int _close(int file) {
+    LKMC_UNUSED(file);
+    return -1;
+}
 
 void _exit(int status) {
-#if defined(GEM5)
+    LKMC_UNUSED(status);
+#if LKMC_GEM5
     LKMC_M5OPS_EXIT;
 #else
 #if defined(__arm__)
@@ -46,6 +51,7 @@ void _exit(int status) {
 }
 
 int _fstat(int file, struct stat *st) {
+    LKMC_UNUSED(file);
     st->st_mode = S_IFCHR;
     return 0;
 }
@@ -55,26 +61,41 @@ int _getpid(void) { return 0; }
 
 /* Required by assert. */
 int _kill(pid_t pid, int sig) {
+    LKMC_UNUSED(pid);
     exit(128 + sig);
 }
 
-int _isatty(int file) { return 1; }
+int _isatty(int file) {
+    LKMC_UNUSED(file);
+    return 1;
+}
 
-int _lseek(int file, int ptr, int dir) { return 0; }
+int _lseek(int file, int ptr, int dir) {
+    LKMC_UNUSED(file);
+    LKMC_UNUSED(ptr);
+    LKMC_UNUSED(dir);
+    return 0;
+}
 
-int _open(const char *name, int flags, int mode) { return -1; }
+int _open(const char *name, int flags, int mode) {
+    LKMC_UNUSED(name);
+    LKMC_UNUSED(flags);
+    LKMC_UNUSED(mode);
+    return -1;
+}
 
 int _read(int file, char *ptr, int len) {
     int todo;
+    LKMC_UNUSED(file);
     if (len == 0)
         return 0;
-    while (UART_FR(UART0_ADDR) & UART_FR_RXFE);
-    *ptr++ = UART_DR(UART0_ADDR);
+    while (UART_FR(LKMC_UART0_ADDR) & UART_FR_RXFE);
+    *ptr++ = UART_DR(LKMC_UART0_ADDR);
     for (todo = 1; todo < len; todo++) {
-        if (UART_FR(UART0_ADDR) & UART_FR_RXFE) {
+        if (UART_FR(LKMC_UART0_ADDR) & UART_FR_RXFE) {
             break;
         }
-        *ptr++ = UART_DR(UART0_ADDR);
+        *ptr++ = UART_DR(LKMC_UART0_ADDR);
     }
     return todo;
 }
@@ -98,8 +119,9 @@ caddr_t _sbrk(int incr) {
 
 int _write(int file, char *ptr, int len) {
     int todo;
+    LKMC_UNUSED(file);
     for (todo = 0; todo < len; todo++) {
-        UART_DR(UART0_ADDR) = *ptr++;
+        UART_DR(LKMC_UART0_ADDR) = *ptr++;
     }
     return len;
 }

build-baremetal

@@ -2,6 +2,7 @@
 import os
 
 import common
+import thread_pool
 from shell_helpers import LF
 
 class Main(common.BuildCliFunction):
@@ -18,14 +19,20 @@ Build the baremetal examples with crosstool-NG.
         self._add_argument('--ccflags')
         self._add_argument('--force-rebuild')
         self._add_argument('--optimization-level')
+        self.add_argument(
+            'targets',
+            default=[],
+            help='Analogous to ./build-userland target selection',
+            nargs='*',
+        )
 
     def build(self):
         build_dir = self.get_build_dir()
-        bootloader_obj = os.path.join(
+        extra_obj_baremetal_bootloader = os.path.join(
             self.env['baremetal_build_lib_dir'],
             'bootloader{}'.format(self.env['obj_ext'])
         )
-        common_obj = os.path.join(
+        extra_obj_lkmc_common = os.path.join(
             self.env['baremetal_build_lib_dir'],
             self.env['common_basename_noext'] + self.env['obj_ext']
         )
@@ -38,17 +45,14 @@ Build the baremetal examples with crosstool-NG.
             self.env['baremetal_build_lib_dir'],
             syscalls_basename_noext + self.env['obj_ext']
         )
-        common_objs = [common_obj, syscalls_obj]
-        cflags = [
-            '-I', self.env['baremetal_source_lib_dir'], LF,
+        cc_flags = [
             '-I', self.env['root_dir'], LF,
             '-O{}'.format(self.env['optimization_level']), LF,
-            '-ggdb3', LF,
             '-mcpu={}'.format(self.env['mcpu']), LF,
             '-nostartfiles', LF,
         ]
         if self.env['arch'] == 'arm':
-            cflags.extend([
+            cc_flags.extend([
                 '-mhard-float', LF,
                 # This uses the soft float ABI for calling functions from objets in Newlib which
                 # our crosstool-NG config compiles with soft floats, while emiting hard float
@@ -61,7 +65,6 @@ Build the baremetal examples with crosstool-NG.
                 '-mfloat-abi=softfp', LF,
                 '-mfpu=crypto-neon-fp-armv8', LF,
             ])
-        cflags_after = ['-lm']
         if self.env['emulator'] == 'gem5':
             if self.env['machine'] == 'VExpress_GEM5_V1':
                 entry_address = 0x80000000
@@ -71,98 +74,80 @@ Build the baremetal examples with crosstool-NG.
                 uart_address = 0x10009000
             else:
                 raise Exception('unknown machine: ' + self.env['machine'])
-            cflags.extend([
-                '-D', 'GEM5'.format(uart_address), LF,
+            cc_flags.extend([
+                '-DLKMC_GEM5=1', LF,
                 '-DLKMC_M5OPS_ENABLE=1', LF,
             ])
         else:
             entry_address = 0x40000000
             uart_address = 0x09000000
-        os.makedirs(build_dir, exist_ok=True)
-        os.makedirs(self.env['baremetal_build_lib_dir'], exist_ok=True)
-        src = os.path.join(
+        cc_flags.extend(['-D', 'LKMC_UART0_ADDR={:#x}'.format(uart_address), LF])
+        cc_flags.extend(self.sh.shlex_split(self.env['ccflags']))
+        bootloader_src = os.path.join(
             self.env['baremetal_source_lib_dir'],
             '{}{}'.format(
                 self.env['arch'],
                 self.env['asm_ext']
             )
         )
-        cflags.extend(self.sh.shlex_split(self.env['ccflags']))
-        if self.need_rebuild([src], bootloader_obj):
-            self.sh.run_cmd(
-                [self.env['gcc_path'],  LF] +
-                cflags +
-                [
-                    '-c', LF,
-                    '-o', bootloader_obj, LF,
-                    src, LF,
-                ] +
-                cflags_after
-            )
-        for src, obj in [
-            (self.env['common_c'], common_obj),
+        for in_path, out_path in [
+            (bootloader_src, extra_obj_baremetal_bootloader),
+            (self.env['common_c'], extra_obj_lkmc_common),
             (syscalls_src, syscalls_obj),
         ]:
-            if self.need_rebuild([src, self.env['common_h']], obj):
-                self.sh.run_cmd(
-                    [self.env['gcc_path'],  LF] +
-                    cflags +
-                    [
-                        '-D', 'UART0_ADDR={:#x}'.format(uart_address), LF,
-                        '-c', LF,
-                        '-o', obj, LF,
-                        src, LF,
-                    ] +
-                    cflags_after
-                )
-        for subpath in [
-            '',
-            'interactive',
-            self.env['baremetal_source_arch_subpath'],
-            os.path.join(self.env['baremetal_source_arch_subpath'], 'no_bootloader'),
-        ]:
-            in_dir = os.path.join(self.env['baremetal_source_dir'], subpath)
-            if os.path.isdir(in_dir):
-                out_dir = os.path.join(self.env['baremetal_build_dir'], subpath)
-                os.makedirs(out_dir, exist_ok=True)
-                common_objs_bootloader = common_objs.copy()
-                if os.path.basename(subpath) != 'no_bootloader':
-                    common_objs_bootloader.append(bootloader_obj)
-                for in_basename in sorted(os.listdir(in_dir)):
-                    in_path = os.path.join(in_dir, in_basename)
-                    in_name, in_ext = os.path.splitext(in_basename)
-                    if  (
-                        os.path.isfile(in_path) and
-                        in_ext in self.env['build_in_exts']
-                    ):
-                        out = os.path.join(out_dir, in_name + self.env['baremetal_build_ext'])
-                        src = os.path.join(self.env['baremetal_source_dir'], in_path)
-                        if self.need_rebuild(
-                            common_objs_bootloader +
-                            [
-                                src,
-                                self.env['baremetal_link_script'],
-                                self.env['common_h']
-                            ],
-                            out
-                        ):
-                            self.sh.run_cmd(
-                                [self.env['gcc_path'],  LF] +
-                                cflags +
-                                [
-                                    '-Wl,--section-start=.text={:#x}'.format(entry_address), LF,
-                                    '-o', out, LF,
-                                    '-T', self.env['baremetal_link_script'], LF,
-                                ] +
-                                [
-                                    src, LF,
-                                ] +
-                                self.sh.add_newlines(common_objs_bootloader) +
-                                cflags_after
-                            )
+            self._build_one(
+                in_path=in_path,
+                out_path=out_path,
+                cc_flags=cc_flags,
+                extra_deps=[self.env['common_h']],
+                link=False,
+            )
+        cc_flags.extend([
+            '-Wl,--section-start=.text={:#x}'.format(entry_address), LF,
+            '-T', self.env['baremetal_link_script'], LF,
+        ])
+        with thread_pool.ThreadPool(
+            self._build_one,
+            nthreads=self.env['nproc'],
+        ) as my_thread_pool:
+            try:
+                for target in self.env['targets']:
+                    for path, in_dirnames, in_filenames in self.sh.walk(target):
+                        for in_filename in in_filenames:
+                            in_ext = os.path.splitext(in_filename)[1]
+                            if not in_ext in self.env['build_in_exts']:
+                                continue
+                            in_path = os.path.join(path, in_filename)
+                            error = my_thread_pool.submit({
+                                'cc_flags': cc_flags,
+                                'extra_deps': [
+                                    self.env['baremetal_link_script'],
+                                    self.env['common_h']
+                                ],
+                                'extra_objs': [syscalls_obj],
+                                'extra_objs_baremetal_bootloader': [extra_obj_baremetal_bootloader],
+                                'extra_objs_lkmc_common': [extra_obj_lkmc_common],
+                                'in_path': in_path,
+                                'out_path': self.resolve_baremetal_executable(in_path),
+                            })
+                            if error is not None:
+                                raise common.ExitLoop()
+            except common.ExitLoop:
+                pass
+        error = my_thread_pool.get_error()
+        if error is not None:
+            print(error)
+            return 1
+
 
     def get_build_dir(self):
         return self.env['baremetal_build_dir']
 
+    def setup_one(self):
+        self.env['targets'] = self.resolve_targets(
+            self.env['baremetal_source_dir'],
+            self.env['targets']
+        )
+
 if __name__ == '__main__':
     Main().cli()

build-userland

@@ -16,6 +16,9 @@ class Main(common.BuildCliFunction):
 Build our compiled userland examples.
 '''
         super().__init__(*args, **kwargs)
+        self._add_argument('--ccflags')
+        self._add_argument('--force-rebuild')
+        self._add_argument('--optimization-level')
         self.add_argument(
             'targets',
             default=[],
@@ -37,9 +40,6 @@ Default: build all examples that have their package dependencies met, e.g.:
 ''',
             nargs='*',
         )
-        self._add_argument('--ccflags')
-        self._add_argument('--force-rebuild')
-        self._add_argument('--optimization-level')
 
     def build(self):
         build_dir = self.get_build_dir()
@@ -92,11 +92,11 @@ Default: build all examples that have their package dependencies met, e.g.:
                                 continue
                             in_path = os.path.join(path, in_filename)
                             error = my_thread_pool.submit({
-                                'in_path': in_path,
-                                'out_path': self.resolve_userland_executable(in_path),
                                 'cc_flags': cc_flags,
                                 'extra_objs_lkmc_common': [extra_obj_lkmc_common],
                                 'extra_objs_userland_asm': [extra_obj_userland_asm],
+                                'in_path': in_path,
+                                'out_path': self.resolve_userland_executable(in_path),
                             })
                             if error is not None:
                                 raise common.ExitLoop()

common.py

@@ -63,6 +63,7 @@ consts['userland_subdir'] = 'userland'
 consts['userland_source_dir'] = os.path.join(consts['root_dir'], consts['userland_subdir'])
 consts['userland_source_arch_dir'] = os.path.join(consts['userland_source_dir'], 'arch')
 consts['userland_executable_ext'] = '.out'
+consts['baremetal_executable_ext'] = '.elf'
 consts['include_subdir'] = consts['repo_short_id']
 consts['include_source_dir'] = os.path.join(consts['root_dir'], consts['include_subdir'])
 consts['submodules_dir'] = os.path.join(consts['root_dir'], 'submodules')
@@ -867,7 +868,6 @@ Incompatible archs are skipped.
             env['simulator_name'] = 'qemu'
         env['baremetal_build_dir'] = join(env['out_dir'], 'baremetal', env['arch'], env['simulator_name'], env['machine'])
         env['baremetal_build_lib_dir'] = join(env['baremetal_build_dir'], env['baremetal_lib_basename'])
-        env['baremetal_build_ext'] = '.elf'
 
         # Userland / baremetal common source.
         env['common_basename_noext'] = env['repo_short_id']
@@ -896,24 +896,16 @@ Incompatible archs are skipped.
         # Image
         if env['_args_given']['baremetal']:
             env['disk_image'] = env['gem5_fake_iso']
-            if env['baremetal'] == 'all':
-                path = env['baremetal']
-            else:
-                path = self.resolve_executable(
-                    env['baremetal'],
-                    env['baremetal_source_dir'],
-                    env['baremetal_build_dir'],
-                    env['baremetal_build_ext'],
-                )
-                source_path_noext = os.path.splitext(join(
-                    env['baremetal_source_dir'],
-                    os.path.relpath(path, env['baremetal_build_dir'])
-                ))[0]
-                for ext in [env['c_ext'], env['asm_ext']]:
-                    source_path = source_path_noext + ext
-                    if os.path.exists(source_path):
-                        env['source_path'] = source_path
-                        break
+            path = self.resolve_baremetal_executable(env['baremetal'])
+            source_path_noext = os.path.splitext(join(
+                env['baremetal_source_dir'],
+                os.path.relpath(path, env['baremetal_build_dir'])
+            ))[0]
+            for ext in [env['c_ext'], env['asm_ext']]:
+                source_path = source_path_noext + ext
+                if os.path.exists(source_path):
+                    env['source_path'] = source_path
+                    break
             env['image'] = path
         elif env['userland'] is not None:
             env['image'] = self.resolve_userland_executable(env['userland'])
@@ -1302,6 +1294,14 @@ lunch aosp_{}-eng
             new_targets.append(target)
         return new_targets
 
+    def resolve_baremetal_executable(self, path):
+        return self.resolve_executable(
+            path,
+            self.env['baremetal_source_dir'],
+            self.env['baremetal_build_dir'],
+            self.env['baremetal_executable_ext'],
+        )
+
     def resolve_userland_executable(self, path):
         return self.resolve_executable(
             path,
@@ -1389,8 +1389,10 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-debug-build
         build_exts=None,
         cc_flags=None,
         cc_flags_after=None,
+        extra_objs=None,
         extra_objs_userland_asm=None,
         extra_objs_lkmc_common=None,
+        extra_objs_baremetal_bootloader=None,
         extra_deps=None,
         link=True,
     ):
@@ -1418,11 +1420,15 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-debug-build
             in_basename
         ))
         if my_path_properties.should_be_built(self.env, link):
-            extra_objs= []
-            if my_path_properties['extra_objs_lkmc_common']:
-                extra_objs.extend(extra_objs_lkmc_common)
-            if my_path_properties['extra_objs_userland_asm']:
-                extra_objs.extend(extra_objs_userland_asm)
+            if extra_objs is None:
+                extra_objs= []
+            if link:
+                if my_path_properties['extra_objs_lkmc_common']:
+                    extra_objs.extend(extra_objs_lkmc_common)
+                if my_path_properties['extra_objs_userland_asm']:
+                    extra_objs.extend(extra_objs_userland_asm)
+                if my_path_properties['extra_objs_baremetal_bootloader']:
+                    extra_objs.extend(extra_objs_baremetal_bootloader)
             if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path):
                 cc_flags.extend(my_path_properties['cc_flags'])
                 cc_flags_after.extend(my_path_properties['cc_flags_after'])

lkmc.c

@@ -6,11 +6,6 @@
 
 #include <lkmc.h>
 
-void lkmc_assert(bool condition) {
-    if (!condition)
-        lkmc_assert_fail();
-}
-
 void lkmc_assert_fail(void) {
     exit(1);
 }

lkmc.h

@@ -4,12 +4,13 @@
 #define LKMC_H
 
 /* Common C definitions. */
+#define LKMC_UNUSED(x) (void)x
+
 #if !defined(__ASSEMBLER__)
 #include <inttypes.h>
 #include <stdbool.h>
 #include <stdlib.h>
 
-void lkmc_assert(bool);
 void lkmc_assert_fail();
 void lkmc_baremetal_on_exit_callback(int status, void *arg);
 #endif

lkmc/m5ops.h

@@ -3,7 +3,7 @@
 #ifndef LKMC_M5OPS_H
 #define LKMC_M5OPS_H
 
-#if LKMC_M5OPS_ENABLE == 1
+#if LKMC_M5OPS_ENABLE
 
 #if defined(__arm__)
 

path_properties.py

@@ -15,7 +15,6 @@ class PathProperties:
             '-Werror', LF,
             '-Wextra', LF,
             '-Wno-unused-function', LF,
-            '-fopenmp', LF,
             '-ggdb3', LF,
             # PIE causes the following problems:
             # * QEMU GDB step debug does not find breakpoints:
@@ -27,12 +26,13 @@ class PathProperties:
             '-fno-pie', LF,
             '-no-pie', LF,
         ],
-        'cc_flags_after': [],
+        'cc_flags_after': ['-lm'],
         'cc_pedantic': True,
         'cxx_std': default_cxx_std,
         # Expected program exit status. When signals are raised, this refers
         # to the native exit status. as reported by Bash #?.
         'exit_status': 0,
+        'extra_objs': [],
         'extra_objs_baremetal_bootloader': False,
         # We should get rid of this if we ever properly implement dependency graphs.
         'extra_objs_lkmc_common': False,
@@ -142,12 +142,17 @@ class PathProperties:
             )
         )
 
+    def _update_list(self, other_tmp_properties, key):
+        if key in self.properties and key in other_tmp_properties:
+            other_tmp_properties[key] = \
+                self.properties[key] + \
+                other_tmp_properties[key]
+
     def update(self, other):
         other_tmp_properties = other.properties.copy()
-        if 'cc_flags' in self.properties and 'cc_flags' in other_tmp_properties:
-            other_tmp_properties['cc_flags'] = \
-                self.properties['cc_flags'] + \
-                other_tmp_properties['cc_flags']
+        self._update_list(other_tmp_properties, 'cc_flags')
+        self._update_list(other_tmp_properties, 'cc_flags_after')
+        self._update_list(other_tmp_properties, 'extra_objs')
         if 'test_run_args' in self.properties and 'test_run_args' in other_tmp_properties:
             other_tmp_properties['test_run_args'] = {
                 **self.properties['test_run_args'],
@@ -213,7 +218,10 @@ path_properties_tuples = (
     PathProperties.default_properties,
     {
         'baremetal': (
-            {},
+            {
+                'extra_objs_baremetal_bootloader': True,
+                'extra_objs_lkmc_common': True,
+            },
             {
                 'arch': (
                     {},
@@ -256,10 +264,7 @@ path_properties_tuples = (
                 'lkmc_assert_fail.c': {'exit_status': 1},
                 'exit1.c': {'exit_status': 1},
                 'infinite_loop.c': {'more_than_1s': True},
-                'lib': (
-                    {'no_executable': True},
-                    {}
-                ),
+                'lib': {'no_executable': True},
                 'getchar.c': {'interactive': True},
                 'return1.c': {'exit_status': 1},
                 'return2.c': {'exit_status': 2},
@@ -267,8 +272,10 @@ path_properties_tuples = (
         ),
         'userland': (
             {
+                'cc_flags': [
+                    '-fopenmp', LF,
+                ],
                 'cc_flags_after': [
-                    '-lm', LF,
                     '-pthread', LF,
                 ],
             },

release-zip

@@ -23,7 +23,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip
         for root, dirnames, filenames in os.walk(self.env['baremetal_build_dir']):
             for filename in sorted(filenames):
                 path = os.path.join(root, filename)
-                if os.path.splitext(path)[1] == self.env['baremetal_build_ext']:
+                if os.path.splitext(path)[1] == self.env['baremetal_executable_ext']:
                     self.zip_files.append(path)
 
     def teardown(self):

run-gdb

@@ -146,7 +146,7 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-s
             break_at = []
         linux_full_system = (self.env['baremetal'] is None and self.env['userland'] is None)
         if self.env['userland']:
-            image = self.resolve_userland_executable(self.env['userland'])
+            image = self.env['image']
         elif self.env['baremetal']:
             image = self.env['image']
             test_script_path = os.path.splitext(self.env['source_path'])[0] + '.py'