mtops.h is perfect now

.gitignore

@@ -1,6 +1,7 @@
 # Extensions and prefixes.
 *.tmp
 tmp.*
+*.tmp.*
 *~
 *.gitignore
 gitignore.*
@@ -22,6 +23,9 @@ __pycache__
 *.o
 *.out
 
+# Data to be plotted output.
+*.dat
+
 # Kernel modules.
 .cache.mk
 *.ko.cmd

README.adoc

@@ -1005,12 +1005,6 @@ cd linux-kernel-module-cheat
 ./build --download-dependencies userland-host
 ....
 
-If you modify a dependency that is not currently considered such as a header file, force the rebuild with:
-
-....
-./build --force-rebuild
-....
-
 The `build` scripts inside link:userland/[] are just symlinks to link:build-userland-in-tree[] which you can also use from toplevel as:
 
 ....
@@ -1021,6 +1015,22 @@ The `build` scripts inside link:userland/[] are just symlinks to link:build-user
 
 which is in turn just a thin wrapper around link:build-userland[], so you can use any option supported by that script freely.
 
+Pass custom compiler options with:
+
+....
+./build-userland --ccflags='-foptimize-sibling-calls -foptimize-strlen' --force-rebuild
+....
+
+Here we used `--force-rebuild` to force rebuild since the sources weren't modified since the last build.
+
+Some CLI options have more specialized flags, e.g. `-O` optimization level:
+
+....
+./build-userland --optimization-level 3 --force-rebuild
+....
+
+See also <<user-mode-static-executables>> for `--static`.
+
 Do a more clean out-of-tree build and run the program instead:
 
 ....
@@ -3407,13 +3417,14 @@ Let's run link:userland/c/print_argv.c[] built with the Buildroot toolchain on Q
 ./build user-mode-qemu
 ./run \
   --userland c/print_argv \
-  --userland-args 'asdf "qw er"' \
+  --userland-args='asdf "qw er"' \
 ;
 ....
 
 Output:
 
 ....
+/path/to/linux-kernel-module-cheat/out/userland/default/x86_64/c/print_argv.out
 asdf
 qw er
 ....
@@ -3428,33 +3439,72 @@ If you modify the userland programs, rebuild simply with:
 ./build-userland
 ....
 
-==== User mode GDB step debug
+==== User mode GDB
 
-The commands are analogous to full system <<gdb>>, e.g. without tmux:
+It's nice when <<gdb,the obvious>> just works, right?
 
 ....
 ./run \
+  --arch aarch64 \
   --userland c/print_argv \
   --userland-args 'asdf "qw er"' \
   --wait-gdb \
 ;
+....
+
+and on another shell:
+
+....
 ./run-gdb \
+  --arch aarch64 \
   --userland c/print_argv \
-  main
+  main \
 ;
 ....
 
-or with <<tmux>>:
+Or alternatively, if you are using <<tmux>>, do everything in one go with:
 
 ....
 ./run \
+  --arch aarch64 \
+  --tmux-args main \
   --userland c/print_argv \
   --userland-args 'asdf "qw er"' \
-  --tmux-args main \
   --wait-gdb \
 ;
 ....
 
+To stop at the very first instruction of a freestanding program, just use `--no-continue` TODO example.
+
+=== User mode tests
+
+Automatically run userland tests that can be run in user mode simulation, and check that they exit with status 0:
+
+....
+./build --all-archs test-user-mode
+./test-user-mode --all-archs --all-emulators
+....
+
+Or just for QEMU:
+
+....
+./build --all-archs test-user-mode-qemu
+./test-user-mode --all-archs --emulator qemu
+....
+
+Source: link:test-user-mode[]
+
+This script skips a manually configured list of tests, notably:
+
+* tests that depend on a full running kernel and cannot be run in user mode simulation, e.g. those that rely on kernel modules
+* tests that require user interaction
+* tests that take perceptible ammounts of time
+* known bugs we didn't have time to fix ;-)
+
+The gem5 tests require building statically with build id `static`, see also: <<gem5-syscall-emulation-mode>>. TODO automate this better.
+
+See: <<test-this-repo>> for more useful testing tips.
+
 ==== User mode with host toolchain and QEMU
 
 If you are lazy to built the Buildroot toolchain and QEMU, you can get away on Ubuntu 18.04 with just:
@@ -3484,13 +3534,15 @@ We must pass this to `./run` as well because QEMU must know which dynamic librar
 
 This present the usual trade-offs of using prebuilts as mentioned at: <<prebuilt>>.
 
+When you build with the native host toolchain, you can also execute many of the executables directly natively on the host: <<userland-setup-getting-started-natively>>.
+
 ==== User mode simulation with glibc
 
 At 125d14805f769104f93c510bedaa685a52ec025d we <<libc-choice,moved Buildroot from uClibc to glibc>>, and caused some user mode pain, which we document here.
 
 ===== FATAL: kernel too old
 
-Happens on all gem5 setups, but not on QEMU on Ubuntu 18.04 host.
+Happens on all gem5 <<user-mode-simulation>> setups, but not on QEMU on Ubuntu 18.04 host.
 
 glibc has a check for kernel version, likely obtained from the `uname` syscall, and if the kernel is not new enough, it quits.
 
@@ -3504,11 +3556,12 @@ We don't have this failure for QEMU, only gem5. QEMU by default copies the host
 
 Source: link:userland/posix/uname.c[].
 
+gem5 does not have such runtime configuration, but the error can be worked around for now by patching the hardcoded Linux version as mentioned at: https://stackoverflow.com/questions/48959349/how-to-solve-fatal-kernel-too-old-when-running-gem5-in-syscall-emulation-se-m to be a recent Linux version such as `v4.17.0`.
+
 The QEMU source that does this is at: https://github.com/qemu/qemu/blob/v3.1.0/linux-user/syscall.c#L8931
 
 In gem5, there are tons of missing syscalls, and that number currently just gets bumped up randomly from time to time when someone gets fed up:
 
-* https://stackoverflow.com/questions/48959349/how-to-solve-fatal-kernel-too-old-when-running-gem5-in-syscall-emulation-se-m
 * https://stackoverflow.com/questions/53085048/how-to-compile-and-run-an-executable-in-gem5-syscall-emulation-mode-with-se-py/53085049#53085049
 * https://gem5-review.googlesource.com/c/public/gem5/+/15855
 
@@ -3597,43 +3650,6 @@ it fails with:
 ld: cannot find -lopenblas
 ....
 
-==== User mode GDB
-
-It's nice when <<gdb,the obvious>> just works, right?
-
-....
-./run \
-  --arch aarch64 \
-  --userland c/print_argv \
-  --userland-args 'asdf "qw er"' \
-  --wait-gdb \
-;
-....
-
-and on another shell:
-
-....
-./run-gdb \
-  --arch aarch64 \
-  --userland c/print_argv \
-  main \
-;
-....
-
-Or alternatively, if you are using <<tmux>>, do everything in one go with:
-
-....
-./run \
-  --arch aarch64 \
-  --tmux-args main \
-  --userland c/print_argv \
-  --userland-args 'asdf "qw er"' \
-  --wait-gdb \
-;
-....
-
-To stop at the very first instruction of a freestanding program, just use `--no-continue` TODO example.
-
 === gem5 syscall emulation mode
 
 Less robust than QEMU's, but still usable:
@@ -3671,7 +3687,7 @@ So let's just play with some static ones:
 
 TODO: how to escape spaces on the command line arguments?
 
-<<user-mode-gdb-step-debug,GDB step debug>> also works normally on gem5:
+<<user-mode-gdb,GDB step debug>> also works normally on gem5:
 
 ....
 ./run \
@@ -3775,30 +3791,6 @@ Result on <<p51>> at bad30f513c46c1b0995d3a10c0d9bc2a33dc4fa0:
 * QEMU user: 45 seconds
 * QEMU full system: 223 seconds
 
-=== User mode tests
-
-Automatically run non-interactive userland tests that can be run in user mode simulation:
-
-....
-./build --all-archs test-user-mode
-./test-user-mode --all-archs --all-emulators
-....
-
-Or just for QEMU:
-
-....
-./build --all-archs test-user-mode-qemu
-./test-user-mode --all-archs --emulator qemu
-....
-
-Source: link:test-user-mode[]
-
-This testing excludes notably kernel module tests which depend on a full running kernel.
-
-The gem5 tests require building statically with build id `static`, see also: <<gem5-syscall-emulation-mode>>. TODO automate this better.
-
-See: <<test-this-repo>> for more useful testing tips.
-
 == Kernel module utilities
 
 === insmod
@@ -9908,21 +9900,30 @@ https://stackoverflow.com/questions/6147242/heap-vs-binary-search-tree-bst/29548
 Usage:
 
 ....
+./build-userland \
+  --arch aarch64 \
+  --ccflagg='-DLKMC_M5OPS_ENABLE=1' \
+  --force-build cpp/bst_vs_heap \
+  --static \
+;
 ./run \
   --arch aarch64 \
-  --eval-after './gem5.sh' \
   --emulator gem5 \
-  --gem5-readfile './bst_vs_heap.out' \
+  --static \
+  --userland cpp/bst_vs_heap \
 ;
-./bst-vs-heap --arch aarch64 --emulator gem5 > bst_vs_heap.dat
+./bst-vs-heap --arch aarch64 > bst_vs_heap.dat
+./bst-vs-heap.gnuplot
+xdg-open bst-vs-heap.tmp.png
 ....
 
-and then feed `bst_vs_heap.dat` into: https://github.com/cirosantilli/cpp-cheat/blob/9d0f77792fc8e55b20b6ee32018761ef3c5a3f2f/cpp/interactive/bst_vs_heap.gnuplot
-
 Sources:
 
+* link:userland/cpp/bst_vs_heap.cpp[]
 * link:bst-vs-heap[]
-* link:userland/bst_vs_heap.cpp[]
+* link:bst-vs-heap.gnuplot[]
+
+Tested on e70103b9b32e6e33dbab9eaf2ff00c358f55d8db + 1 with the workaround patch mentioned at: <<fatal-kernel-too-old>>.
 
 ===== BLAS
 
@@ -10585,29 +10586,44 @@ adsf
 
 ==== m5ops instructions
 
-The executable `./m5ops.out` illustrates how to hard code with inline assembly the m5ops that you are most likely to hack into the benchmark you are analysing:
+gem5 allocates some magic instructions on unused instruction encodings for convenient guest instrumentation.
+
+Those instructions are exposed through the <<m5>> in tree executable.
+
+To make things simpler to understand, you can play around with our own minimized educational `m5` subset link:userland/c/m5ops.c[].
+
+The instructions used by `./c/m5ops.out` are present in link:lkmc/m5ops.h[] in a very simple to understand and reuse inline assembly form.
+
+To use that file, first rebuild `m5ops.out` with the m5ops instructions enabled and install it on the root filesystem:
+
+....
+./build-userland \
+  --arch aarch64 \
+  --ccflags='-DLKMC_M5OPS_ENABLE=1' \
+  --force-build c/m5ops \
+  --static \
+;
+./build-buildroot --arch aarch64
+....
+
+We don't enable `-DLKMC_M5OPS_ENABLE=1` by default on userland executables because we try to use a single image for both gem5, QEMU and <<userland-setup-getting-started-natively,native>>, and those instructions would break the latter two. We enable it in the <<baremetal-setup>> by default since we already have different images for QEMU and gem5 there.
+
+Then, from inside <<gem5-buildroot-setup>>, test it out with:
 
 ....
 # checkpoint
-./m5ops.out c
+./c/m5ops.out c
 
 # dumpstats
-./m5ops.out d
+./c/m5ops.out d
 
 # exit
-./m5ops.out e
+./c/m5ops.out e
 
 # dump resetstats
-./m5ops.out r
+./c/m5ops.out r
 ....
 
-Sources:
-
-* link:userland/m5ops.h[]
-* link:userland/m5ops.c[]
-
-That executable is of course a subset of <<m5>> and useless by itself: its goal is only illustrate how to hardcode some <<m5ops>> yourself as one-liners.
-
 In theory, the cleanest way to add m5ops to your benchmarks would be to do exactly what the `m5` tool does:
 
 * include link:https://github.com/gem5/gem5/blob/05c4c2b566ce351ab217b2bd7035562aa7a76570/include/gem5/asm/generic/m5ops.h[`include/gem5/asm/generic/m5ops.h`]

baremetal/arch/aarch64/no_bootloader/gem5_exit.S

@@ -1,3 +1,5 @@
+#include <lkmc/m5ops.h>
+
 .global mystart
 mystart:
-    mov x0, #0; .inst 0XFF000110 | (0x21 << 16);
+    LKMC_M5OPS_EXIT_ASM

baremetal/arch/arm/gem5_assert.S

@@ -1,8 +1,10 @@
 /* assert 0x12345678 + 1 == 0x12345679 */
+
+#include <lkmc/m5ops.h>
+
 .global main
 main:
-    movw r0, #:lower16:myvar
+    adr r0, myvar
-    movt r0, #:upper16:myvar
     ldr r1, [r0]
     add r1, r1, #1
     str r1, [r0]
@@ -10,10 +12,8 @@ main:
     movt r2, #0x1234
     cmp r1, r2
     beq ok
-    /* m5 fail 1 */
+    LKMC_M5OPS_FAIL_1_ASM
-    mov r0, #0; mov r1, #0; mov r2, #1; mov r3, #0; .inst 0xEE000110 | (0x22 << 16);
 ok:
-    /* m5 exit */
+    LKMC_M5OPS_EXIT_ASM
-    mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);
 myvar:
     .word 0x12345678

baremetal/lib/syscalls.c

@@ -2,6 +2,8 @@
 #include <stdlib.h>
 #include <sys/stat.h>
 
+#include <lkmc/m5ops.h>
+
 enum {
     UART_FR_RXFE = 0x10,
 };
@@ -63,11 +65,7 @@ int _write(int file, char *ptr, int len) {
 /* Only 0 is supported for now, arm semihosting cannot handle other values. */
 void _exit(int status) {
 #if defined(GEM5)
-#if defined(__arm__)
+    LKMC_M5OPS_EXIT;
-    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);" : : : "r0", "r1");
-#elif defined(__aarch64__)
-    __asm__ __volatile__ ("mov x0, #0; .inst 0XFF000110 | (0x21 << 16);" : : : "x0");
-#endif
 #else
 #if defined(__arm__)
     __asm__ __volatile__ ("mov r0, #0x18; ldr r1, =#0x20026; svc 0x00123456" : : : "r0", "r1");

bst-vs-heap

@@ -1,17 +1,31 @@
 #!/usr/bin/env python3
+
 import common
-parser = self.get_argparse(
+
-    argparse_args={'description':'Convert a BST vs heap stat file into a gnuplot input'}
+class Main(common.LkmcCliFunction):
-)
+    def __init__(self):
-args = self.setup(parser)
+        super().__init__(
-stats = self.get_stats()
+            defaults={
-it = iter(stats)
+                'emulator': 'gem5',
-i = 1
+                'print_time': False,
-for stat in it:
+            },
-    try:
+            description='''\
-        next_stat = next(it)
+Convert a BST vs heap stat file into a gnuplot input
-    except StopIteration:
+''',
-        # Automatic dumpstats at end may lead to odd number of stats.
+        )
-        break
+
-    print('{} {} {}'.format(i, stat, next_stat))
+    def timed_main(self):
-    i += 1
+        stats = self.get_stats()
+        it = iter(stats)
+        i = 1
+        for stat in it:
+            try:
+                next_stat = next(it)
+            except StopIteration:
+                # Automatic dumpstats at end may lead to odd number of stats.
+                break
+            print('{} {} {}'.format(i, stat, next_stat))
+            i += 1
+
+if __name__ == '__main__':
+    Main().cli()

bst-vs-heap.gnuplot

@@ -0,0 +1,25 @@
+#!/usr/bin/env gnuplot
+set terminal png size 1024, 2048
+set output "bst-vs-heap.tmp.png"
+set multiplot layout 5,1 title "Heap vs BST vs Hash map insert time"
+set xlabel "size"
+set ylabel "nanoseconds"
+
+set title "Heap"
+plot "bst_vs_heap.dat" using 1:2 notitle
+
+set title "Heap (zoom)"
+set yrange [0:25]
+plot "bst_vs_heap.dat" using 1:2 notitle
+
+set title "BST"
+set yrange [*:*]
+plot "bst_vs_heap.dat" using 1:3 notitle
+
+#set title "Hash map"
+#set yrange [*:*]
+#plot "bst_vs_heap.dat" using 1:4 notitle
+#
+#set title "Hash map zoom"
+#set yrange [0:350]
+#plot "bst_vs_heap.dat" using 1:4 notitle

build-baremetal

@@ -15,6 +15,9 @@ Build the baremetal examples with crosstool-NG.
 ''',
             supported_archs=common.consts['crosstool_ng_supported_archs']
         )
+        self._add_argument('--ccflags')
+        self._add_argument('--force-rebuild')
+        self._add_argument('--optimization-level')
 
     def build(self):
         build_dir = self.get_build_dir()
@@ -39,7 +42,7 @@ Build the baremetal examples with crosstool-NG.
         cflags = [
             '-I', self.env['baremetal_source_lib_dir'], LF,
             '-I', self.env['root_dir'], LF,
-            '-O0', LF,
+            '-O{}'.format(self.env['optimization_level']), LF,
             '-ggdb3', LF,
             '-mcpu={}'.format(self.env['mcpu']), LF,
             '-nostartfiles', LF,
@@ -54,7 +57,10 @@ Build the baremetal examples with crosstool-NG.
                 uart_address = 0x10009000
             else:
                 raise Exception('unknown machine: ' + self.env['machine'])
-            cflags.extend(['-D', 'GEM5'.format(uart_address), LF])
+            cflags.extend([
+                '-D', 'GEM5'.format(uart_address), LF,
+                '-DLKMC_M5OPS_ENABLE=1', LF,
+            ])
         else:
             entry_address = 0x40000000
             uart_address = 0x09000000
@@ -67,6 +73,7 @@ Build the baremetal examples with crosstool-NG.
                 self.env['asm_ext']
             )
         )
+        cflags.extend(self.sh.shlex_split(self.env['ccflags']))
         if self.need_rebuild([src], bootloader_obj):
             self.sh.run_cmd(
                 [self.env['gcc'],  LF] +

build-buildroot

@@ -70,6 +70,7 @@ Extra arguments to be passed to the Buildroot make,
 usually extra Buildroot targets.
 '''
         )
+        self._add_argument('--force-rebuild')
 
     def build(self):
         build_dir = self.get_build_dir()
@@ -147,6 +148,8 @@ usually extra Buildroot targets.
             cwd=self.env['buildroot_source_dir'],
         )
         self.make_build_dirs()
+        if self.env['force_rebuild']:
+            extra_make_args.extend(['-B', LF])
         if not self.env['no_all']:
             extra_make_args.extend(['all', LF])
         self.sh.run_cmd(

build-linux

@@ -81,6 +81,7 @@ Run `make modules_install` after `make`.
             metavar='extra-make-args',
             nargs='*'
         )
+        self._add_argument('--force-rebuild')
 
     def build(self):
         build_dir = self.get_build_dir()
@@ -108,6 +109,8 @@ Run `make modules_install` after `make`.
             'CC={}'.format(cc), LF,
             'O={}'.format(build_dir), LF,
         ] + verbose
+        if self.env['force_rebuild']:
+            common_make_args.extend(['-B', LF])
         if self.env['configure']:
             if self.env['custom_config_target']:
                 base_config_given = True

build-modules

@@ -20,6 +20,9 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host
         self.add_argument(
             '--make-args',
             default='',
+            help='''
+Pass custom options to make.
+''',
         )
         self.add_argument(
             '--host',
@@ -35,6 +38,7 @@ Place the modules on a separate magic directory from non --host builds.
             help='Which kernel modules to build. Default: build all',
             nargs='*',
         )
+        self._add_argument('--force-rebuild')
 
     def build(self):
         build_dir = self.get_build_dir()

build-userland

@@ -63,6 +63,9 @@ Default: build all examples that have their package dependencies met, e.g.:
 ''',
             nargs='*',
         )
+        self._add_argument('--ccflags')
+        self._add_argument('--force-rebuild')
+        self._add_argument('--optimization-level')
 
     def _build_one(
         self,
@@ -156,13 +159,13 @@ Default: build all examples that have their package dependencies met, e.g.:
         has_all_packages = self.env['has_all_packages']
         ccflags = [
             '-I', self.env['root_dir'], LF,
-            '-O0', LF,
+            '-O{}'.format(self.env['optimization_level']), LF,
             '-Wall', LF,
             '-Werror', LF,
             '-Wextra', LF,
             '-Wno-unused-function', LF,
             '-ggdb3', LF,
-        ]
+        ] + self.sh.shlex_split(self.env['ccflags'])
         if self.env['static']:
             ccflags.extend(['-static', LF])
         common_obj = os.path.join(
@@ -335,13 +338,13 @@ Default: build all examples that have their package dependencies met, e.g.:
                             in_name + self.env['userland_build_ext']
                         )
                         error = thread_pool.submit({
-                                'in_path': in_path,
+                            'in_path': in_path,
-                                'out_path': out_path,
+                            'out_path': out_path,
-                                'ccflags': ccflags_dir,
+                            'ccflags': ccflags_dir,
-                                'cstd': cstd,
+                            'cstd': cstd,
-                                'cxxstd': cxxstd,
+                            'cxxstd': cxxstd,
-                                'extra_objs': common_objs_dir,
+                            'extra_objs': common_objs_dir,
-                                'ccflags_after': ccflags_after,
+                            'ccflags_after': ccflags_after,
                         })
                         if error is not None:
                             raise ExitLoop()

build-userland-in-tree

@@ -18,8 +18,7 @@ class Main(build_userland.Main):
     def __init__(self):
         super().__init__(
             description='''\
-Same as build-userland, but with pre-set defaults to build in-tree
+https://github.com/cirosantilli/linux-kernel-module-cheat#userland-setup-getting-started-natively
-for the native toolchain.
 ''',
             defaults={
                 'gcc_which': 'host',

common.py

@@ -1240,14 +1240,6 @@ class BuildCliFunction(LkmcCliFunction):
             default=False,
             help='Clean the build instead of building.',
         ),
-        self.add_argument(
-            '--force-rebuild',
-            default=False,
-            help='''\
-Force rebuild even if sources didn't chage.
-TODO: not yet implemented on all scripts.
-'''
-        )
         self.add_argument(
             '-j',
             '--nproc',
@@ -1256,6 +1248,36 @@ TODO: not yet implemented on all scripts.
             help='Number of processors to use for the build.',
         )
         self.test_results = []
+        self._build_arguments = {
+            '--ccflags': {
+                'default': '',
+                'help': '''\
+Pass the given compiler flags to all languages (C, C++, Fortran, etc.)
+''',
+            },
+            '--force-rebuild': {
+                'default': False,
+                "help": '''\
+Force rebuild even if sources didn't change.
+''',
+            },
+            '--optimization-level': {
+                'default': '0',
+                'help': '''
+Use the given GCC -O optimization level.
+For some scripts, there are hard technical challenges why it cannot
+be implemented, e.g.: https://github.com/cirosantilli/linux-kernel-module-cheat#kernel-o0
+and for others such as gem5 have their custom mechanism:
+https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-debug-build
+''',
+            }
+        }
+
+    def _add_argument(self, argument_name):
+        self.add_argument(
+            argument_name,
+            **self._build_arguments[argument_name]
+        )
 
     def clean(self):
         build_dir = self.get_build_dir()

lkmc/m5ops.h

@@ -0,0 +1,56 @@
+/* https://github.com/cirosantilli/linux-kernel-module-cheat#m5ops-instructions */
+
+#ifndef LKMC_M5OPS_H
+#define LKMC_M5OPS_H
+
+#if LKMC_M5OPS_ENABLE == 1
+
+#if defined(__arm__)
+
+#define LKMC_M5OPS_CHECKPOINT_ASM mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x43 << 16)
+#define LKMC_M5OPS_DUMPSTATS_ASM  mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x41 << 16)
+#define LKMC_M5OPS_EXIT_ASM       mov r0, #0; mov r1, #0;                         .inst 0xEE000110 | (0x21 << 16)
+#define LKMC_M5OPS_FAIL_1_ASM     mov r0, #0; mov r1, #0; mov r2, #1; mov r3, #0; .inst 0xEE000110 | (0x22 << 16)
+#define LKMC_M5OPS_RESETSTATS_ASM mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x40 << 16)
+
+#define LKMC_M5OPS_CHECKPOINT __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x43 << 16);" : : : "r0", "r1", "r2", "r3")
+#define LKMC_M5OPS_DUMPSTATS  __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x41 << 16);" : : : "r0", "r1", "r2", "r3")
+#define LKMC_M5OPS_EXIT       __asm__ __volatile__ ("mov r0, #0; mov r1, #0;                         .inst 0xEE000110 | (0x21 << 16);" : : : "r0", "r1"            )
+#define LKMC_M5OPS_FAIL_1     __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #1; mov r3, #0; .inst 0xEE000110 | (0x22 << 16);" : : : "r0", "r1", "r2", "r3")
+#define LKMC_M5OPS_RESETSTATS __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x40 << 16);" : : : "r0", "r1", "r2", "r3")
+
+#elif defined(__aarch64__)
+
+#define LKMC_M5OPS_CHECKPOINT_ASM mov x0, #0; mov x1, #0; .inst 0xFF000110 | (0x43 << 16);
+#define LKMC_M5OPS_DUMPSTATS_ASM  mov x0, #0; mov x1, #0; .inst 0xFF000110 | (0x41 << 16);
+#define LKMC_M5OPS_EXIT_ASM       mov x0, #0;             .inst 0XFF000110 | (0x21 << 16);
+#define LKMC_M5OPS_FAIL_1_ASM     mov x0, #0; mov x1, #1; .inst 0xFF000110 | (0x22 << 16);
+#define LKMC_M5OPS_RESETSTATS_ASM mov x0, #0; mov x1, #0; .inst 0XFF000110 | (0x40 << 16);
+
+#define LKMC_M5OPS_CHECKPOINT __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0xFF000110 | (0x43 << 16);" : : : "x0", "x1")
+#define LKMC_M5OPS_DUMPSTATS  __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0xFF000110 | (0x41 << 16);" : : : "x0", "x1")
+#define LKMC_M5OPS_EXIT       __asm__ __volatile__ ("mov x0, #0;             .inst 0XFF000110 | (0x21 << 16);" : : : "x0"      )
+#define LKMC_M5OPS_FAIL_1     __asm__ __volatile__ ("mov x0, #0; mov x1, #1; .inst 0xFF000110 | (0x22 << 16);" : : : "x0", "x1")
+#define LKMC_M5OPS_RESETSTATS __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0XFF000110 | (0x40 << 16);" : : : "x0", "x1")
+
+#else
+#error m5ops not implemented for the current arch
+#endif
+
+#else
+
+#define LKMC_M5OPS_CHECKPOINT_ASM
+#define LKMC_M5OPS_DUMPSTATS_ASM
+#define LKMC_M5OPS_EXIT_ASM
+#define LKMC_M5OPS_FAIL_1_ASM
+#define LKMC_M5OPS_RESETSTATS_ASM
+
+#define LKMC_M5OPS_CHECKPOINT
+#define LKMC_M5OPS_DUMPSTATS
+#define LKMC_M5OPS_EXIT
+#define LKMC_M5OPS_FAIL_1
+#define LKMC_M5OPS_RESETSTATS
+
+#endif
+
+#endif

lkmc/ring0.h

@@ -2,9 +2,7 @@
 
 #ifndef LKMC_RING0_H
 #define LKMC_RING0_H
-
 #if defined(__x86_64__) || defined(__i386__)
-
 #ifdef THIS_MODULE
 #include <linux/kernel.h>
 #if defined(__x86_64__)
@@ -74,5 +72,5 @@ void lkmc_ring0_get_control_regs(LkmcRing0Regs *ring0_regs) {
     );
 #endif
 }
-
+#endif
 #endif

run

@@ -592,10 +592,11 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
                 # Part of me wants to: https://github.com/jonathanslenders/pymux
                 # but it cannot be used as a library properly it seems, and it is
                 # slower than tmux.
-                tmux_args += " --arch {} --linux-build-id '{}' --run-id '{}'".format(
+                tmux_args += " --arch {} --linux-build-id '{}' --run-id '{}' --userland-build-id '{}'".format(
                     self.env['arch'],
                     self.env['linux_build_id'],
                     self.env['run_id'],
+                    self.env['userland_build_id'],
                 )
                 if self.env['baremetal']:
                     tmux_args += " --baremetal '{}'".format(self.env['baremetal'])

userland/arch/arm/adr.S

@@ -11,7 +11,7 @@ ENTRY
      * it does not get converted to adr, which is the better
      * alternative here.
      */
-    adr r1, label
+    ldr r1, =label
     adrl r2, label
 label:
     ASSERT_EQ_REG(r0, r1)

userland/c/m5ops.c

@@ -4,7 +4,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "m5ops.h"
+#include <lkmc/m5ops.h>
 
 int main(int argc, char **argv) {
     char action;
@@ -15,19 +15,19 @@ int main(int argc, char **argv) {
     }
     switch (action) {
         case 'c':
-            m5_checkpoint();
+            LKMC_M5OPS_CHECKPOINT;
             break;
         case 'd':
-            m5_dumpstats();
+            LKMC_M5OPS_DUMPSTATS;
             break;
         case 'e':
-            m5_exit();
+            LKMC_M5OPS_EXIT;
             break;
         case 'f':
-            m5_fail_1();
+            LKMC_M5OPS_FAIL_1;
             break;
         case 'r':
-            m5_resetstats();
+            LKMC_M5OPS_RESETSTATS;
             break;
     }
     return EXIT_SUCCESS;

userland/cpp/bst_vs_heap.cpp

@@ -6,7 +6,7 @@
 #include <random>
 #include <set>
 
-#include "m5ops.h"
+#include <lkmc/m5ops.h>
 
 int main(int argc, char **argv) {
     typedef uint64_t I;
@@ -32,13 +32,13 @@ int main(int argc, char **argv) {
         auto random = randoms[i];
 
         // Heap.
-        m5_resetstats();
+        LKMC_M5OPS_RESETSTATS;
         heap.emplace(random);
-        m5_dumpstats();
+        LKMC_M5OPS_DUMPSTATS;
 
         // BST.
-        m5_resetstats();
+        LKMC_M5OPS_RESETSTATS;
         bst.insert(random);
-        m5_dumpstats();
+        LKMC_M5OPS_DUMPSTATS;
     }
 }

userland/m5ops.h

@@ -1,44 +0,0 @@
-#ifndef M5OPS_H
-#define M5OPS_H
-
-#if defined(__arm__)
-static void m5_checkpoint(void) {
-    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x43 << 16);" : : : "r0", "r1", "r2", "r3");
-}
-static void m5_dumpstats(void) {
-    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x41 << 16);" : : : "r0", "r1", "r2", "r3");
-}
-static void m5_exit() {
-    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);" : : : "r0", "r1");
-}
-static void m5_fail_1(void) {
-    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #1; mov r3, #0; .inst 0xEE000110 | (0x22 << 16);" : : : "r0", "r1", "r2", "r3");
-}
-static void m5_resetstats(void) {
-    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x40 << 16);" : : : "r0", "r1", "r2", "r3");
-}
-#elif defined(__aarch64__)
-static void m5_checkpoint(void) {
-    __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0xFF000110 | (0x43 << 16);" : : : "x0", "x1");
-}
-static void m5_dumpstats(void) {
-    __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0xFF000110 | (0x41 << 16);" : : : "x0", "x1");
-}
-static void m5_exit(void) {
-    __asm__ __volatile__ ("mov x0, #0; .inst 0XFF000110 | (0x21 << 16);" : : : "x0");
-}
-static void m5_fail_1(void) {
-    __asm__ __volatile__ ("mov x0, #0; mov x1, #1; .inst 0xFF000110 | (0x22 << 16);" : : : "x0", "x1");
-}
-static void m5_resetstats(void) {
-    __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0XFF000110 | (0x40 << 16);" : : : "x0", "x1");
-}
-#else
-static void m5_checkpoint(void) {}
-static void m5_dumpstats(void) {}
-static void m5_exit(void) {}
-static void m5_fail_1(void) {}
-static void m5_resetstats(void) {}
-#endif
-
-#endif