userland: make uber awesome with --baremetal-like executable resolution

README.adoc

@@ -795,13 +795,17 @@ which will run respectively:
 
 which just make the emulator quit via <<semihosting>>.
 
-Alternatively, for the sake of tab completion, we also accept full paths inside `baremetal`:
+Alternatively, for the sake of tab completion, we also accept relative paths inside `baremetal/`:
 
 ....
 ./run --arch arm --baremetal baremetal/exit.c
-./run --arch arm --baremetal "$(pwd)/baremetal/exit.c"
 ./run --arch arm --baremetal baremetal/arch/arm/semihost_exit.c
-./run --arch arm --baremetal "$(pwd)/baremetal/arch/arm/semihost_exit.c"
+....
+
+Absolute paths however as used as is an must point to the actual executable:
+
+....
+./run --arch arm --baremetal "$(./getvar --arch arm baremetal_build_dir)/exit.elf"
 ....
 
 To use gem5 instead of QEMU do:
@@ -1002,7 +1006,7 @@ This automatically clears the GDB pane, and starts a new one.
 Pass extra GDB arguments with:
 
 ....
-./run --debug-guest --tmux --tmux-args start_kernel
+./run --debug-guest --tmux=start_kernel
 ....
 
 See the tmux manual for further details:
@@ -4665,7 +4669,7 @@ If `CONFIG_KALLSYMS=n`, then addresses are shown on traces instead of symbol plu
 In v4.16 it does not seem possible to configure that at runtime. GDB step debugging with:
 
 ....
-./run --eval-busybox 'insmod /dump_stack.ko' --debug-guest --tmux --tmux-args dump_stack
+./run --eval-busybox 'insmod /dump_stack.ko' --debug-guest --tmux=dump_stack
 ....
 
 shows that traces are printed at `arch/x86/kernel/dumpstack.c`:
@@ -7650,49 +7654,54 @@ QEMU user mode completely bypasses the kernel that we've built: all it takes is
 
 === QEMU user mode getting started
 
-You can run statically linked executables simply with:
-
-....
-./build-qemu --arch arm --user
-./build-userland --arch arm --userland-build-id static --make-args='CCFLAGS_EXTRA=-static'
-./run \
-  --arch arm \
-  --user "$(./getvar --arch arm --userland-build-id static userland_build_dir)/hello.out" \
-;
-....
-
-QEMU user mode also supports dynamically linked ones.
-
-We just have to point it to the root filesystem with the `-L` option so that it can find the libraries.
-
-Here we run:
-
-....
-ls . ..
-....
-
-with the ARM dynamically linked Buildroot `ls`:
-
 ....
+./build-userland --arch arm
 ./build-buildroot --arch arm
 ./run \
   --arch arm \
-  --user "$(./getvar --arch arm target_dir)/bin/ls" \
+  --userland print_argv \
-  --user-before="-L $(./getvar --arch arm target_dir)" \
   -- \
-  . .. \
+  asdf qwer \
+;
+....
+
+This runs link:userland/print_argv.c[]. `--userland` path resolution is analogous to <<baremetal-setup-getting-started,that of `--baremetal`>>.
+
+QEMU user mode also supports dynamically linked executables.
+
+This requires point it to the root filesystem with the `-L` option so that it can find the dynamic linker and shared libraries.
+
+We pass `-L` by default, so everything just works:
+
+You can also try statically linked executables with:
+
+....
+./build-qemu --arch arm --userland
+./build-userland \
+  --arch arm \
+  --make-args='CCFLAGS_EXTRA=-static' \
+  --userland-build-id static \
+;
+./run \
+  --arch arm \
+  --userland-build-id static \
+  --userland print_argv \
+  -- \
+  asdf qwer \
 ;
 ....
 
 ==== QEMU user mode GDB
 
-It's nice when the <<gdb,obvious>> works, right?
+It's nice when <<gdb,the obvious>> just works, right?
 
 ....
 ./run \
   --arch arm \
   --debug-guest \
-  --user "$(./getvar --arch arm --userland-build-id static userland_build_dir)/hello.out" \
+  --userland print_argv \
+  -- \
+  asdf qwer \
 ;
 ....
 
@@ -7701,7 +7710,7 @@ and on another shell:
 ....
 ./run-gdb \
   --arch arm \
-  --user "$(./getvar --arch arm --userland-build-id static userland_build_dir)/hello.out" \
+  --userland print_argv \
   main \
 ;
 ....
@@ -7729,26 +7738,44 @@ fatal: Unable to open dynamic executable's interpreter.
 So let's just play with some static ones:
 
 ....
-./build-userland --arch aarch64 --userland-build-id static --make-args='CCFLAGS_EXTRA=-static'
+./build-userland \
-./run --arch aarch64 --gem5 --user "$(./getvar --arch aarch64 --userland-build-id static userland_build_dir)/hello.out"
+  --arch aarch64 \
-....
+  --userland-build-id static \
-
+  --make-args='CCFLAGS_EXTRA=-static' \
-CLI options may be passed as:
+;
-
-....
 ./run \
   --arch aarch64 \
   --gem5 \
-  --user "$(./getvar --arch aarch64 --userland-build-id static userland_build_dir)/print_argv.out" \
+  --userland print_argv \
+  --userland-build-id static \
   -- \
   --options 'asdf "qw er"' \
 ;
 ....
 
-Source: link:userland/print_argv.c[]
-
 TODO: how to escape spaces?
 
+Step debug also works:
+
+....
+./run \
+  --arch arm \
+  --debug-guest \
+  --gem5 \
+  --userland print_argv \
+  --userland-build-id static \
+  -- \
+  --options 'asdf "qw er"' \
+;
+./run-gdb \
+  --arch arm \
+  --gem5 \
+  --userland print_argv \
+  --userland-build-id static \
+  main \
+;
+....
+
 ==== User mode vs full system benchmark
 
 Let's see if user mode runs considerably faster than full system or not.
@@ -7767,7 +7794,7 @@ time \
   ./run \
   --arch arm \
   --gem5 \
-  --user \
+  --userland \
   "$(./getvar --arch arm build_dir)/dhrystone-2/dhrystone" \
   -- \
   --options 100000 \

build-qemu

@@ -7,7 +7,7 @@ import common
 class QemuComponent(common.Component):
     def add_parser_arguments(self, parser):
         parser.add_argument(
-            '--user',
+            '--userland',
             default=False,
             action='store_true',
             help='Build QEMU user mode instead of system.',
@@ -26,7 +26,7 @@ class QemuComponent(common.Component):
             verbose = ['V=1']
         else:
             verbose = []
-        if args.user:
+        if args.userland:
             target_list = '{}-linux-user'.format(args.arch)
         else:
             target_list = '{}-softmmu'.format(args.arch)

build-userland

@@ -54,7 +54,7 @@ has the OpenBLAS libraries and headers installed.
                 ] +
                 ['HAS_{}=y'.format(package.upper()) for package in args.has_package] +
                 shlex.split(args.make_args) +
-                [os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.executable_ext for target in args.targets]
+                [os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets]
             ),
             cwd=common.userland_src_dir,
             extra_paths=[common.ccache_dir],
@@ -62,7 +62,7 @@ has the OpenBLAS libraries and headers installed.
         common.copy_dir_if_update_non_recursive(
             srcdir=build_dir,
             destdir=common.out_rootfs_overlay_dir,
-            filter_ext=common.executable_ext,
+            filter_ext=common.userland_build_ext,
         )
 
     def get_argparse_args(self):

common.py

@@ -35,6 +35,7 @@ kernel_modules_subdir = 'kernel_modules'
 kernel_modules_src_dir = os.path.join(this_module.root_dir, this_module.kernel_modules_subdir)
 userland_subdir = 'userland'
 userland_src_dir = os.path.join(this_module.root_dir, this_module.userland_subdir)
+userland_build_ext = '.out'
 include_subdir = 'include'
 include_src_dir = os.path.join(this_module.root_dir, this_module.include_subdir)
 submodules_dir = os.path.join(root_dir, 'submodules')
@@ -69,7 +70,6 @@ c_ext = '.c'
 header_ext = '.h'
 kernel_module_ext = '.ko'
 obj_ext = '.o'
-executable_ext = '.out'
 config_file = os.path.join(data_dir, 'config')
 command_prefix = '+ '
 if os.path.exists(config_file):
@@ -215,8 +215,15 @@ def get_argparse(default_args=None, argparse_args=None):
         help='CPU architecture. Default: %(default)s'
     )
     parser.add_argument(
-        '--baremetal',
+        '-b', '--baremetal',
-        help='Use Baremetal examples instead of Linux kernel ones'
+        help='''\
+Use the given baremetal executable instead of the Linux kernel.
+
+If the path is absolute, it is used as is.
+
+If the path is relative, we assume that it points to a source code
+inside baremetal/ and then try to use corresponding executable.
+'''
     )
     parser.add_argument(
         '--buildroot-build-id',
@@ -307,7 +314,7 @@ Default: the run ID (-n) if that is an integer, otherwise 0.
         help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
     )
     parser.add_argument(
-        '-t', '--gem5-build-type', default='opt',
+        '--gem5-build-type', default='opt',
         help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
     )
     parser.add_argument(
@@ -854,27 +861,47 @@ def setup(parser):
             this_module.disk_image = this_module.qcow2_file
     else:
         this_module.disk_image = this_module.gem5_fake_iso
-        paths = [
+        if args.baremetal == 'all':
-            os.path.join(this_module.baremetal_build_dir, this_module.baremetal),
+            path = args.baremetal
-            os.path.join(
+        else:
+            path = this_module.resolve_executable(
+                args.baremetal,
+                this_module.baremetal_src_dir,
                 this_module.baremetal_build_dir,
-                os.path.relpath(this_module.baremetal, this_module.baremetal_src_dir),
+                this_module.baremetal_build_ext,
             )
-        ]
-        paths[:] = [os.path.splitext(path)[0] + this_module.baremetal_build_ext for path in paths]
-        found = False
-        for path in paths:
-            if os.path.exists(path):
-                found = True
-                break
-        if not found and this_module.baremetal != 'all':
-            raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
         this_module.image = path
     return args
 
 def setup_dry_run_arguments(args):
     this_module.dry_run = args.dry_run
 
+def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext):
+    if os.path.isabs(in_path):
+        return in_path
+    else:
+        paths = [
+            os.path.join(magic_out_dir, in_path),
+            os.path.join(
+                magic_out_dir,
+                os.path.relpath(in_path, magic_in_dir),
+            )
+        ]
+        paths[:] = [os.path.splitext(path)[0] + out_ext for path in paths]
+        for path in paths:
+            if os.path.exists(path):
+                return path
+        raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths))
+
+def resolve_userland(path):
+    global this_module
+    return this_module.resolve_executable(
+        path,
+        this_module.userland_src_dir,
+        this_module.userland_build_dir,
+        this_module.userland_build_ext,
+    )
+
 def write_configs(config_path, configs, config_fragments=None):
     """
     Write extra configs into the Buildroot config file.

run

@@ -32,11 +32,10 @@ defaults = {
     'record': False,
     'replay': False,
     'terminal': False,
-    'tmux': False,
+    'tmux': None,
-    'tmux_args': '',
     'trace': None,
-    'user': None,
+    'userland': None,
-    'user_before': '',
+    'userland_before': '',
     'vnc': False,
 }
 
@@ -140,8 +139,8 @@ def main(args, extra_args=None):
                 '-d', common.m5out_dir
             ]
         )
-        if args.user is not None:
+        if args.userland is not None:
-            cmd.extend([common.gem5_se_file, '-c', args.user])
+            cmd.extend([common.gem5_se_file, '-c', common.resolve_userland(args.userland)])
         else:
             if args.gem5_script == 'fs':
                 # TODO port
@@ -196,7 +195,7 @@ def main(args, extra_args=None):
         qemu_user_and_system_options = [
             '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file),
         ]
-        if args.user is not None:
+        if args.userland is not None:
             if args.debug_guest:
                 debug_args = ['-g', str(common.gdb_port)]
             else:
@@ -204,12 +203,13 @@ def main(args, extra_args=None):
             cmd.extend(
                 [
                     os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)),
+                    '-L', common.target_dir,
                 ] +
                 qemu_user_and_system_options +
-                shlex.split(args.user_before) +
+                shlex.split(args.userland_before) +
                 debug_args +
                 [
-                    args.user
+                    common.resolve_userland(args.userland)
                 ]
             )
         else:
@@ -321,11 +321,11 @@ def main(args, extra_args=None):
                 )
             if args.baremetal is None:
                 cmd.extend(append)
-    if args.tmux:
+    if args.tmux is not None:
         if args.gem5:
             subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
                 'sleep 2;./gem5-shell -n {} {}' \
-                .format(args.run_id, args.tmux_args)
+                .format(args.run_id, args.tmux)
             ])
         elif args.debug_guest:
             # TODO find a nicer way to forward all those args automatically.
@@ -333,8 +333,13 @@ def main(args, extra_args=None):
             # but it cannot be used as a library properly it seems, and it is
             # slower than tmux.
             subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
-                "sleep 2;./run-gdb -a '{}' -L '{}' -n '{}' {}" \
+                "sleep 2;./run-gdb --arch '{}' --linux-build-id '{}' --run-id '{}' {}" \
-                .format(args.arch, args.linux_build_id, args.run_id, args.tmux_args)
+                .format(
+                    args.arch,
+                    args.linux_build_id,
+                    args.run_id,
+                    args.tmux
+                )
             ])
     cmd.extend(extra_emulator_args)
     cmd.extend(args.extra_emulator_args)
@@ -486,31 +491,34 @@ gem5 Python scripts.
 '''
     )
     parser.add_argument(
-        '-U', '--tmux-args', default=defaults['tmux_args'],
+        '-t', '--tmux', default=defaults['tmux'], nargs='?', action='store', const='',
-        help='Pass extra parameters to the program running on the `-u` tmux split'
-    )
-    parser.add_argument(
-        '-u', '--tmux', default=defaults['tmux'], action='store_true',
         help='''\
-Create a tmUx split the window. You must already be inside of a `tmux` session
+Create a tmux split the window. You must already be inside of a `tmux` session
 to use this option:
 * on the main window, run the emulator as usual
 * on the split:
 ** if on QEMU and `-d` is given, GDB
 ** if on gem5, the gem5 terminal
+If values are given to this option, pass those as parameters
+to the program running on the split.
 '''
     )
     parser.add_argument(
-        '--user', default=defaults['user'],
+        '--userland', default=defaults['userland'],
         help='''\
-Run the given userland executable in user mode instead of full system mode.
+Run the given userland executable in user mode instead of booting the Linux kernel
-In gem5, user mode is called Syscall Emulation (SE) mode and uses se.py.
+in full system mode. In gem5, user mode is called Syscall Emulation (SE) mode and
+uses se.py.
+
+Path resolution is similar to --baremetal.
 '''
     )
     parser.add_argument(
-        '--user-before', default=defaults['user_before'],
+        '--userland-before', default=defaults['userland_before'],
         help='''\
-Arguments to pass to the QEMU user mode CLI before the program to execute.
+Pass these Krguments to the QEMU user mode CLI before the program to execute.
+This is required with --userland since arguments that come at the end are interpreted
+as command line arguments to that executable.
 '''
     )
     parser.add_argument(

run-gdb

@@ -16,7 +16,7 @@ defaults = {
     'no_continue': False,
     'no_lxsymbols': False,
     'sim': False,
-    'user': None,
+    'userland': None,
 }
 
 def main(args, extra_args=None):
@@ -39,9 +39,9 @@ def main(args, extra_args=None):
         break_at = ['-ex', 'break {}'.format(args.break_at)]
     else:
         break_at = []
-    linux_full_system = (args.baremetal is None and args.user is None)
+    linux_full_system = (args.baremetal is None and args.userland is None)
-    if args.user:
+    if args.userland:
-        image = args.user
+        image = common.resolve_userland(args.userland)
     elif args.baremetal:
         image = args.baremetal
     else:
@@ -100,7 +100,7 @@ if __name__ == '__main__':
         help='Pass extra arguments to GDB, to be appended after all other arguments'
     )
     parser.add_argument(
-        '-b', '--before', default=defaults['before'],
+        '--before', default=defaults['before'],
         help='Pass extra arguments to GDB to be prepended before any of the arguments passed by this script'
     )
     parser.add_argument(
@@ -120,7 +120,7 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-s
         '-X', '--no-lxsymbols', default=defaults['no_lxsymbols'], action='store_true'
     )
     parser.add_argument(
-        '--user', default=defaults['user'],
+        '--userland', default=defaults['userland'],
     )
     parser.add_argument(
         'break_at', nargs='?',

userland/Makefile

@@ -2,6 +2,8 @@
 
 CFLAGS = -fopenmp -std=c99 $(CCFLAGS) $(CFLAGS_EXTRA)
 CXXFLAGS = -std=c++17 $(CCFLAGS) $(CXXFLAGS_EXTRA)
+# -Wno-unused-function for function definitions on headers,
+#  because we are lazy to make a shared object. TODO.
 CCFLAGS = -ggdb3 -O0 -Wall -Werror -Wextra -Wno-unused-function $(CCFLAGS_EXTRA)
 IN_EXT_C = .c
 IN_EXT_CXX = .cpp