From 4f4749148273c282e80b58c59db1b47049e190bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Tue, 30 Oct 2018 22:00:02 +0000 Subject: [PATCH] userland: make uber awesome with --baremetal-like executable resolution --- README.adoc | 117 ++++++++++++++++++++++++++++------------------ build-qemu | 4 +- build-userland | 4 +- common.py | 61 +++++++++++++++++------- run | 56 ++++++++++++---------- run-gdb | 12 ++--- userland/Makefile | 2 + 7 files changed, 160 insertions(+), 96 deletions(-) diff --git a/README.adoc b/README.adoc index 0778336..e964230 100644 --- a/README.adoc +++ b/README.adoc @@ -795,13 +795,17 @@ which will run respectively: which just make the emulator quit via <>. -Alternatively, for the sake of tab completion, we also accept full paths inside `baremetal`: +Alternatively, for the sake of tab completion, we also accept relative paths inside `baremetal/`: .... ./run --arch arm --baremetal baremetal/exit.c -./run --arch arm --baremetal "$(pwd)/baremetal/exit.c" ./run --arch arm --baremetal baremetal/arch/arm/semihost_exit.c -./run --arch arm --baremetal "$(pwd)/baremetal/arch/arm/semihost_exit.c" +.... + +Absolute paths however as used as is an must point to the actual executable: + +.... +./run --arch arm --baremetal "$(./getvar --arch arm baremetal_build_dir)/exit.elf" .... To use gem5 instead of QEMU do: @@ -1002,7 +1006,7 @@ This automatically clears the GDB pane, and starts a new one. Pass extra GDB arguments with: .... -./run --debug-guest --tmux --tmux-args start_kernel +./run --debug-guest --tmux=start_kernel .... See the tmux manual for further details: @@ -4665,7 +4669,7 @@ If `CONFIG_KALLSYMS=n`, then addresses are shown on traces instead of symbol plu In v4.16 it does not seem possible to configure that at runtime. GDB step debugging with: .... -./run --eval-busybox 'insmod /dump_stack.ko' --debug-guest --tmux --tmux-args dump_stack +./run --eval-busybox 'insmod /dump_stack.ko' --debug-guest --tmux=dump_stack .... shows that traces are printed at `arch/x86/kernel/dumpstack.c`: @@ -7650,49 +7654,54 @@ QEMU user mode completely bypasses the kernel that we've built: all it takes is === QEMU user mode getting started -You can run statically linked executables simply with: - -.... -./build-qemu --arch arm --user -./build-userland --arch arm --userland-build-id static --make-args='CCFLAGS_EXTRA=-static' -./run \ - --arch arm \ - --user "$(./getvar --arch arm --userland-build-id static userland_build_dir)/hello.out" \ -; -.... - -QEMU user mode also supports dynamically linked ones. - -We just have to point it to the root filesystem with the `-L` option so that it can find the libraries. - -Here we run: - -.... -ls . .. -.... - -with the ARM dynamically linked Buildroot `ls`: - .... +./build-userland --arch arm ./build-buildroot --arch arm ./run \ --arch arm \ - --user "$(./getvar --arch arm target_dir)/bin/ls" \ - --user-before="-L $(./getvar --arch arm target_dir)" \ + --userland print_argv \ -- \ - . .. \ + asdf qwer \ +; +.... + +This runs link:userland/print_argv.c[]. `--userland` path resolution is analogous to <>. + +QEMU user mode also supports dynamically linked executables. + +This requires point it to the root filesystem with the `-L` option so that it can find the dynamic linker and shared libraries. + +We pass `-L` by default, so everything just works: + +You can also try statically linked executables with: + +.... +./build-qemu --arch arm --userland +./build-userland \ + --arch arm \ + --make-args='CCFLAGS_EXTRA=-static' \ + --userland-build-id static \ +; +./run \ + --arch arm \ + --userland-build-id static \ + --userland print_argv \ + -- \ + asdf qwer \ ; .... ==== QEMU user mode GDB -It's nice when the <> works, right? +It's nice when <> just works, right? .... ./run \ --arch arm \ --debug-guest \ - --user "$(./getvar --arch arm --userland-build-id static userland_build_dir)/hello.out" \ + --userland print_argv \ + -- \ + asdf qwer \ ; .... @@ -7701,7 +7710,7 @@ and on another shell: .... ./run-gdb \ --arch arm \ - --user "$(./getvar --arch arm --userland-build-id static userland_build_dir)/hello.out" \ + --userland print_argv \ main \ ; .... @@ -7729,26 +7738,44 @@ fatal: Unable to open dynamic executable's interpreter. So let's just play with some static ones: .... -./build-userland --arch aarch64 --userland-build-id static --make-args='CCFLAGS_EXTRA=-static' -./run --arch aarch64 --gem5 --user "$(./getvar --arch aarch64 --userland-build-id static userland_build_dir)/hello.out" -.... - -CLI options may be passed as: - -.... +./build-userland \ + --arch aarch64 \ + --userland-build-id static \ + --make-args='CCFLAGS_EXTRA=-static' \ +; ./run \ --arch aarch64 \ --gem5 \ - --user "$(./getvar --arch aarch64 --userland-build-id static userland_build_dir)/print_argv.out" \ + --userland print_argv \ + --userland-build-id static \ -- \ --options 'asdf "qw er"' \ ; .... -Source: link:userland/print_argv.c[] - TODO: how to escape spaces? +Step debug also works: + +.... +./run \ + --arch arm \ + --debug-guest \ + --gem5 \ + --userland print_argv \ + --userland-build-id static \ + -- \ + --options 'asdf "qw er"' \ +; +./run-gdb \ + --arch arm \ + --gem5 \ + --userland print_argv \ + --userland-build-id static \ + main \ +; +.... + ==== User mode vs full system benchmark Let's see if user mode runs considerably faster than full system or not. @@ -7767,7 +7794,7 @@ time \ ./run \ --arch arm \ --gem5 \ - --user \ + --userland \ "$(./getvar --arch arm build_dir)/dhrystone-2/dhrystone" \ -- \ --options 100000 \ diff --git a/build-qemu b/build-qemu index 6eac992..3135204 100755 --- a/build-qemu +++ b/build-qemu @@ -7,7 +7,7 @@ import common class QemuComponent(common.Component): def add_parser_arguments(self, parser): parser.add_argument( - '--user', + '--userland', default=False, action='store_true', help='Build QEMU user mode instead of system.', @@ -26,7 +26,7 @@ class QemuComponent(common.Component): verbose = ['V=1'] else: verbose = [] - if args.user: + if args.userland: target_list = '{}-linux-user'.format(args.arch) else: target_list = '{}-softmmu'.format(args.arch) diff --git a/build-userland b/build-userland index f410e4c..f3a82d7 100755 --- a/build-userland +++ b/build-userland @@ -54,7 +54,7 @@ has the OpenBLAS libraries and headers installed. ] + ['HAS_{}=y'.format(package.upper()) for package in args.has_package] + shlex.split(args.make_args) + - [os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.executable_ext for target in args.targets] + [os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets] ), cwd=common.userland_src_dir, extra_paths=[common.ccache_dir], @@ -62,7 +62,7 @@ has the OpenBLAS libraries and headers installed. common.copy_dir_if_update_non_recursive( srcdir=build_dir, destdir=common.out_rootfs_overlay_dir, - filter_ext=common.executable_ext, + filter_ext=common.userland_build_ext, ) def get_argparse_args(self): diff --git a/common.py b/common.py index 43e2d53..eab4ae4 100644 --- a/common.py +++ b/common.py @@ -35,6 +35,7 @@ kernel_modules_subdir = 'kernel_modules' kernel_modules_src_dir = os.path.join(this_module.root_dir, this_module.kernel_modules_subdir) userland_subdir = 'userland' userland_src_dir = os.path.join(this_module.root_dir, this_module.userland_subdir) +userland_build_ext = '.out' include_subdir = 'include' include_src_dir = os.path.join(this_module.root_dir, this_module.include_subdir) submodules_dir = os.path.join(root_dir, 'submodules') @@ -69,7 +70,6 @@ c_ext = '.c' header_ext = '.h' kernel_module_ext = '.ko' obj_ext = '.o' -executable_ext = '.out' config_file = os.path.join(data_dir, 'config') command_prefix = '+ ' if os.path.exists(config_file): @@ -215,8 +215,15 @@ def get_argparse(default_args=None, argparse_args=None): help='CPU architecture. Default: %(default)s' ) parser.add_argument( - '--baremetal', - help='Use Baremetal examples instead of Linux kernel ones' + '-b', '--baremetal', + help='''\ +Use the given baremetal executable instead of the Linux kernel. + +If the path is absolute, it is used as is. + +If the path is relative, we assume that it points to a source code +inside baremetal/ and then try to use corresponding executable. +''' ) parser.add_argument( '--buildroot-build-id', @@ -307,7 +314,7 @@ Default: the run ID (-n) if that is an integer, otherwise 0. help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s' ) parser.add_argument( - '-t', '--gem5-build-type', default='opt', + '--gem5-build-type', default='opt', help='gem5 build type, most often used for "debug" builds. Default: %(default)s' ) parser.add_argument( @@ -854,27 +861,47 @@ def setup(parser): this_module.disk_image = this_module.qcow2_file else: this_module.disk_image = this_module.gem5_fake_iso - paths = [ - os.path.join(this_module.baremetal_build_dir, this_module.baremetal), - os.path.join( + if args.baremetal == 'all': + path = args.baremetal + else: + path = this_module.resolve_executable( + args.baremetal, + this_module.baremetal_src_dir, this_module.baremetal_build_dir, - os.path.relpath(this_module.baremetal, this_module.baremetal_src_dir), + this_module.baremetal_build_ext, ) - ] - paths[:] = [os.path.splitext(path)[0] + this_module.baremetal_build_ext for path in paths] - found = False - for path in paths: - if os.path.exists(path): - found = True - break - if not found and this_module.baremetal != 'all': - raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) this_module.image = path return args def setup_dry_run_arguments(args): this_module.dry_run = args.dry_run +def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext): + if os.path.isabs(in_path): + return in_path + else: + paths = [ + os.path.join(magic_out_dir, in_path), + os.path.join( + magic_out_dir, + os.path.relpath(in_path, magic_in_dir), + ) + ] + paths[:] = [os.path.splitext(path)[0] + out_ext for path in paths] + for path in paths: + if os.path.exists(path): + return path + raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths)) + +def resolve_userland(path): + global this_module + return this_module.resolve_executable( + path, + this_module.userland_src_dir, + this_module.userland_build_dir, + this_module.userland_build_ext, + ) + def write_configs(config_path, configs, config_fragments=None): """ Write extra configs into the Buildroot config file. diff --git a/run b/run index 79af610..474ee7c 100755 --- a/run +++ b/run @@ -32,11 +32,10 @@ defaults = { 'record': False, 'replay': False, 'terminal': False, - 'tmux': False, - 'tmux_args': '', + 'tmux': None, 'trace': None, - 'user': None, - 'user_before': '', + 'userland': None, + 'userland_before': '', 'vnc': False, } @@ -140,8 +139,8 @@ def main(args, extra_args=None): '-d', common.m5out_dir ] ) - if args.user is not None: - cmd.extend([common.gem5_se_file, '-c', args.user]) + if args.userland is not None: + cmd.extend([common.gem5_se_file, '-c', common.resolve_userland(args.userland)]) else: if args.gem5_script == 'fs': # TODO port @@ -196,7 +195,7 @@ def main(args, extra_args=None): qemu_user_and_system_options = [ '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), ] - if args.user is not None: + if args.userland is not None: if args.debug_guest: debug_args = ['-g', str(common.gdb_port)] else: @@ -204,12 +203,13 @@ def main(args, extra_args=None): cmd.extend( [ os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)), + '-L', common.target_dir, ] + qemu_user_and_system_options + - shlex.split(args.user_before) + + shlex.split(args.userland_before) + debug_args + [ - args.user + common.resolve_userland(args.userland) ] ) else: @@ -321,11 +321,11 @@ def main(args, extra_args=None): ) if args.baremetal is None: cmd.extend(append) - if args.tmux: + if args.tmux is not None: if args.gem5: subprocess.Popen([os.path.join(common.root_dir, 'tmu'), 'sleep 2;./gem5-shell -n {} {}' \ - .format(args.run_id, args.tmux_args) + .format(args.run_id, args.tmux) ]) elif args.debug_guest: # TODO find a nicer way to forward all those args automatically. @@ -333,8 +333,13 @@ def main(args, extra_args=None): # but it cannot be used as a library properly it seems, and it is # slower than tmux. subprocess.Popen([os.path.join(common.root_dir, 'tmu'), - "sleep 2;./run-gdb -a '{}' -L '{}' -n '{}' {}" \ - .format(args.arch, args.linux_build_id, args.run_id, args.tmux_args) + "sleep 2;./run-gdb --arch '{}' --linux-build-id '{}' --run-id '{}' {}" \ + .format( + args.arch, + args.linux_build_id, + args.run_id, + args.tmux + ) ]) cmd.extend(extra_emulator_args) cmd.extend(args.extra_emulator_args) @@ -486,31 +491,34 @@ gem5 Python scripts. ''' ) parser.add_argument( - '-U', '--tmux-args', default=defaults['tmux_args'], - help='Pass extra parameters to the program running on the `-u` tmux split' - ) - parser.add_argument( - '-u', '--tmux', default=defaults['tmux'], action='store_true', + '-t', '--tmux', default=defaults['tmux'], nargs='?', action='store', const='', help='''\ -Create a tmUx split the window. You must already be inside of a `tmux` session +Create a tmux split the window. You must already be inside of a `tmux` session to use this option: * on the main window, run the emulator as usual * on the split: ** if on QEMU and `-d` is given, GDB ** if on gem5, the gem5 terminal +If values are given to this option, pass those as parameters +to the program running on the split. ''' ) parser.add_argument( - '--user', default=defaults['user'], + '--userland', default=defaults['userland'], help='''\ -Run the given userland executable in user mode instead of full system mode. -In gem5, user mode is called Syscall Emulation (SE) mode and uses se.py. +Run the given userland executable in user mode instead of booting the Linux kernel +in full system mode. In gem5, user mode is called Syscall Emulation (SE) mode and +uses se.py. + +Path resolution is similar to --baremetal. ''' ) parser.add_argument( - '--user-before', default=defaults['user_before'], + '--userland-before', default=defaults['userland_before'], help='''\ -Arguments to pass to the QEMU user mode CLI before the program to execute. +Pass these Krguments to the QEMU user mode CLI before the program to execute. +This is required with --userland since arguments that come at the end are interpreted +as command line arguments to that executable. ''' ) parser.add_argument( diff --git a/run-gdb b/run-gdb index 9a4ea83..5b1e653 100755 --- a/run-gdb +++ b/run-gdb @@ -16,7 +16,7 @@ defaults = { 'no_continue': False, 'no_lxsymbols': False, 'sim': False, - 'user': None, + 'userland': None, } def main(args, extra_args=None): @@ -39,9 +39,9 @@ def main(args, extra_args=None): break_at = ['-ex', 'break {}'.format(args.break_at)] else: break_at = [] - linux_full_system = (args.baremetal is None and args.user is None) - if args.user: - image = args.user + linux_full_system = (args.baremetal is None and args.userland is None) + if args.userland: + image = common.resolve_userland(args.userland) elif args.baremetal: image = args.baremetal else: @@ -100,7 +100,7 @@ if __name__ == '__main__': help='Pass extra arguments to GDB, to be appended after all other arguments' ) parser.add_argument( - '-b', '--before', default=defaults['before'], + '--before', default=defaults['before'], help='Pass extra arguments to GDB to be prepended before any of the arguments passed by this script' ) parser.add_argument( @@ -120,7 +120,7 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-s '-X', '--no-lxsymbols', default=defaults['no_lxsymbols'], action='store_true' ) parser.add_argument( - '--user', default=defaults['user'], + '--userland', default=defaults['userland'], ) parser.add_argument( 'break_at', nargs='?', diff --git a/userland/Makefile b/userland/Makefile index 78f0683..41ab709 100644 --- a/userland/Makefile +++ b/userland/Makefile @@ -2,6 +2,8 @@ CFLAGS = -fopenmp -std=c99 $(CCFLAGS) $(CFLAGS_EXTRA) CXXFLAGS = -std=c++17 $(CCFLAGS) $(CXXFLAGS_EXTRA) +# -Wno-unused-function for function definitions on headers, +# because we are lazy to make a shared object. TODO. CCFLAGS = -ggdb3 -O0 -Wall -Werror -Wextra -Wno-unused-function $(CCFLAGS_EXTRA) IN_EXT_C = .c IN_EXT_CXX = .cpp