From a5ec63dc2898a71b3e4afb854a5f946e94bb0309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Tue, 22 Jan 2019 00:00:00 +0000 Subject: [PATCH] CliFunction --- README.adoc | 47 +- arm | 3 - bench-all | 6 +- bisect-linux-boot-gem5 | 10 +- bst-vs-heap | 6 +- build | 95 +- build-baremetal | 148 ++-- build-buildroot | 168 ++-- build-crosstool-ng | 83 +- build-docker | 35 +- build-gem5 | 111 ++- build-linux | 123 ++- build-m5 | 55 +- build-modules | 110 ++- build-qemu | 55 +- build-userland | 81 +- cli_function.py | 262 ++++++ cli_function_test_config.py | 5 + common.py | 1673 +++++++++++++++-------------------- copy-overlay | 29 +- gem5-shell | 13 +- gem5-stat | 6 +- getvar | 13 +- qemu-monitor | 11 +- qemu-trace2txt | 17 +- release | 14 +- release-download-latest | 3 +- release-upload | 17 +- release-zip | 17 +- run | 1096 +++++++++++------------ run-docker | 49 +- run-gdb | 302 +++---- run-gdb-user | 12 +- run-gdbserver | 15 +- run-toolchain | 24 +- shell_helpers.py | 262 ++++++ trace-boot | 16 +- trace2line | 29 +- userland/Makefile | 8 +- 39 files changed, 2630 insertions(+), 2399 deletions(-) delete mode 100755 arm create mode 100755 cli_function.py create mode 100644 cli_function_test_config.py create mode 100644 shell_helpers.py diff --git a/README.adoc b/README.adoc index 60ea605..ffcd565 100644 --- a/README.adoc +++ b/README.adoc @@ -830,6 +830,14 @@ For more information on baremetal, see the section: <>. The following * <> * <> +=== User mode setup + +Much like <>, this is another fun setup that does not require Buildroot or the Linux kernel. + +See: <> + +TODO: test it out on a clean repo. + [[gdb]] == GDB step debug @@ -960,7 +968,7 @@ This automatically clears the GDB pane, and starts a new one. Pass extra GDB arguments with: .... -./run --wait-gdb --tmux=start_kernel +./run --wait-gdb --tmux --tmux-args start_kernel .... See the tmux manual for further details: @@ -2986,7 +2994,8 @@ Or alternatively, if you are using <>, do everything in one go with: ./run \ --arch aarch64 \ --userland print_argv \ - --tmux=main \ + --tmux \ + --tmux-args main \ --wait-gdb \ -- \ asdf qwer \ @@ -4106,6 +4115,12 @@ That file contains `BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="board/qemu/x86_64/linux `arm`, on the other hand, uses link:https://github.com/buildroot/buildroot/blob/2018.05/configs/qemu_arm_vexpress_defconfig[`buildroot/configs/qemu_arm_vexpress_defconfig`], which contains `BR2_LINUX_KERNEL_DEFCONFIG="vexpress"`, and therefore just does a `make vexpress_defconfig`, and gets its config from the Linux kernel tree itself. +====== Linux kernel defconfigs + +It would be interesting to test out if `make defconfig` configs boot and work on QEMU + Buildroot: https://unix.stackexchange.com/questions/29439/compiling-the-kernel-with-default-configurations/204512#204512 + +TODO. + ===== Notable alternate gem5 kernel configs Other configs which we had previously tested at 4e0d9af81fcce2ce4e777cb82a1990d7c2ca7c1e are: @@ -5074,7 +5089,7 @@ If `CONFIG_KALLSYMS=n`, then addresses are shown on traces instead of symbol plu In v4.16 it does not seem possible to configure that at runtime. GDB step debugging with: .... -./run --eval-after 'insmod /dump_stack.ko' --wait-gdb --tmux=dump_stack +./run --eval-after 'insmod /dump_stack.ko' --wait-gdb --tmux --tmux-args dump_stack .... shows that traces are printed at `arch/x86/kernel/dumpstack.c`: @@ -8206,7 +8221,7 @@ And in QEMU: Or for a faster development loop: .... -./run --debug-vm='-ex "break edu_mmio_read" -ex "run"' +./run --debug-vm --debug-vm-args '-ex "break edu_mmio_read" -ex "run"' .... When in <>, using `--debug-vm` makes Ctrl-C not get passed to the QEMU guest anymore: it is instead captured by GDB itself, so allow breaking. So e.g. you won't be able to easily quit from a guest program like: @@ -8529,8 +8544,8 @@ List all available debug flags: but to understand most of them you have to look at the source code: .... -less "$(./getvar gem5_src_dir)/src/cpu/SConscript" -less "$(./getvar gem5_src_dir)/src/cpu/exetrace.cc" +less "$(./getvar gem5_source_dir)/src/cpu/SConscript" +less "$(./getvar gem5_source_dir)/src/cpu/exetrace.cc" .... The traces are generated from `DPRINTF(` calls scattered throughout the code. @@ -9971,7 +9986,7 @@ The `--gem5-script biglittle` option enables the alternative `configs/example/ar First apply: .... -patch -d "$(./getvar gem5_src_dir)" -p 1 < patches/manual/gem5-biglittle.patch +patch -d "$(./getvar gem5_source_dir)" -p 1 < patches/manual/gem5-biglittle.patch .... then: @@ -10327,13 +10342,13 @@ then on the second shell: Or if you are a <>, do everything in one go with: .... -./run --arch arm --baremetal interactive/prompt --wait-gdb --tmux=main +./run --arch arm --baremetal interactive/prompt --wait-gdb --tmux --tmux-args main .... Alternatively, to start from the very first executed instruction of our tiny <>: .... -./run --arch arm --baremetal interactive/prompt --wait-gdb --tmux=--no-continue +./run --arch arm --baremetal interactive/prompt --wait-gdb --tmux --tmux-args --no-continue .... Now you can just `stepi` to when jumping into main to go to the C code in link:baremetal/interactive/prompt.c[]. @@ -10341,7 +10356,7 @@ Now you can just `stepi` to when jumping into main to go to the C code in link:b This is specially interesting for the executables that don't use the bootloader from under `baremetal/arch//no_bootloader/*.S`, e.g.: .... -./run --arch arm --baremetal arch/arm/no_bootloader/semihost_exit --wait-gdb --tmux=--no-continue +./run --arch arm --baremetal arch/arm/no_bootloader/semihost_exit --wait-gdb --tmux --tmux-args --no-continue .... The cool thing about those examples is that you start at the very first instruction of your program, which gives more control. @@ -10369,7 +10384,7 @@ The most important things that we setup in the bootloaders are: The C functions that become available as a result are: * Newlib functions implemented at link:baremetal/lib/syscalls.c[] -* non-Newlib functions implemented at link:common.c[] +* non-Newlib functions implemented at link:kwargs['c'][] It is not possible to call those C functions from the examples that don't use a bootloader. @@ -10401,7 +10416,7 @@ svc 0x00123456 and we can see from the docs that `0x18` stands for the `SYS_EXIT` command. -This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/common.c[]. +This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/kwargs['c'][]. Other magic operations we can do with semihosting besides exiting the on the host include: @@ -10441,7 +10456,7 @@ Bibliography: For gem5, you need: .... -patch -d "$(./getvar gem5_src_dir)" -p 1 < patches/manual/gem5-semihost.patch +patch -d "$(./getvar gem5_source_dir)" -p 1 < patches/manual/gem5-semihost.patch .... https://stackoverflow.com/questions/52475268/how-to-enable-arm-semihosting-in-gem5/52475269#52475269 @@ -11500,17 +11515,17 @@ Analogous to the <> but with the `--gem5-build-id` ./build-gem5 # Build another branch. -git -C "$(./getvar gem5_src_dir)" checkout some-branch +git -C "$(./getvar gem5_source_dir)" checkout some-branch ./build-gem5 --gem5-build-id some-branch # Restore master. -git -C "$(./getvar gem5_src_dir)" checkout - +git -C "$(./getvar gem5_source_dir)" checkout - # Run master. ./run --gem5 # Run another branch. -git -C "$(./getvar gem5_src_dir)" checkout some-branch +git -C "$(./getvar gem5_source_dir)" checkout some-branch ./run --gem5-build-id some-branch --gem5 .... diff --git a/arm b/arm deleted file mode 100755 index c54105c..0000000 --- a/arm +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -build-crosstool-ng \ -; \ No newline at end of file diff --git a/bench-all b/bench-all index 6915a02..27cfa07 100755 --- a/bench-all +++ b/bench-all @@ -100,9 +100,9 @@ if "$bench_gem5_build"; then common_arch="$default_arch" gem5_build_id=bench common_gem5_build_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_build_dir)" - common_gem5_src_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_src_dir)" + common_gem5_source_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_src_dir)" results_file="${common_gem5_build_dir}/lkmc-bench-build.txt" - git -C "${common_gem5_src_dir}" clean -xdf + git -C "${common_gem5_source_dir}" clean -xdf rm -f "$results_file" "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id" # TODO understand better: --foreground required otherwise we cannot @@ -110,7 +110,7 @@ if "$bench_gem5_build"; then # bash -c "eval 'timeout 5 sleep 3'" "${root_dir}/bench-cmd" "timeout --foreground 900 ./build-gem5 --arch '$common_arch' --gem5-build-id '$gem5_build_id'" "$results_file" cp "$results_file" "${new_dir}/gem5-bench-build-${common_arch}.txt" - git -C "${common_gem5_src_dir}" clean -xdf + git -C "${common_gem5_source_dir}" clean -xdf "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id" fi diff --git a/bisect-linux-boot-gem5 b/bisect-linux-boot-gem5 index 91aa346..c204fac 100755 --- a/bisect-linux-boot-gem5 +++ b/bisect-linux-boot-gem5 @@ -6,10 +6,10 @@ import shutil import sys import common -build_linux = imp.load_source('build-linux', os.path.join(common.root_dir, 'build_linux')) -run = imp.load_source('run', os.path.join(common.root_dir, 'run')) +build_linux = imp.load_source('build-linux', os.path.join(kwargs['root_dir'], 'build_linux')) +run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run')) -parser = common.get_argparse( +parser = self.get_argparse( argparse_args={ 'description': '''Bisect the Linux kernel on gem5 boots. @@ -20,11 +20,11 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#b 'linux_build_id': 'bisect', }, ) -args = common.setup(parser) +args = self.setup(parser) # We need a clean rebuild because rebuilds at different revisions: # - may fail # - may not actually rebuild all files, e.g. on header changes -common.rmrf(common.linux_build_dir) +self.rmrf(kwargs['linux_build_dir']) build_linux.LinuxComponent().do_build(args) status = run.main(args, { 'eval': 'm5 exit', diff --git a/bst-vs-heap b/bst-vs-heap index b8cbda5..69ec9b9 100755 --- a/bst-vs-heap +++ b/bst-vs-heap @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import common -parser = common.get_argparse( +parser = self.get_argparse( argparse_args={'description':'Convert a BST vs heap stat file into a gnuplot input'} ) -args = common.setup(parser) -stats = common.get_stats() +args = self.setup(parser) +stats = self.get_stats() it = iter(stats) i = 1 for stat in it: diff --git a/build b/build index 050f3ff..a35c281 100755 --- a/build +++ b/build @@ -7,6 +7,7 @@ import re import os import common +from shell_helpers import LF class Component: ''' @@ -49,11 +50,11 @@ class Component: def run_cmd(cmd, arch): global args cmd_abs = cmd.copy() - cmd_abs[0] = os.path.join(common.root_dir, cmd[0]) + cmd_abs[0] = os.path.join(kwargs['root_dir'], cmd[0]) cmd_abs.extend(['--arch', arch]) - if args.extra_args: - cmd_abs.append(args.extra_args) - common.run_cmd(cmd_abs, dry_run=args.dry_run) + if kwargs['extra_args']: + cmd_abs.append(kwargs['extra_args']) + self.sh.run_cmd(cmd_abs, dry_run=kwargs['dry_run']) buildroot_component = Component( lambda arch: run_cmd(['build-buildroot'], arch), @@ -86,15 +87,15 @@ name_to_component_map = { # Leaves without dependencies. 'baremetal-qemu': Component( lambda arch: run_cmd(['build-baremetal', '--qemu'], arch), - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'baremetal-gem5': Component( lambda arch: run_cmd(['build-baremetal', '--gem5'], arch), - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'baremetal-gem5-pbx': Component( lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch), - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'buildroot': buildroot_component, 'buildroot-gcc': buildroot_component, @@ -103,7 +104,7 @@ name_to_component_map = { ), 'crosstool-ng': Component( lambda arch: run_cmd(['build-crosstool-ng'], arch), - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], # http://crosstool-ng.github.io/docs/os-setup/ apt_get_pkgs={ 'bison', @@ -197,7 +198,7 @@ name_to_component_map = { 'gem5-baremetal', 'baremetal-gem5-pbx', ], - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'all-linux': Component(dependencies=[ 'qemu-gem5-buildroot', @@ -296,10 +297,10 @@ group = parser.add_mutually_exclusive_group(required=False) group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\ Build the selected components for all archs. ''') -group.add_argument('-a', '--arch', choices=common.arch_choices, default=[], action='append', help='''\ +group.add_argument('-a', '--arch', choices=kwargs['arch_choices'], default=[], action='append', help='''\ Build the selected components for this arch. Select multiple archs by passing this option multiple times. Default: [{}] -'''.format(common.default_arch)) +'''.format(kwargs['default_arch'])) parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\ Also download all dependencies required for a given build: Ubuntu packages, Python packages and git submodules. @@ -314,27 +315,27 @@ Extra args to pass to all scripts. ) parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\ Which components to build. Default: qemu-buildroot -'''.format(common.default_arch)) -common.add_dry_run_argument(parser) +'''.format(kwargs['default_arch'])) +self.add_dry_run_argument(parser) args = parser.parse_args() -common.setup_dry_run_arguments(args) +self.setup_dry_run_arguments(args) # Decide archs. -if args.arch == []: - if args.all or args.all_archs: - archs = common.all_archs.copy() +if kwargs['arch'] == []: + if kwargs['all'] or kwargs['all_archs']: + archs = kwargs['all_archs'].copy() else: - archs = set([common.default_arch]) + archs = set([kwargs['default_arch']]) else: archs = set() - for arch in args.arch: - if arch in common.arch_short_to_long_dict: - arch = common.arch_short_to_long_dict[arch] + for arch in kwargs['arch']: + if arch in kwargs['arch_short_to_long_dict']: + arch = kwargs['arch_short_to_long_dict'][arch] archs.add(arch) # Decide components. -components = args.components -if args.all: +components = kwargs['components'] +if kwargs['all']: components = ['all'] elif components == []: components = ['qemu-buildroot'] @@ -350,7 +351,7 @@ for component_name in components: selected_components.append(component) todo.extend(component.dependencies) -if args.download_dependencies: +if kwargs['download_dependencies']: apt_get_pkgs = { # Core requirements for this repo. 'git', @@ -388,12 +389,12 @@ if args.download_dependencies: python2_pkgs.update(component.python2_pkgs) python3_pkgs.update(component.python3_pkgs) if apt_get_pkgs or apt_build_deps: - if args.travis: + if kwargs['travis']: interacive_pkgs = { 'libsdl2-dev', } apt_get_pkgs.difference_update(interacive_pkgs) - if common.in_docker: + if kwargs['in_docker']: sudo = [] # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai os.environ['DEBIAN_FRONTEND'] = 'noninteractive' @@ -406,35 +407,35 @@ if args.download_dependencies: f.write(sources_txt) else: sudo = ['sudo'] - if common.in_docker or args.travis: + if kwargs['in_docker'] or kwargs['travis']: y = ['-y'] else: y = [] - common.run_cmd( - sudo + ['apt-get', 'update', common.Newline] + self.sh.run_cmd( + sudo + ['apt-get', 'update', LF] ) if apt_get_pkgs: - common.run_cmd( - sudo + ['apt-get', 'install'] + y + [common.Newline] + - common.add_newlines(sorted(apt_get_pkgs)) + self.sh.run_cmd( + sudo + ['apt-get', 'install'] + y + [LF] + + self.sh.add_newlines(sorted(apt_get_pkgs)) ) if apt_build_deps: - common.run_cmd( + self.sh.run_cmd( sudo + - ['apt-get', 'build-dep'] + y + [common.Newline] + - common.add_newlines(sorted(apt_build_deps)) + ['apt-get', 'build-dep'] + y + [LF] + + self.sh.add_newlines(sorted(apt_build_deps)) ) if python2_pkgs: - common.run_cmd( - ['python', '-m', 'pip', 'install', '--user', common.Newline] + - common.add_newlines(sorted(python2_pkgs)) + self.sh.run_cmd( + ['python', '-m', 'pip', 'install', '--user', LF] + + self.sh.add_newlines(sorted(python2_pkgs)) ) if python3_pkgs: # Not with pip executable directly: # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 - common.run_cmd( - ['python3', '-m', 'pip', 'install', '--user', common.Newline] + - common.add_newlines(sorted(python3_pkgs)) + self.sh.run_cmd( + ['python3', '-m', 'pip', 'install', '--user', LF] + + self.sh.add_newlines(sorted(python3_pkgs)) ) git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive'] if submodules: @@ -448,9 +449,9 @@ if args.download_dependencies: # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone # # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 - common.run_cmd( - git_cmd_common + ['--', common.Newline] + - common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules)]) + self.sh.run_cmd( + git_cmd_common + ['--', LF] + + self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules)]) ) if submodules_shallow: # == Shallow cloning. @@ -472,9 +473,9 @@ if args.download_dependencies: # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702 # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac # - common.run_cmd( - git_cmd_common + ['--depth', '1', '--', common.Newline] + - common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules_shallow)]) + self.sh.run_cmd( + git_cmd_common + ['--depth', '1', '--', LF] + + self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules_shallow)]) ) # Do the build. diff --git a/build-baremetal b/build-baremetal index 16594bb..c37bc7d 100755 --- a/build-baremetal +++ b/build-baremetal @@ -1,73 +1,79 @@ #!/usr/bin/env python3 - import os import common +from shell_helpers import LF -class BaremetalComponent(common.Component): - def do_build(self, args): - common.assert_crosstool_ng_supports_arch(args.arch) - build_dir = self.get_build_dir(args) - bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext)) +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ +Build the baremetal examples with crosstool-NG. +''') + + def build(self): + self.assert_crosstool_ng_supports_arch(self.env['arch']) + build_dir = self.get_build_dir() + bootloader_obj = os.path.join(self.env['baremetal_build_lib_dir'], 'bootloader{}'.format(self.env['obj_ext'])) common_basename_noext = 'common' - common_src = os.path.join(common.root_dir, common_basename_noext + common.c_ext) - common_obj = os.path.join(common.baremetal_build_lib_dir, common_basename_noext + common.obj_ext) + common_src = os.path.join(self.env['root_dir'], common_basename_noext + self.env['c_ext']) + common_obj = os.path.join(self.env['baremetal_build_lib_dir'], common_basename_noext + self.env['obj_ext']) syscalls_basename_noext = 'syscalls' - syscalls_src = os.path.join(common.baremetal_src_lib_dir, syscalls_basename_noext + common.c_ext) - syscalls_obj = os.path.join(common.baremetal_build_lib_dir, syscalls_basename_noext + common.obj_ext) + syscalls_src = os.path.join(self.env['baremetal_src_lib_dir'], syscalls_basename_noext + self.env['c_ext']) + syscalls_obj = os.path.join(self.env['baremetal_build_lib_dir'], syscalls_basename_noext + self.env['obj_ext']) common_objs = [common_obj, syscalls_obj] cflags = [ - '-I', common.baremetal_src_lib_dir, common.Newline, - '-I', common.root_dir, common.Newline, - '-O0', common.Newline, - '-ggdb3', common.Newline, - '-mcpu={}'.format(common.mcpu), common.Newline, - '-nostartfiles', common.Newline, + '-I', self.env['baremetal_src_lib_dir'], LF, + '-I', self.env['root_dir'], LF, + '-O0', LF, + '-ggdb3', LF, + '-mcpu={}'.format(self.env['mcpu']), LF, + '-nostartfiles', LF, ] - if args.prebuilt: + if self.env['prebuilt']: gcc = 'arm-none-eabi-gcc' else: - os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH'] - gcc = common.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng']) - if common.emulator == 'gem5': - if common.machine == 'VExpress_GEM5_V1': + os.environ['PATH'] = self.env['crosstool_ng_bin_dir'] + os.environ['PATH'] + gcc = self.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng']) + if self.env['emulator'] == 'gem5': + if self.env['machine'] == 'VExpress_GEM5_V1': entry_address = 0x80000000 uart_address = 0x1c090000 - elif common.machine == 'RealViewPBX': + elif self.env['machine'] == 'RealViewPBX': entry_address = 0x10000 uart_address = 0x10009000 else: - raise Exception('unknown machine: ' + common.machine) - cflags.extend(['-D', 'GEM5'.format(uart_address), common.Newline]) + raise Exception('unknown machine: ' + self.env['machine']) + cflags.extend(['-D', 'GEM5'.format(uart_address), LF]) else: entry_address = 0x40000000 uart_address = 0x09000000 os.makedirs(build_dir, exist_ok=True) - os.makedirs(common.baremetal_build_lib_dir, exist_ok=True) - src = os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)) - if common.need_rebuild([src], bootloader_obj): - common.run_cmd( - [gcc, common.Newline] + + os.makedirs(self.env['baremetal_build_lib_dir'], exist_ok=True) + src = os.path.join(self.env['baremetal_src_lib_dir'], '{}{}'.format(self.env['arch'], self.env['asm_ext'])) + if self.need_rebuild([src], bootloader_obj): + self.sh.run_cmd( + [gcc, LF] + cflags + [ - '-c', common.Newline, - '-o', bootloader_obj, common.Newline, - src, common.Newline, + '-c', LF, + '-o', bootloader_obj, LF, + src, LF, ] ) for src, obj in [ (common_src, common_obj), (syscalls_src, syscalls_obj), ]: - if common.need_rebuild([src], obj): - common.run_cmd( - [gcc, common.Newline] + + if self.need_rebuild([src], obj): + self.sh.run_cmd( + [gcc, LF] + cflags + [ - '-c', common.Newline, - '-D', 'UART0_ADDR={:#x}'.format(uart_address), common.Newline, - '-o', obj, common.Newline, - src, common.Newline, + '-c', LF, + '-D', 'UART0_ADDR={:#x}'.format(uart_address), LF, + '-o', obj, LF, + src, LF, ] ) self._build_dir( @@ -86,8 +92,8 @@ class BaremetalComponent(common.Component): bootloader_obj=bootloader_obj, common_objs=common_objs, ) - arch_dir = os.path.join('arch', args.arch) - if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)): + arch_dir = os.path.join('arch', self.env['arch']) + if os.path.isdir(os.path.join(self.env['baremetal_src_dir'], arch_dir)): self._build_dir( arch_dir, gcc=gcc, @@ -96,8 +102,8 @@ class BaremetalComponent(common.Component): bootloader_obj=bootloader_obj, common_objs=common_objs, ) - arch_dir = os.path.join('arch', args.arch, 'no_bootloader') - if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)): + arch_dir = os.path.join('arch', self.env['arch'], 'no_bootloader') + if os.path.isdir(os.path.join(self.env['baremetal_src_dir'], arch_dir)): self._build_dir( arch_dir, gcc=gcc, @@ -108,18 +114,8 @@ class BaremetalComponent(common.Component): bootloader=False, ) - def get_argparse_args(self): - return { - 'description': '''\ -Build the baremetal examples with crosstool-NG. -''' - } - - def get_build_dir(self, args): - return common.baremetal_build_dir - - def get_default_args(self): - return {'baremetal': 'all'} + def get_build_dir(self): + return self.env['baremetal_build_dir'] def _build_dir( self, @@ -137,42 +133,42 @@ Build the baremetal examples with crosstool-NG. Place outputs on the same subpath or the output directory. """ - in_dir = os.path.join(common.baremetal_src_dir, subpath) - out_dir = os.path.join(common.baremetal_build_dir, subpath) + in_dir = os.path.join(self.env['baremetal_src_dir'], subpath) + out_dir = os.path.join(self.env['baremetal_build_dir'], subpath) os.makedirs(out_dir, exist_ok=True) common_objs = common_objs.copy() if bootloader: common_objs.append(bootloader_obj) for in_basename in os.listdir(in_dir): in_path = os.path.join(in_dir, in_basename) - if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext): + if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (self.env['c_ext'], self.env['asm_ext']): in_name = os.path.splitext(in_basename)[0] - main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext)) - src = os.path.join(common.baremetal_src_dir, in_path) - if common.need_rebuild([src], main_obj): - common.run_cmd( - [gcc, common.Newline] + + main_obj = os.path.join(self.env['baremetal_build_dir'], subpath, '{}{}'.format(in_name, self.env['obj_ext'])) + src = os.path.join(self.env['baremetal_src_dir'], in_path) + if self.need_rebuild([src], main_obj): + self.sh.run_cmd( + [gcc, LF] + cflags + [ - '-c', common.Newline, - '-o', main_obj, common.Newline, - src, common.Newline, + '-c', LF, + '-o', main_obj, LF, + src, LF, ] ) objs = common_objs + [main_obj] - out = os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext) - link_script = os.path.join(common.baremetal_src_dir, 'link.ld') - if common.need_rebuild(objs + [link_script], out): - common.run_cmd( - [gcc, common.Newline] + + out = os.path.join(self.env['baremetal_build_dir'], subpath, in_name + self.env['baremetal_build_ext']) + link_script = os.path.join(self.env['baremetal_src_dir'], 'link.ld') + if self.need_rebuild(objs + [link_script], out): + self.sh.run_cmd( + [gcc, LF] + cflags + [ - '-Wl,--section-start=.text={:#x}'.format(entry_address), common.Newline, - '-o', out, common.Newline, - '-T', link_script, common.Newline, + '-Wl,--section-start=.text={:#x}'.format(entry_address), LF, + '-o', out, LF, + '-T', link_script, LF, ] + - common.add_newlines(objs) + self.sh.add_newlines(objs) ) if __name__ == '__main__': - BaremetalComponent().build() + Main().cli() diff --git a/build-buildroot b/build-buildroot index 0d0e9ea..ad2687e 100755 --- a/build-buildroot +++ b/build-buildroot @@ -9,11 +9,16 @@ import time import re import common +from shell_helpers import LF -class BuildrootComponent(common.Component): - def add_parser_arguments(self, parser): - parser.add_argument( - '--build-linux', default=self._defaults['build_linux'], action='store_true', +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ +Run Linux on an emulator +''') + self.add_argument( + '--build-linux', default=False, help='''\ Enable building the Linux kernel with Buildroot. This is done mostly to extract Buildroot's default kernel configurations when updating Buildroot. @@ -21,35 +26,35 @@ This kernel will not be use by our other scripts. Configuring this kernel is not currently supported, juse use ./build-linux script if you want to do that. ''' ) - parser.add_argument( - '--baseline', default=self._defaults['baseline'], action='store_true', + self.add_argument( + '--baseline', default=False, help='''Do a default-ish Buildroot defconfig build, without any of our extra options. Mostly to track how much slower we are than a basic build. ''' ) - parser.add_argument( - '--config', default=self._defaults['config'], action='append', + self.add_argument( + '--config', default=[], action='append', help='''Add a single Buildroot config to the current build. Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'. Can be used multiple times to add multiple configs. Takes precedence over any Buildroot config files. ''' ) - parser.add_argument( - '--config-fragment', default=self._defaults['config_fragment'], action='append', + self.add_argument( + '--config-fragment', default=[], action='append', help='''Also use the given Buildroot configuration fragment file. Pass multiple times to use multiple fragment files. ''' ) - parser.add_argument( - '--no-all', default=self._defaults['no_all'], action='store_true', + self.add_argument( + '--no-all', default=False, help='''\ Don't build the all target which normally gets build by default. That target builds the root filesystem and all its dependencies. ''' ) - parser.add_argument( - '--no-overlay', default=self._defaults['no_all'], action='store_true', + self.add_argument( + '--no-overlay', default=False, help='''\ Don't add our overlay which contains all files we build without going through Buildroot. This prevents us from overwriting certain Buildroot files. Remember however that you must @@ -57,132 +62,117 @@ still rebuild the Buildroot package that provides those files to actually put th files on the root filesystem. ''' ) - parser.add_argument( - 'extra_make_args', default=self._defaults['extra_make_args'], metavar='extra-make-args', nargs='*', + self.add_argument( + 'extra-make-args', default=[], nargs='*', help='''\ Extra arguments to be passed to the Buildroot make, usually extra Buildroot targets. ''' ) - def do_build(self, args): - build_dir = self.get_build_dir(args) - os.makedirs(common.out_dir, exist_ok=True) - extra_make_args = common.add_newlines(args.extra_make_args) - if args.build_linux: - extra_make_args.extend(['linux-reconfigure', common.Newline]) - if common.emulator == 'gem5': - extra_make_args.extend(['gem5-reconfigure', common.Newline]) - if args.arch == 'x86_64': + def build(self): + build_dir = self.get_build_dir() + os.makedirs(self.env['out_dir'], exist_ok=True) + extra_make_args = self.sh.add_newlines(self.env['extra_make_args']) + if self.env['build_linux']: + extra_make_args.extend(['linux-reconfigure', LF]) + if self.env['emulator'] == 'gem5': + extra_make_args.extend(['gem5-reconfigure', LF]) + if self.env['arch'] == 'x86_64': defconfig = 'qemu_x86_64_defconfig' - elif args.arch == 'arm': + elif self.env['arch'] == 'arm': defconfig = 'qemu_arm_vexpress_defconfig' - elif args.arch == 'aarch64': + elif self.env['arch'] == 'aarch64': defconfig = 'qemu_aarch64_virt_defconfig' br2_external_dirs = [] - for package_dir in os.listdir(common.packages_dir): - package_dir_abs = os.path.join(common.packages_dir, package_dir) + for package_dir in os.listdir(self.env['packages_dir']): + package_dir_abs = os.path.join(self.env['packages_dir'], package_dir) if os.path.isdir(package_dir_abs): br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs)) br2_external_str = ':'.join(br2_external_dirs) - common.run_cmd( + self.sh.run_cmd( [ - 'make', common.Newline, - 'O={}'.format(common.buildroot_build_dir), common.Newline, - 'BR2_EXTERNAL={}'.format(br2_external_str), common.Newline, - defconfig, common.Newline, + 'make', LF, + 'O={}'.format(self.env['buildroot_build_dir']), LF, + 'BR2_EXTERNAL={}'.format(br2_external_str), LF, + defconfig, LF, ], - cwd=common.buildroot_src_dir, + cwd=self.env['buildroot_src_dir'], ) - configs = args.config + configs = self.env['config'] configs.extend([ - 'BR2_JLEVEL={}'.format(args.nproc), - 'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir), + 'BR2_JLEVEL={}'.format(self.env['nproc']), + 'BR2_DL_DIR="{}"'.format(self.env['buildroot_download_dir']), ]) - if not args.build_linux: + if not self.env['build_linux']: configs.extend([ '# BR2_LINUX_KERNEL is not set', ]) config_fragments = [] - if not args.baseline: + if not self.env['baseline']: configs.extend([ 'BR2_GLOBAL_PATCH_DIR="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global')) + self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'patches', 'global')) ), 'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment')) + self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'busybox_config_fragment')) ), 'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override')) + self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'buildroot_override')) ), 'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script')) + self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'rootfs-post-build-script')) ), 'BR2_ROOTFS_USERS_TABLES="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table')) + self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'user_table')) ), ]) - if not args.no_overlay: + if not self.env['no_overlay']: configs.append('BR2_ROOTFS_OVERLAY="{}"'.format( - self._path_relative_to_buildroot(common.out_rootfs_overlay_dir) + self._path_relative_to_buildroot(self.env['out_rootfs_overlay_dir']) )) config_fragments = [ - os.path.join(common.root_dir, 'buildroot_config', 'default') - ] + args.config_fragment - common.write_configs(common.buildroot_config_file, configs, config_fragments) - common.run_cmd( + os.path.join(self.env['root_dir'], 'buildroot_config', 'default') + ] + self.env['config_fragment'] + # TODO Can't get rid of these for now with nice fragments on Buildroot: + # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config + self.sh.write_configs(self.env['buildroot_config_file'], configs, config_fragments) + self.sh.run_cmd( [ - 'make', common.Newline, - 'O={}'.format(common.buildroot_build_dir), common.Newline, - 'olddefconfig', common.Newline, + 'make', LF, + 'O={}'.format(self.env['buildroot_build_dir']), LF, + 'olddefconfig', LF, ], - cwd=common.buildroot_src_dir, + cwd=self.env['buildroot_src_dir'], ) - common.make_build_dirs() - if not args.no_all: - extra_make_args.extend(['all', common.Newline]) - common.run_cmd( + self.make_build_dirs() + if not self.env['no_all']: + extra_make_args.extend(['all', LF]) + self.sh.run_cmd( [ - 'make', common.Newline, - 'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_src_dir), common.Newline, - 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), common.Newline, - 'O={}'.format(common.buildroot_build_dir), common.Newline, - 'V={}'.format(int(args.verbose)), common.Newline, + 'make', LF, + 'LKMC_GEM5_SRCDIR="{}"'.format(self.env['gem5_source_dir']), LF, + 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(self.env['parsec_benchmark_src_dir']), LF, + 'O={}'.format(self.env['buildroot_build_dir']), LF, + 'V={}'.format(int(self.env['verbose'])), LF, ] + extra_make_args , - out_file=os.path.join(common.buildroot_build_dir, 'lkmc.log'), + out_file=os.path.join(self.env['buildroot_build_dir'], 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], - cwd=common.buildroot_src_dir, + cwd=self.env['buildroot_src_dir'], ) # Create the qcow2 from ext2. # Skip if qemu is not present, because gem5 does not need the qcow2. # so we don't force a QEMU build for gem5. - if not args.no_all and os.path.exists(common.qemu_img_executable): - common.raw_to_qcow2() + if not self.env['no_all'] and os.path.exists(self.env['qemu_img_executable']): + self.raw_to_qcow2() - def get_argparse_args(self): - return { - 'description': '''\ -Run Linux on an emulator -''' - } - - def get_build_dir(self, args): - return common.buildroot_build_dir - - _defaults = { - 'baseline': False, - 'build_linux': False, - 'config': [], - 'config_fragment': [], - 'extra_make_args': [], - 'no_all': False, - 'skip_configure': False, - } + def get_build_dir(self): + return self.env['buildroot_build_dir'] def _path_relative_to_buildroot(self, abspath): - return os.path.relpath(abspath, common.buildroot_src_dir) + return os.path.relpath(abspath, self.env['buildroot_src_dir']) if __name__ == '__main__': - BuildrootComponent().build() + Main().cli() diff --git a/build-crosstool-ng b/build-crosstool-ng index 3386ad6..74618a1 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -3,75 +3,70 @@ import os import common +from shell_helpers import LF -class CrosstoolNgComponent(common.Component): - def do_build(self, args): - common.assert_crosstool_ng_supports_arch(args.arch) - build_dir = self.get_build_dir(args) - defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig') - os.makedirs(common.crosstool_ng_util_dir, exist_ok=True) - os.makedirs(common.crosstool_ng_download_dir, exist_ok=True) +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ +Build crosstool-NG with Newlib for bare metal compilation +''') + + def build(self): + self.assert_crosstool_ng_supports_arch(self.env['arch']) + build_dir = self.get_build_dir() + defconfig_dest = os.path.join(self.env['crosstool_ng_util_dir'], 'defconfig') + os.makedirs(self.env['crosstool_ng_util_dir'], exist_ok=True) + os.makedirs(self.env['crosstool_ng_download_dir'], exist_ok=True) # Bootstrap out-ot-tree WONTFIX. I've tried. # https://github.com/crosstool-ng/crosstool-ng/issues/1021 - os.chdir(common.crosstool_ng_src_dir) - common.run_cmd( - [os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), common.Newline], + os.chdir(self.env['crosstool_ng_src_dir']) + self.sh.run_cmd( + [os.path.join(self.env['crosstool_ng_src_dir'], 'bootstrap'), LF], ) - os.chdir(common.crosstool_ng_util_dir) - common.run_cmd( + os.chdir(self.env['crosstool_ng_util_dir']) + self.sh.run_cmd( [ - os.path.join(common.crosstool_ng_src_dir, 'configure'), common.Newline, - '--enable-local', common.Newline, - ], - ) - common.run_cmd( - [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, + os.path.join(self.env['crosstool_ng_src_dir'], 'configure'), LF, + '--enable-local', LF, ], ) + self.sh.run_cmd(['make', '-j', str(self.env['nproc']), LF]) # Build the toolchain. - common.cp( - os.path.join(common.root_dir, 'crosstool_ng_config', args.arch), + self.sh.cp( + os.path.join(self.env['root_dir'], 'crosstool_ng_config', self.env['arch']), defconfig_dest ) - common.write_configs( - common.crosstool_ng_defconfig, + self.sh.write_configs( + self.env['crosstool_ng_defconfig'], [ - 'CT_PREFIX_DIR="{}"'.format(common.crosstool_ng_install_dir), + 'CT_PREFIX_DIR="{}"'.format(self.env['crosstool_ng_install_dir']), 'CT_WORK_DIR="{}"'.format(build_dir), - 'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir), + 'CT_LOCAL_TARBALLS_DIR="{}"'.format(self.env['crosstool_ng_download_dir']), ] ) - common.run_cmd( + self.sh.run_cmd( [ - common.crosstool_ng_executable, common.Newline, - 'defconfig', common.Newline, + self.env['crosstool_ng_executable'], LF, + 'defconfig', LF, ], ) os.unlink(defconfig_dest) - common.run_cmd( + self.sh.run_cmd( [ - common.crosstool_ng_executable, common.Newline, - 'build', common.Newline, - 'CT_JOBS={}'.format(str(args.nproc)), common.Newline, + self.env['crosstool_ng_executable'], LF, + 'build', LF, + 'CT_JOBS={}'.format(str(self.env['nproc'])), LF, ], out_file=os.path.join(build_dir, 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], - extra_paths=[common.ccache_dir], + extra_paths=[self.env['ccache_dir']], ) - def get_argparse_args(self): - return { - 'description': '''\ -Build crosstool-NG with Newlib for bare metal compilation' -''' - } - - def get_build_dir(self, args): - return common.crosstool_ng_build_dir + def get_build_dir(self): + return self.env['crosstool_ng_build_dir'] if __name__ == '__main__': - CrosstoolNgComponent().build() + Main().cli() diff --git a/build-docker b/build-docker index 1882b56..6c2ef2c 100755 --- a/build-docker +++ b/build-docker @@ -5,9 +5,10 @@ import subprocess import tarfile import common +from shell_helpers import LF -class DockerComponent(common.Component): +class DockerComponent(self.Component): def get_argparse_args(self): return { 'description': '''\ @@ -17,8 +18,8 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- ''' } - def do_build(self, args): - build_dir = self.get_build_dir(args) + def build(self): + build_dir = self.get_build_dir() container_name = 'lkmc-guest' target_dir = os.path.join('/root', 'linux-kernel-module-cheat') os.makedirs(build_dir, exist_ok=True) @@ -29,12 +30,12 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- '--format', '{{.Names}}', ]).decode() if container_name in containers.split(): - common.run_cmd([ + self.sh.run_cmd([ 'docker', 'rm', container_name, ]) - common.run_cmd([ + self.sh.run_cmd([ 'docker', 'create', '--name', container_name, @@ -44,36 +45,36 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- '--privileged', '-t', '-w', target_dir, - '-v', '{}:{}'.format(common.root_dir, target_dir), + '-v', '{}:{}'.format(kwargs['root_dir'], target_dir), 'ubuntu:18.04', 'bash', ]) - common.run_cmd([ + self.sh.run_cmd([ 'docker', 'export', '-o', - common.docker_tar_file, + kwargs['docker_tar_file'], container_name, ]) - tar = tarfile.open(common.docker_tar_file) - tar.extractall(common.docker_tar_dir) + tar = tarfile.open(kwargs['docker_tar_file']) + tar.extractall(kwargs['docker_tar_dir']) tar.close() # sudo not required in theory # https://askubuntu.com/questions/1046828/how-to-run-libguestfs-tools-tools-such-as-virt-make-fs-without-sudo - common.run_cmd([ + self.sh.run_cmd([ 'virt-make-fs', '--format', 'raw', '--size', '+1G', '--type', 'ext2', - common.docker_tar_dir, - common.docker_rootfs_raw_file, + kwargs['docker_tar_dir'], + kwargs['docker_rootfs_raw_file'], ]) - common.raw_to_qcow2(prebuilt=True) + self.raw_to_qcow2(prebuilt=True) - def get_build_dir(self, args): - return common.docker_build_dir + def get_build_dir(self): + return kwargs['docker_build_dir'] def get_default_args(self): return {'docker': True} -DockerComponent().build() +Main().cli() diff --git a/build-gem5 b/build-gem5 index 4a89b61..7147ac1 100755 --- a/build-gem5 +++ b/build-gem5 @@ -5,59 +5,57 @@ import pathlib import subprocess import common +from shell_helpers import LF -class Gem5Component(common.Component): - def add_parser_arguments(self, parser): - parser.add_argument( +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__() + self.add_argument( 'extra_scons_args', - default=[], metavar='extra-scons-args', - nargs='*' + nargs='*', ) - def do_build(self, args): - build_dir = self.get_build_dir(args) - binaries_dir = os.path.join(common.gem5_system_dir, 'binaries') - disks_dir = os.path.join(common.gem5_system_dir, 'disks') + def build(self): + build_dir = self.get_build_dir() + binaries_dir = os.path.join(self.env['gem5_system_dir'], 'binaries') + disks_dir = os.path.join(self.env['gem5_system_dir'], 'disks') os.makedirs(binaries_dir, exist_ok=True) os.makedirs(disks_dir, exist_ok=True) - if args.gem5_source_dir is None: - if not os.path.exists(os.path.join(common.gem5_src_dir, '.git')): - if common.gem5_src_dir == common.gem5_default_src_dir: + if self.env['gem5_source_dir'] is None: + if not os.path.exists(os.path.join(self.env['gem5_source_dir'], '.git')): + if self.env['gem5_source_dir'] == self.env['gem5_default_src_dir']: raise Exception('gem5 submodule not checked out') - common.run_cmd([ - 'git', common.Newline, - '-C', common.gem5_default_src_dir, common.Newline, - 'worktree', 'add', common.Newline, - '-b', os.path.join('wt', args.gem5_build_id), common.Newline, - common.gem5_src_dir, common.Newline, + self.sh.run_cmd([ + 'git', LF, + '-C', self.env['gem5_default_src_dir'], LF, + 'worktree', 'add', LF, + '-b', os.path.join('wt', self.env['gem5_build_id']), LF, + self.env['gem5_source_dir'], LF, ]) - if args.verbose: - verbose = ['--verbose', common.Newline] + if self.env['verbose']: + verbose = ['--verbose', LF] else: verbose = [] - if args.arch == 'x86_64': + if self.env['arch'] == 'x86_64': dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img') with open(dummy_img_path, 'wb') as dummy_img_file: zeroes = b'\x00' * (2 ** 16) for i in range(2 ** 10): dummy_img_file.write(zeroes) - common.run_cmd(['mkswap', dummy_img_path]) + self.sh.run_cmd(['mkswap', dummy_img_path, LF]) with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'): # This file must always be present, despite --kernel overriding that default and selecting the kernel. # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present. pass - elif args.arch == 'arm' or args.arch == 'aarch64': - gem5_system_src_dir = os.path.join(common.gem5_src_dir, 'system') + elif self.env['arch'] == 'arm' or self.env['arch'] == 'aarch64': + gem5_system_src_dir = os.path.join(self.env['gem5_source_dir'], 'system') # dtb dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') - dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt') - common.run_cmd([ - 'make', common.Newline, - '-C', dt_src_dir, common.Newline, - ]) - common.copy_dir_if_update_non_recursive( + dt_build_dir = os.path.join(self.env['gem5_system_dir'], 'arm', 'dt') + self.sh.run_cmd(['make', '-C', dt_src_dir, LF]) + self.sh.copy_dir_if_update_non_recursive( srcdir=dt_src_dir, destdir=dt_build_dir, filter_ext='.dtb', @@ -66,49 +64,46 @@ class Gem5Component(common.Component): # Bootloader 32. bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader') # TODO use the buildroot cross compiler here, and remove the dependencies from configure. - common.run_cmd([ - 'make', common.Newline, - '-C', bootloader32_dir, common.Newline, - 'CROSS_COMPILE=arm-linux-gnueabihf-', common.Newline, + self.sh.run_cmd([ + 'make', LF, + '-C', bootloader32_dir, LF, + 'CROSS_COMPILE=arm-linux-gnueabihf-', LF, ]) # bootloader - common.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) + self.sh.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) # Bootloader 64. bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') # TODO cross_compile is ignored because the make does not use CC... - common.run_cmd([ - 'make', common.Newline, - '-C', bootloader64_dir, common.Newline - ]) - common.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) - common.run_cmd( + self.sh.run_cmd(['make', '-C', bootloader64_dir, LF]) + self.sh.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) + self.sh.run_cmd( ( [ - 'scons', common.Newline, - '-j', str(args.nproc), common.Newline, - '--gold-linker', common.Newline, - '--ignore-style', common.Newline, - common.gem5_executable, common.Newline, + 'scons', LF, + '-j', str(self.env['nproc']), LF, + '--gold-linker', LF, + '--ignore-style', LF, + self.env['gem5_executable'], LF, ] + verbose + - common.add_newlines(args.extra_scons_args) + self.sh.add_newlines(self.env['extra_scons_args']) ), - cwd=common.gem5_src_dir, - extra_paths=[common.ccache_dir], + cwd=self.env['gem5_source_dir'], + extra_paths=[self.env['ccache_dir']], ) - term_src_dir = os.path.join(common.gem5_src_dir, 'util/term') + term_src_dir = os.path.join(self.env['gem5_source_dir'], 'util/term') m5term_build = os.path.join(term_src_dir, 'm5term') - common.run_cmd(['make', '-C', term_src_dir]) - if os.path.exists(common.gem5_m5term): - # Otherwise common.cp would fail with "Text file busy" if you + self.sh.run_cmd(['make', '-C', term_src_dir, LF]) + if os.path.exists(self.env['gem5_m5term']): + # Otherwise self.sh.cp would fail with "Text file busy" if you # tried to rebuild while running m5term: # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512 - os.unlink(common.gem5_m5term) - common.cp(m5term_build, common.gem5_m5term) + os.unlink(self.env['gem5_m5term']) + self.sh.cp(m5term_build, self.env['gem5_m5term']) - def get_build_dir(self, args): - return common.gem5_build_dir + def get_build_dir(self): + return self.env['gem5_build_dir'] if __name__ == '__main__': - Gem5Component().build() + Main().cli() diff --git a/build-linux b/build-linux index 8545684..0ef3424 100755 --- a/build-linux +++ b/build-linux @@ -4,10 +4,16 @@ import os import shutil import common +from shell_helpers import LF -class LinuxComponent(common.Component): - def add_parser_arguments(self, parser): - parser.add_argument( +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ +Build the Linux kernel. +''' + ) + self.add_argument( '--config', default=[], action='append', help='''\ Add a single kernel config configs to the current build. Sample value: @@ -15,14 +21,14 @@ Add a single kernel config configs to the current build. Sample value: configs. Takes precedence over any config files. ''' ) - parser.add_argument( + self.add_argument( '--config-fragment', default=[], action='append', help='''\ Also use the given kernel configuration fragment file. Pass multiple times to use multiple fragment files. ''' ) - parser.add_argument( + self.add_argument( '--custom-config-file', help='''\ Ignore all default kernel configurations and use this file instead. @@ -30,137 +36,124 @@ Still uses options explicitly passed with `--config` and `--config-fragment` on top of it. ''' ) - parser.add_argument( - '--custom-config-file-gem5', default=False, action='store_true', + self.add_argument( + '--custom-config-file-gem5', default=False, help='''\ Use the gem5 Linux kernel fork config as the custom config file. Ignore --custom-config-file. ''' ) - parser.add_argument( - '--config-only', default=False, action='store_true', + self.add_argument( + '--config-only', default=False, help='''\ Configure the kernel, but don't build it. ''' ) - parser.add_argument( - '--initramfs', default=False, action='store_true', - ) - parser.add_argument( - '--initrd', default=False, action='store_true', - ) - parser.add_argument( + self.add_argument( 'extra_make_args', default=[], metavar='extra-make-args', nargs='*' ) - def do_build(self, args): - build_dir = self.get_build_dir(args) - if args.initrd or args.initramfs: + def build(self): + build_dir = self.get_build_dir() + if self.env['initrd'] or self.env['initramfs']: raise Exception('just trolling, --initrd and --initramfs are broken for now') os.makedirs(build_dir, exist_ok=True) tool = 'gcc' - gcc = common.get_toolchain_tool(tool) + gcc = self.get_toolchain_tool(tool) prefix = gcc[:-len(tool)] common_args = { - 'cwd': common.linux_source_dir, + 'cwd': self.env['linux_source_dir'], } ccache = shutil.which('ccache') if ccache is not None: cc = '{} {}'.format(ccache, gcc) else: cc = gcc - if args.verbose: + if self.env['verbose']: verbose = ['V=1'] else: verbose = [] common_make_args = [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, - 'ARCH={}'.format(common.linux_arch), common.Newline, - 'CROSS_COMPILE={}'.format(prefix), common.Newline, - 'CC={}'.format(cc), common.Newline, - 'O={}'.format(build_dir), common.Newline, + 'make', LF, + '-j', str(self.env['nproc']), LF, + 'ARCH={}'.format(self.env['linux_arch']), LF, + 'CROSS_COMPILE={}'.format(prefix), LF, + 'CC={}'.format(cc), LF, + 'O={}'.format(build_dir), LF, ] + verbose - if args.custom_config_file_gem5: - custom_config_file = os.path.join(common.linux_source_dir, 'arch', common.linux_arch, 'configs', 'gem5_defconfig') + if self.env['custom_config_file_gem5']: + custom_config_file = os.path.join(self.env['linux_source_dir'], 'arch', self.env['linux_arch'], 'configs', 'gem5_defconfig') else: - custom_config_file = args.custom_config_file + custom_config_file = self.env['custom_config_file'] if custom_config_file is not None: if not os.path.exists(custom_config_file): - raise Exception('config fragment file does not exist: {}'.format(args.custom_config_file)) + raise Exception('config fragment file does not exist: {}'.format(custom_config_file)) base_config_file = custom_config_file config_fragments = [] else: - base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch)) + base_config_file = os.path.join(self.env['linux_config_dir'], 'buildroot-{}'.format(self.env['arch'])) config_fragments = ['min', 'default'] for i, config_fragment in enumerate(config_fragments): - config_fragments[i] = os.path.join(common.linux_config_dir, config_fragment) - config_fragments.extend(args.config_fragment) - if args.config != []: + config_fragments[i] = os.path.join(self.env['linux_config_dir'], config_fragment) + config_fragments.extend(self.env['config_fragment']) + if self.env['config'] != []: cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment') - cli_config_str = '\n'.join(args.config) - common.write_string_to_file(cli_config_fragment_path, cli_config_str) + cli_config_str = '\n'.join(self.env['config']) + self.sh.write_string_to_file(cli_config_fragment_path, cli_config_str) config_fragments.append(cli_config_fragment_path) - common.cp( + self.sh.cp( base_config_file, os.path.join(build_dir, '.config'), ) - common.run_cmd( + self.sh.run_cmd( [ - os.path.join(common.linux_source_dir, 'scripts', 'kconfig', 'merge_config.sh'), common.Newline, - '-m', common.Newline, - '-O', build_dir, common.Newline, - os.path.join(build_dir, '.config'), common.Newline, + os.path.join(self.env['linux_source_dir'], 'scripts', 'kconfig', 'merge_config.sh'), LF, + '-m', LF, + '-O', build_dir, LF, + os.path.join(build_dir, '.config'), LF, ] + - common.add_newlines(config_fragments) + self.sh.add_newlines(config_fragments) ) - common.run_cmd( + self.sh.run_cmd( ( common_make_args + - ['olddefconfig', common.Newline] + ['olddefconfig', LF] ), **common_args ) - if not args.config_only: - common.run_cmd( + if not self.env['config_only']: + self.sh.run_cmd( ( common_make_args + - common.add_newlines(args.extra_make_args) + self.sh.add_newlines(self.env['extra_make_args']) ), extra_env={ 'KBUILD_BUILD_VERSION': '1', 'KBUILD_BUILD_TIMESTAMP': 'Thu Jan 1 00:00:00 UTC 1970', 'KBUILD_BUILD_USER': 'lkmc', - 'KBUILD_BUILD_HOST': common.git_sha(common.linux_source_dir), + 'KBUILD_BUILD_HOST': common.git_sha(self.env['linux_source_dir']), }, **common_args ) - common.run_cmd( + self.sh.run_cmd( ( common_make_args + [ - 'INSTALL_MOD_PATH={}'.format(common.out_rootfs_overlay_dir), common.Newline, - 'modules_install', common.Newline, + 'INSTALL_MOD_PATH={}'.format(self.env['out_rootfs_overlay_dir']), LF, + 'modules_install', LF, ] ), **common_args ) # TODO: remove build and source https://stackoverflow.com/questions/13578618/what-does-build-and-source-link-do-in-lib-modules-kernel-version # TODO Basically all kernel modules also basically leak full host paths. Just terrible. Buildroot deals with that stuff nicely for us. - # common.rmrf() + # self.rmrf() - def get_argparse_args(self): - return { - 'description': '''\ -Build the Linux kernel. -''' - } - - def get_build_dir(self, args): - return common.linux_build_dir + def get_build_dir(self): + return self.env['linux_build_dir'] if __name__ == '__main__': - LinuxComponent().build() + Main().cli() diff --git a/build-m5 b/build-m5 index 18c7ee2..5713e36 100755 --- a/build-m5 +++ b/build-m5 @@ -3,42 +3,47 @@ import os import common +from shell_helpers import LF -class M5Component(common.Component): - def get_make_cmd(self, args): +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__() + + def _get_make_cmd(self): allowed_toolchains = ['buildroot'] - cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) - ld = common.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains) - if args.arch == 'x86_64': + cc = self.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) + ld = self.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains) + if self.env['arch'] == 'x86_64': arch = 'x86' else: - arch = args.arch + arch = self.env['arch'] return [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, - '-f', 'Makefile.{}'.format(arch), common.Newline, - 'CC={}'.format(cc), common.Newline, - 'LD={}'.format(ld), common.Newline, - 'PWD={}'.format(common.gem5_m5_src_dir), common.Newline, + 'make', LF, + '-j', str(self.env['nproc']), LF, + '-f', 'Makefile.{}'.format(arch), LF, + 'CC={}'.format(cc), LF, + 'LD={}'.format(ld), LF, + 'PWD={}'.format(self.env['gem5_m5_source_dir']), LF, ] - def do_build(self, args): - os.makedirs(common.gem5_m5_build_dir, exist_ok=True) + def build(self): + os.makedirs(self.env['gem5_m5_build_dir'], exist_ok=True) # We must clean first or else the build outputs of one arch can conflict with the other. # I should stop being lazy and go actually patch gem5 to support out of tree m5 build... - self.clean(args) - common.run_cmd( - self.get_make_cmd(args), - cwd=common.gem5_m5_src_dir, + self.clean() + self.sh.run_cmd( + self._get_make_cmd(), + cwd=self.env['gem5_m5_source_dir'], ) - os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True) - common.cp(os.path.join(common.gem5_m5_src_dir, 'm5'), common.out_rootfs_overlay_bin_dir) + os.makedirs(self.env['out_rootfs_overlay_bin_dir'], exist_ok=True) + self.sh.cp(os.path.join(self.env['gem5_m5_source_dir'], 'm5'), self.env['out_rootfs_overlay_bin_dir']) - def clean(self, args): - common.run_cmd( - self.get_make_cmd(args) + ['clean', common.Newline], - cwd=common.gem5_m5_src_dir, + def clean(self): + self.sh.run_cmd( + self._get_make_cmd() + ['clean', LF], + cwd=self.env['gem5_m5_source_dir'], ) + return None if __name__ == '__main__': - M5Component().build() + Main().cli() diff --git a/build-modules b/build-modules index 73432e5..b8aae9c 100755 --- a/build-modules +++ b/build-modules @@ -6,32 +6,37 @@ import platform import shutil import common +from shell_helpers import LF -class ModulesComponent(common.Component): - def add_parser_arguments(self, parser): - parser.add_argument( +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ +Build our Linux kernel modules without using Buildroot. + +See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host +''') + self.add_argument( '--make-args', default='', ) - parser.add_argument( + self.add_argument( '--host', - action='store_true', default=False, help='''\ Build the Linux kernel modules for the host instead of guest. Use the host packaged cross toolchain. ''', ) - parser.add_argument( - 'kernel_modules', + self.add_argument( + 'kernel-modules', default=[], help='Which kernel modules to build. Default: build all', - metavar='kernel-modules', nargs='*', ) - def do_build(self, args): - build_dir = self.get_build_dir(args) + def build(self): + build_dir = self.get_build_dir() os.makedirs(build_dir, exist_ok=True) # I kid you not, out-of-tree build is not possible, O= does not work as for the kernel build: # @@ -42,87 +47,78 @@ Use the host packaged cross toolchain. # This copies only modified files as per: # https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory distutils.dir_util.copy_tree( - common.kernel_modules_src_dir, - os.path.join(build_dir, common.kernel_modules_subdir), + self.env['kernel_modules_src_dir'], + os.path.join(build_dir, self.env['kernel_modules_subdir']), update=1, ) distutils.dir_util.copy_tree( - common.include_src_dir, - os.path.join(build_dir, common.include_subdir), + self.env['include_src_dir'], + os.path.join(build_dir, self.env['include_subdir']), update=1, ) all_kernel_modules = [] - for basename in os.listdir(common.kernel_modules_src_dir): - src = os.path.join(common.kernel_modules_src_dir, basename) + for basename in os.listdir(self.env['kernel_modules_src_dir']): + src = os.path.join(self.env['kernel_modules_src_dir'], basename) if os.path.isfile(src): noext, ext = os.path.splitext(basename) - if ext == common.c_ext: + if ext == self.env['c_ext']: all_kernel_modules.append(noext) - if args.kernel_modules == []: + if self.env['kernel_modules'] == []: kernel_modules = all_kernel_modules else: - kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], args.kernel_modules) - object_files = map(lambda x: x + common.obj_ext, kernel_modules) + kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], self.env['kernel_modules']) + object_files = map(lambda x: x + self.env['obj_ext'], kernel_modules) tool = 'gcc' - if args.host: + if self.env['host']: allowed_toolchains = ['host'] - build_subdir = common.kernel_modules_build_host_subdir + build_subdir = self.env['kernel_modules_build_host_subdir'] else: allowed_toolchains = None - build_subdir = common.kernel_modules_build_subdir - gcc = common.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains) + build_subdir = self.env['kernel_modules_build_subdir'] + gcc = self.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains) prefix = gcc[:-len(tool)] ccache = shutil.which('ccache') if ccache is not None: cc = '{} {}'.format(ccache, gcc) else: cc = gcc - if args.verbose: + if self.env['verbose']: verbose = ['V=1'] else: verbose = [] - if args.host: + if self.env['host']: linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build') else: - linux_dir = common.linux_build_dir - common.run_cmd( + linux_dir = self.env['linux_build_dir'] + self.sh.run_cmd( ( [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, - 'ARCH={}'.format(common.linux_arch), common.Newline, - 'CC={}'.format(cc), common.Newline, - 'CROSS_COMPILE={}'.format(prefix), common.Newline, - 'LINUX_DIR={}'.format(linux_dir), common.Newline, - 'M={}'.format(build_subdir), common.Newline, - 'OBJECT_FILES={}'.format(' '.join(object_files)), common.Newline, + 'make', LF, + '-j', str(self.env['nproc']), LF, + 'ARCH={}'.format(self.env['linux_arch']), LF, + 'CC={}'.format(cc), LF, + 'CROSS_COMPILE={}'.format(prefix), LF, + 'LINUX_DIR={}'.format(linux_dir), LF, + 'M={}'.format(build_subdir), LF, + 'OBJECT_FILES={}'.format(' '.join(object_files)), LF, ] + - common.shlex_split(args.make_args) + + self.sh.shlex_split(self.env['make_args']) + verbose ), - cwd=os.path.join(common.kernel_modules_build_subdir), + cwd=os.path.join(self.env['kernel_modules_build_subdir']), ) - if not args.host: - common.copy_dir_if_update_non_recursive( - srcdir=common.kernel_modules_build_subdir, - destdir=common.out_rootfs_overlay_dir, - filter_ext=common.kernel_module_ext, + if not self.env['host']: + self.sh.copy_dir_if_update_non_recursive( + srcdir=self.env['kernel_modules_build_subdir'], + destdir=self.env['out_rootfs_overlay_dir'], + filter_ext=self.env['kernel_module_ext'], ) - def get_argparse_args(self): - return { - 'description': '''\ -Build our Linux kernel modules without using Buildroot. - -See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host -''' - } - - def get_build_dir(self, args): - if args.host: - return os.path.join(common.kernel_modules_build_host_dir) + def get_build_dir(self): + if self.env['host']: + return self.env['kernel_modules_build_host_dir'] else: - return os.path.join(common.kernel_modules_build_dir) + return self.env['kernel_modules_build_dir'] if __name__ == '__main__': - ModulesComponent().build() + Main().cli() diff --git a/build-qemu b/build-qemu index 5299d07..21a53ff 100755 --- a/build-qemu +++ b/build-qemu @@ -3,61 +3,62 @@ import os import common +from shell_helpers import LF -class QemuComponent(common.Component): - def add_parser_arguments(self, parser): - parser.add_argument( +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__() + self.add_argument( '--userland', default=False, - action='store_true', help='Build QEMU user mode instead of system.', ) - parser.add_argument( + self.add_argument( 'extra_config_args', default=[], metavar='extra-config-args', nargs='*' ) - def do_build(self, args): - build_dir = self.get_build_dir(args) + def build(self): + build_dir = self.get_build_dir() os.makedirs(build_dir, exist_ok=True) - if args.verbose: + if self.env['verbose']: verbose = ['V=1'] else: verbose = [] - if args.userland: - target_list = '{}-linux-user'.format(args.arch) + if self.env['userland']: + target_list = '{}-linux-user'.format(self.env['arch']) else: - target_list = '{}-softmmu'.format(args.arch) - common.run_cmd( + target_list = '{}-softmmu'.format(self.env['arch']) + self.sh.run_cmd( [ - os.path.join(common.qemu_src_dir, 'configure'), common.Newline, - '--enable-debug', common.Newline, - '--enable-trace-backends=simple', common.Newline, - '--target-list={}'.format(target_list), common.Newline, - '--enable-sdl', common.Newline, - '--with-sdlabi=2.0', common.Newline, + os.path.join(self.env['qemu_src_dir'], 'configure'), LF, + '--enable-debug', LF, + '--enable-trace-backends=simple', LF, + '--target-list={}'.format(target_list), LF, + '--enable-sdl', LF, + '--with-sdlabi=2.0', LF, ] + - common.add_newlines(args.extra_config_args), - extra_paths=[common.ccache_dir], + self.sh.add_newlines(self.env['extra_config_args']), + extra_paths=[self.env['ccache_dir']], cwd=build_dir ) - common.run_cmd( + self.sh.run_cmd( ( [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, + 'make', LF, + '-j', str(self.env['nproc']), LF, ] + verbose ), cwd=build_dir, - extra_paths=[common.ccache_dir], + extra_paths=[self.env['ccache_dir']], ) - def get_build_dir(self, args): - return common.qemu_build_dir + def get_build_dir(self): + return self.env['qemu_build_dir'] if __name__ == '__main__': - QemuComponent().build() + Main().cli() diff --git a/build-userland b/build-userland index 3a7e527..02f7123 100755 --- a/build-userland +++ b/build-userland @@ -1,16 +1,19 @@ #!/usr/bin/env python3 import os -import platform import shlex -import shutil -import subprocess import common +from shell_helpers import LF -class UserlandComponent(common.Component): - def add_parser_arguments(self, parser): - parser.add_argument( +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ +Build our compiled userland examples. +''' + ) + self.add_argument( '--has-package', action='append', default=[], @@ -19,20 +22,19 @@ Indicate that a given package is present in the root filesystem, which allows us to build examples that rely on it. ''', ) - parser.add_argument( + self.add_argument( '--host', - action='store_true', default=False, help='''\ Build the userland programs for the host instead of guest. Use the host packaged cross toolchain. ''', ) - parser.add_argument( + self.add_argument( '--make-args', default='', ) - parser.add_argument( + self.add_argument( 'targets', default=[], help='''\ @@ -44,49 +46,44 @@ has the OpenBLAS libraries and headers installed. nargs='*', ) - def do_build(self, args): - build_dir = self.get_build_dir(args) + def build(self): + build_dir = self.get_build_dir() os.makedirs(build_dir, exist_ok=True) - if args.host: + if self.env['host']: allowed_toolchains = ['host'] else: allowed_toolchains = ['buildroot'] - cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) - cxx = common.get_toolchain_tool('g++', allowed_toolchains=allowed_toolchains) - common.run_cmd( + cc = self.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) + cxx = self.get_toolchain_tool('g++', allowed_toolchains=allowed_toolchains) + self.sh.run_cmd( ( [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, - 'ARCH={}'.format(args.arch), common.Newline, - 'CCFLAGS_SCRIPT={} {}'.format('-I', common.userland_src_dir), common.Newline, - 'COMMON_DIR={}'.format(common.root_dir), common.Newline, - 'CC={}'.format(cc), common.Newline, - 'CXX={}'.format(cxx), common.Newline, - 'PKG_CONFIG={}'.format(common.buildroot_pkg_config), common.Newline, - 'STAGING_DIR={}'.format(common.buildroot_staging_dir), common.Newline, - 'OUT_DIR={}'.format(build_dir), common.Newline, + 'make', LF, + '-j', str(self.env['nproc']), LF, + 'ARCH={}'.format(self.env['arch']), LF, + 'CCFLAGS_SCRIPT={} {}'.format('-I', self.env['userland_src_dir']), LF, + 'COMMON_DIR={}'.format(self.env['root_dir']), LF, + 'CC={}'.format(cc), LF, + 'CXX={}'.format(cxx), LF, + 'PKG_CONFIG={}'.format(self.env['buildroot_pkg_config']), LF, + 'STAGING_DIR={}'.format(self.env['buildroot_staging_dir']), LF, + 'OUT_DIR={}'.format(build_dir), LF, ] + - common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in args.has_package]) + - shlex.split(args.make_args) + - common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets]) + self.sh.add_newlines(['HAS_{}=y'.format(package.upper()) for package in self.env['has_package']]) + + shlex.split(self.env['make_args']) + + self.sh.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + self.env['userland_build_ext'] for target in self.env['targets']]) ), - cwd=common.userland_src_dir, - extra_paths=[common.ccache_dir], + cwd=self.env['userland_src_dir'], + extra_paths=[self.env['ccache_dir']], ) - common.copy_dir_if_update_non_recursive( + self.sh.copy_dir_if_update_non_recursive( srcdir=build_dir, - destdir=common.out_rootfs_overlay_dir, - filter_ext=common.userland_build_ext, + destdir=self.env['out_rootfs_overlay_dir'], + filter_ext=self.env['userland_build_ext'], ) - def get_argparse_args(self): - return { - 'description': 'Build our compiled userland examples', - } - - def get_build_dir(self, args): - return common.userland_build_dir + def get_build_dir(self): + return self.env['userland_build_dir'] if __name__ == '__main__': - UserlandComponent().build() + Main().cli() diff --git a/cli_function.py b/cli_function.py new file mode 100755 index 0000000..e92e113 --- /dev/null +++ b/cli_function.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 + +import argparse +import imp +import os + +class Argument: + def __init__( + self, + long_or_short_1, + long_or_short_2=None, + default=None, + help=None, + nargs=None, + **kwargs + ): + if long_or_short_2 is None: + shortname = None + longname = long_or_short_1 + else: + shortname = long_or_short_1 + longname = long_or_short_2 + self.args = [] + # argparse is crappy and cannot tell us if arguments were given or not. + # We need that information to decide if the config file should override argparse or not. + # So we just use None as a sentinel. + self.kwargs = {'default': None} + if shortname is not None: + self.args.append(shortname) + if longname[0] == '-': + self.args.append(longname) + self.key = longname.lstrip('-').replace('-', '_') + self.is_option = True + else: + self.key = longname.replace('-', '_') + self.args.append(self.key) + self.kwargs['metavar'] = longname + self.is_option = False + if default is not None and nargs is None: + self.kwargs['nargs'] = '?' + if nargs is not None: + self.kwargs['nargs'] = nargs + if default is True: + bool_action = 'store_false' + self.is_bool = True + elif default is False: + bool_action = 'store_true' + self.is_bool = True + else: + self.is_bool = False + if default is None and nargs in ('*', '+'): + default = [] + if self.is_bool and not 'action' in kwargs: + self.kwargs['action'] = bool_action + if help is not None: + if not self.is_bool and default: + help += ' Default: {}'.format(default) + self.kwargs['help'] = help + self.optional = ( + default is not None or + self.is_bool or + self.is_option or + nargs in ('?', '*', '+') + ) + self.kwargs.update(kwargs) + self.default = default + self.longname = longname + + def __str__(self): + return str(self.args) + ' ' + str(self.kwargs) + +class CliFunction: + ''' + Represent a function that can be called either from Python code, or + from the command line. + + Features: + + * single argument description in format very similar to argparse + * handle default arguments transparently in both cases + * expose a configuration file mechanism to get default parameters from a file + * fix some argparse.ArgumentParser() annoyances: + ** allow dashes in positional arguments: + https://stackoverflow.com/questions/12834785/having-options-in-argparse-with-a-dash + ** boolean defaults automatically use store_true or store_false, and add a --no-* CLI + option to invert them if set from the config + + This somewhat duplicates: https://click.palletsprojects.com but: + + * that decorator API is insane + * CLI + Python for single functions was wontfixed: https://github.com/pallets/click/issues/40 + ''' + def __call__(self, **args): + ''' + Python version of the function call. + + :type arguments: Dict + ''' + args_with_defaults = args.copy() + # Add missing args from config file. + if 'config_file' in args_with_defaults and args_with_defaults['config_file'] is not None: + config_file = args_with_defaults['config_file'] + else: + config_file = self._config_file + if os.path.exists(config_file): + config_configs = {} + config = imp.load_source('config', config_file) + config.set_args(config_configs) + for key in config_configs: + if key not in self._all_keys: + raise Exception('Unknown key in config file: ' + key) + if (not key in args_with_defaults) or args_with_defaults[key] is None: + args_with_defaults[key] = config_configs[key] + # Add missing args from hard-coded defaults. + for argument in self._arguments: + key = argument.key + if (not key in args_with_defaults) or args_with_defaults[key] is None: + if argument.optional: + args_with_defaults[key] = argument.default + else: + raise Exception('Value not given for mandatory argument: ' + key) + return self.main(**args_with_defaults) + + def __init__(self, config_file=None, description=None): + self._all_keys = set() + self._arguments = [] + self._config_file = config_file + self._description = description + if self._config_file is not None: + self.add_argument( + '--config-file', + default=self._config_file, + help='Path to the configuration file to use' + ) + + def __str__(self): + return '\n'.join(str(arg) for arg in self._arguments) + + def add_argument( + self, + *args, + **kwargs + ): + argument = Argument(*args, **kwargs) + self._arguments.append(argument) + self._all_keys.add(argument.key) + + def cli(self, cli_args=None): + ''' + Call the function from the CLI. Parse command line arguments + to get all arguments. + ''' + parser = argparse.ArgumentParser( + description=self._description, + formatter_class=argparse.RawTextHelpFormatter, + ) + for argument in self._arguments: + parser.add_argument(*argument.args, **argument.kwargs) + if argument.is_bool: + new_longname = '--no' + argument.longname[1:] + kwargs = argument.kwargs.copy() + kwargs['default'] = not argument.default + if kwargs['action'] == 'store_false': + kwargs['action'] = 'store_true' + elif kwargs['action'] == 'store_true': + kwargs['action'] = 'store_false' + if 'help' in kwargs: + del kwargs['help'] + parser.add_argument(new_longname, dest=argument.key, **kwargs) + args = parser.parse_args(args=cli_args) + return self(**vars(args)) + + def main(self, **kwargs): + ''' + Do the main function call work. + + :type arguments: Dict + ''' + raise NotImplementedError + +if __name__ == '__main__': + class OneCliFunction(CliFunction): + def __init__(self): + super().__init__( + config_file='cli_function_test_config.py', + description = '''\ +Description of this +amazing function! +''', + ) + self.add_argument('-a', '--asdf', default='A', help='Help for asdf'), + self.add_argument('-q', '--qwer', default='Q', help='Help for qwer'), + self.add_argument('-b', '--bool', default=True, help='Help for bool'), + self.add_argument('--bool-cli', default=False, help='Help for bool'), + self.add_argument('--bool-nargs', default=False, nargs='?', action='store', const='') + self.add_argument('--no-default', help='Help for no-bool'), + self.add_argument('pos-mandatory', help='Help for pos-mandatory', type=int), + self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int), + self.add_argument('args-star', help='Help for args-star', nargs='*'), + def main(self, **kwargs): + del kwargs['config_file'] + return kwargs + + one_cli_function = OneCliFunction() + + # Default code call. + default = one_cli_function(pos_mandatory=1) + assert default == { + 'asdf': 'A', + 'qwer': 'Q', + 'bool': True, + 'bool_nargs': False, + 'bool_cli': True, + 'no_default': None, + 'pos_mandatory': 1, + 'pos_optional': 0, + 'args_star': [] + } + + # Default CLI call. + out = one_cli_function.cli(['1']) + assert out == default + + # asdf + out = one_cli_function(pos_mandatory=1, asdf='B') + assert out['asdf'] == 'B' + out['asdf'] = default['asdf'] + assert(out == default) + + # asdf and qwer + out = one_cli_function(pos_mandatory=1, asdf='B', qwer='R') + assert out['asdf'] == 'B' + assert out['qwer'] == 'R' + out['asdf'] = default['asdf'] + out['qwer'] = default['qwer'] + assert(out == default) + + if '--bool': + out = one_cli_function(pos_mandatory=1, bool=False) + cli_out = one_cli_function.cli(['--bool', '1']) + assert out == cli_out + assert out['bool'] == False + out['bool'] = default['bool'] + assert(out == default) + + if '--bool-nargs': + + out = one_cli_function(pos_mandatory=1, bool_nargs=True) + assert out['bool_nargs'] == True + out['bool_nargs'] = default['bool_nargs'] + assert(out == default) + + out = one_cli_function(pos_mandatory=1, bool_nargs='asdf') + assert out['bool_nargs'] == 'asdf' + out['bool_nargs'] = default['bool_nargs'] + assert(out == default) + + # Force a boolean value set on the config to be False on CLI. + assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False + + # CLI call. + print(one_cli_function.cli()) diff --git a/cli_function_test_config.py b/cli_function_test_config.py new file mode 100644 index 0000000..ccf7d1c --- /dev/null +++ b/cli_function_test_config.py @@ -0,0 +1,5 @@ +def set_args(args): + ''' + :type args: Dict[str, Any] + ''' + args['bool_cli'] = True diff --git a/common.py b/common.py index db9f702..930ab7e 100644 --- a/common.py +++ b/common.py @@ -5,279 +5,182 @@ import base64 import collections import copy import datetime -import distutils.file_util import glob import imp -import itertools import json import multiprocessing import os import re -import shlex import shutil import signal -import stat import subprocess import sys import time import urllib import urllib.request +import cli_function +import shell_helpers +from shell_helpers import LF + common = sys.modules[__name__] -repo_short_id = 'lkmc' + +# Fixed parameters that don't depend on CLI arguments. +consts = {} +consts['repo_short_id'] = 'lkmc' # https://stackoverflow.com/questions/20010199/how-to-determine-if-a-process-runs-inside-lxc-docker -in_docker = os.path.exists('/.dockerenv') -root_dir = os.path.dirname(os.path.abspath(__file__)) -data_dir = os.path.join(root_dir, 'data') -p9_dir = os.path.join(data_dir, '9p') -gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5') -if in_docker: - out_dir = os.path.join(root_dir, 'out.docker') +consts['in_docker'] = os.path.exists('/.dockerenv') +consts['root_dir'] = os.path.dirname(os.path.abspath(__file__)) +consts['data_dir'] = os.path.join(consts['root_dir'], 'data') +consts['p9_dir'] = os.path.join(consts['data_dir'], '9p') +consts['gem5_non_default_src_root_dir'] = os.path.join(consts['data_dir'], 'gem5') +if consts['in_docker']: + consts['out_dir'] = os.path.join(consts['root_dir'], 'out.docker') else: - out_dir = os.path.join(root_dir, 'out') -bench_boot = os.path.join(out_dir, 'bench-boot.txt') -packages_dir = os.path.join(root_dir, 'buildroot_packages') -kernel_modules_subdir = 'kernel_modules' -kernel_modules_src_dir = os.path.join(common.root_dir, common.kernel_modules_subdir) -userland_subdir = 'userland' -userland_src_dir = os.path.join(common.root_dir, common.userland_subdir) -userland_build_ext = '.out' -include_subdir = 'include' -include_src_dir = os.path.join(common.root_dir, common.include_subdir) -submodules_dir = os.path.join(root_dir, 'submodules') -buildroot_src_dir = os.path.join(submodules_dir, 'buildroot') -crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng') -crosstool_ng_supported_archs = set(['arm', 'aarch64']) -linux_config_dir = os.path.join(common.root_dir, 'linux_config') -rootfs_overlay_dir = os.path.join(common.root_dir, 'rootfs_overlay') -qemu_src_dir = os.path.join(submodules_dir, 'qemu') -parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark') -ccache_dir = os.path.join('/usr', 'lib', 'ccache') -default_build_id = 'default' -arch_short_to_long_dict = collections.OrderedDict([ + consts['out_dir'] = os.path.join(consts['root_dir'], 'out') +consts['gem5_out_dir'] = os.path.join(consts['out_dir'], 'gem5') +consts['kernel_modules_build_base_dir'] = os.path.join(consts['out_dir'], 'kernel_modules') +consts['buildroot_out_dir'] = os.path.join(consts['out_dir'], 'buildroot') +consts['gem5_m5_build_dir'] = os.path.join(consts['out_dir'], 'util', 'm5') +consts['run_dir_base'] = os.path.join(consts['out_dir'], 'run') +consts['crosstool_ng_out_dir'] = os.path.join(consts['out_dir'], 'crosstool-ng') +consts['bench_boot'] = os.path.join(consts['out_dir'], 'bench-boot.txt') +consts['packages_dir'] = os.path.join(consts['root_dir'], 'buildroot_packages') +consts['kernel_modules_subdir'] = 'kernel_modules' +consts['kernel_modules_src_dir'] = os.path.join(consts['root_dir'], consts['kernel_modules_subdir']) +consts['userland_subdir'] = 'userland' +consts['userland_src_dir'] = os.path.join(consts['root_dir'], consts['userland_subdir']) +consts['userland_build_ext'] = '.out' +consts['include_subdir'] = 'include' +consts['include_src_dir'] = os.path.join(consts['root_dir'], consts['include_subdir']) +consts['submodules_dir'] = os.path.join(consts['root_dir'], 'submodules') +consts['buildroot_src_dir'] = os.path.join(consts['submodules_dir'], 'buildroot') +consts['crosstool_ng_src_dir'] = os.path.join(consts['submodules_dir'], 'crosstool-ng') +consts['crosstool_ng_supported_archs'] = set(['arm', 'aarch64']) +consts['linux_source_dir'] = os.path.join(consts['submodules_dir'], 'linux') +consts['linux_config_dir'] = os.path.join(consts['root_dir'], 'linux_config') +consts['gem5_default_src_dir'] = os.path.join(consts['submodules_dir'], 'gem5') +consts['rootfs_overlay_dir'] = os.path.join(consts['root_dir'], 'rootfs_overlay') +consts['extract_vmlinux'] = os.path.join(consts['linux_source_dir'], 'scripts', 'extract-vmlinux') +consts['qemu_src_dir'] = os.path.join(consts['submodules_dir'], 'qemu') +consts['parsec_benchmark_src_dir'] = os.path.join(consts['submodules_dir'], 'parsec-benchmark') +consts['ccache_dir'] = os.path.join('/usr', 'lib', 'ccache') +consts['default_build_id'] = 'default' +consts['arch_short_to_long_dict'] = collections.OrderedDict([ ('x', 'x86_64'), ('a', 'arm'), ('A', 'aarch64'), ]) -all_archs = [arch_short_to_long_dict[k] for k in arch_short_to_long_dict] -arch_choices = [] -for key in common.arch_short_to_long_dict: - arch_choices.append(key) - arch_choices.append(common.arch_short_to_long_dict[key]) -default_arch = 'x86_64' -gem5_cpt_prefix = '^cpt\.' +consts['all_archs'] = [consts['arch_short_to_long_dict'][k] for k in consts['arch_short_to_long_dict']] +consts['arch_choices'] = [] +for key in consts['arch_short_to_long_dict']: + consts['arch_choices'].append(key) + consts['arch_choices'].append(consts['arch_short_to_long_dict'][key]) +consts['default_arch'] = 'x86_64' +consts['gem5_cpt_prefix'] = '^cpt\.' def git_sha(repo_path): return subprocess.check_output(['git', '-C', repo_path, 'log', '-1', '--format=%H']).decode().rstrip() -sha = common.git_sha(root_dir) -release_dir = os.path.join(common.out_dir, 'release') -release_zip_file = os.path.join(common.release_dir, 'lkmc-{}.zip'.format(common.sha)) -github_repo_id = 'cirosantilli/linux-kernel-module-cheat' -asm_ext = '.S' -c_ext = '.c' -header_ext = '.h' -kernel_module_ext = '.ko' -obj_ext = '.o' -config_file = os.path.join(data_dir, 'config') -command_prefix = '+ ' -magic_fail_string = b'lkmc_test_fail' -if os.path.exists(config_file): - config = imp.load_source('config', config_file) - configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')} +consts['sha'] = common.git_sha(consts['root_dir']) +consts['release_dir'] = os.path.join(consts['out_dir'], 'release') +consts['release_zip_file'] = os.path.join(consts['release_dir'], 'lkmc-{}.zip'.format(consts['sha'])) +consts['github_repo_id'] = 'cirosantilli/linux-kernel-module-cheat' +consts['asm_ext'] = '.S' +consts['c_ext'] = '.c' +consts['header_ext'] = '.h' +consts['kernel_module_ext'] = '.ko' +consts['obj_ext'] = '.o' +consts['config_file'] = os.path.join(consts['data_dir'], 'config.py') +consts['magic_fail_string'] = b'lkmc_test_fail' +consts['baremetal_lib_basename'] = 'lib' -class Component: - def __init__(self): - pass - - def build(self): - ''' - Parse CLI, and to the build based on it. - - The actual build work is done by do_build in implementing classes. - ''' - parser = common.get_argparse( - argparse_args=self.get_argparse_args(), - default_args=self.get_default_args(), - ) - self.add_parser_arguments(parser) - parser.add_argument( - '--clean', - help='Clean the build instead of building.', - action='store_true', - ) - parser.add_argument( - '-j', '--nproc', - help='Number of processors to use for the build. Default: use all cores.', - type=int, - default=multiprocessing.cpu_count(), - ) - args = common.setup(parser) - if not common.dry_run: - start_time = time.time() - if args.clean: - self.clean(args) - else: - self.do_build(args) - if not common.dry_run: - end_time = time.time() - common.print_time(end_time - start_time) - - def add_parser_arguments(self, parser): - pass - - def clean(self, args): - build_dir = self.get_build_dir(args) - if build_dir is not None: - common.rmrf(build_dir) - - def do_build(self, args): - ''' - Do the actual main build work. - ''' - raise NotImplementedError() - - def get_argparse_args(self): - ''' - Extra arguments for argparse.ArgumentParser. - ''' - return {} - - def get_build_dir(self, args): - ''' - Build directory, gets cleaned by --clean if not None. - ''' - return None - - def get_default_args(self): - ''' - Default values for command line arguments. - ''' - return {} - -class Newline: +class LkmcCliFunction(cli_function.CliFunction): ''' - Singleton class. Can be used in print_cmd to print out nicer command lines - with -key on the same line as "-key value". - ''' - pass + Common functionality shared across our CLI functions: -def add_dry_run_argument(parser): - parser.add_argument('--dry-run', default=False, action='store_true', help='''\ + * command timing + * some common flags, e.g.: --arch, --dry-run, --verbose + ''' + def __init__(self, *args, **kwargs): + kwargs['config_file'] = consts['config_file'] + super().__init__(*args, **kwargs) + + # Args for all scripts. + self.add_argument( + '-a', '--arch', choices=consts['arch_choices'], default=consts['default_arch'], + help='CPU architecture.' + ) + self.add_argument( + '--dry-run', + default=False, + help='''\ Print the commands that would be run, but don't run them. We aim display every command that modifies the filesystem state, and generate Bash equivalents even for actions taken directly in Python without shelling out. -mkdir are generally omitted since those are obvious. -''') +mkdir are generally omitted since those are obvious +''' + ) + self.add_argument( + '-v', '--verbose', default=False, + help='Show full compilation commands when they are not shown by default.' + ) -def add_newlines(cmd): - out = [] - for arg in cmd: - out.extend([arg, common.Newline]) - return out + # Gem5 args. + self.add_argument( + '--gem5-build-dir', + help='''\ +Use the given directory as the gem5 build directory. +''' + ) + self.add_argument( + '-M', '--gem5-build-id', default='default', + help='''\ +gem5 build ID. Allows you to keep multiple separate gem5 builds. +''' + ) + self.add_argument( + '--gem5-build-type', default='opt', + help='gem5 build type, most often used for "debug" builds.' + ) + self.add_argument( + '--gem5-source-dir', + help='''\ +Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`. +''' + ) + self.add_argument( + '-N', '--gem5-worktree', + help='''\ +Create and use a git worktree of the gem5 submodule. +See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree +''' + ) -def assert_crosstool_ng_supports_arch(arch): - if arch not in common.crosstool_ng_supported_archs: - raise Exception('arch not yet supported: ' + arch) + # Linux kernel. + self.add_argument( + '-L', '--linux-build-id', default=consts['default_build_id'], + help='''\ +Linux build ID. Allows you to keep multiple separate Linux builds. +''' + ) + self.add_argument( + '--linux-source-dir', + help='''\ +Use the given directory as the Linux source tree. +''' + ) + self.add_argument( + '--initramfs', default=False, + ) + self.add_argument( + '--initrd', default=False, + ) -def base64_encode(string): - return base64.b64encode(string.encode()).decode() - -def write_string_to_file(path, string, mode='w'): - if mode == 'a': - redirect = '>>' - else: - redirect = '>' - print_cmd("cat << 'EOF' {} {}\n{}\nEOF".format(redirect, path, string)) - if not common.dry_run: - with open(path, 'a') as f: - f.write(string) - -def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None): - ''' - Format a command given as a list of strings so that it can - be viewed nicely and executed by bash directly and print it to stdout. - ''' - last_newline = ' \\\n' - newline_separator = last_newline + ' ' - out = [] - if extra_env is None: - extra_env = {} - if cwd is not None: - out.append('cd {} &&'.format(shlex.quote(cwd))) - if extra_paths is not None: - out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths))) - for key in extra_env: - out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key]))) - cmd_quote = [] - newline_count = 0 - for arg in cmd: - if arg == common.Newline: - cmd_quote.append(arg) - newline_count += 1 - else: - cmd_quote.append(shlex.quote(arg)) - if newline_count > 0: - cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == common.Newline) if not x] - out.extend(cmd_quote) - if newline_count == 1 and cmd[-1] == common.Newline: - ending = '' - else: - ending = last_newline + ';' - return newline_separator.join(out) + ending - -def copy_dir_if_update_non_recursive(srcdir, destdir, filter_ext=None): - # TODO print rsync equivalent. - os.makedirs(destdir, exist_ok=True) - for basename in os.listdir(srcdir): - src = os.path.join(srcdir, basename) - if os.path.isfile(src): - noext, ext = os.path.splitext(basename) - if filter_ext is not None and ext == filter_ext: - distutils.file_util.copy_file( - src, - os.path.join(destdir, basename), - update=1, - ) - -def cp(src, dest): - print_cmd(['cp', src, dest]) - if not common.dry_run: - shutil.copy2(src, dest) - -def gem_list_checkpoint_dirs(): - ''' - List checkpoint directory, oldest first. - ''' - prefix_re = re.compile(common.gem5_cpt_prefix) - files = list(filter(lambda x: os.path.isdir(os.path.join(common.m5out_dir, x)) and prefix_re.search(x), os.listdir(common.m5out_dir))) - files.sort(key=lambda x: os.path.getmtime(os.path.join(common.m5out_dir, x))) - return files - -def get_argparse(default_args=None, argparse_args=None): - ''' - Return an argument parser with common arguments set. - - :type default_args: Dict[str,str] - :type argparse_args: Dict - ''' - if default_args is None: - default_args = {} - if argparse_args is None: - argparse_args = {} - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, - **argparse_args - ) - common.add_dry_run_argument(parser) - emulator_group = parser.add_mutually_exclusive_group(required=False) - parser.add_argument( - '-a', '--arch', choices=common.arch_choices, default=common.default_arch, - help='CPU architecture. Default: %(default)s' - ) - parser.add_argument( - '-b', '--baremetal', - help='''\ + # Baremetal. + self.add_argument( + '-b', '--baremetal', + help='''\ Use the given baremetal executable instead of the Linux kernel. If the path is absolute, it is used as is. @@ -285,753 +188,619 @@ If the path is absolute, it is used as is. If the path is relative, we assume that it points to a source code inside baremetal/ and then try to use corresponding executable. ''' - ) - parser.add_argument( - '--buildroot-build-id', - default=default_build_id, - help='Buildroot build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s' - ) - parser.add_argument( - '--buildroot-linux', default=False, action='store_true', - help='Boot with the Buildroot Linux kernel instead of our custom built one. Mostly for sanity checks.' - ) - parser.add_argument( - '--crosstool-ng-build-id', default=default_build_id, - help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds. Default: %(default)s' - ) - parser.add_argument( - '--docker', default=False, action='store_true', - help='''\ + ) + + # Buildroot. + self.add_argument( + '--buildroot-build-id', + default=consts['default_build_id'], + help='Buildroot build ID. Allows you to keep multiple separate gem5 builds.' + ) + self.add_argument( + '--buildroot-linux', default=False, + help='Boot with the Buildroot Linux kernel instead of our custom built one. Mostly for sanity checks.' + ) + + # crosstool-ng + self.add_argument( + '--crosstool-ng-build-id', default=consts['default_build_id'], + help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds.' + ) + self.add_argument( + '--docker', default=False, + help='''\ Use the docker download Ubuntu root filesystem instead of the default Buildroot one. ''' - ) - parser.add_argument( - '--dp650', default=False, action='store_true' - ) - parser.add_argument( - '-L', '--linux-build-id', default=default_build_id, - help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s' - ) - parser.add_argument( - '--linux-build-dir', - help='Select the directory that contains the Linux kernel build. Overrides linux-build-id.' - ) - parser.add_argument( - '--linux-source-dir', - help='''\ -Use the given directory as the Linux source tree. -''' - ) - parser.add_argument( - '--machine', - help='''Machine type. + ) + + self.add_argument( + '--machine', + help='''Machine type. QEMU default: virt gem5 default: VExpress_GEM5_V1 See the documentation for other values known to work. ''' - ) - emulator_group.add_argument( - '-g', '--gem5', default=False, action='store_true', - help='Use gem5 instead of QEMU. Default: False' - ) - parser.add_argument( - '--gem5-build-dir', - help='''\ -Use the given directory as the gem5 build directory. -''' - ) - parser.add_argument( - '-M', '--gem5-build-id', - help='''\ -gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: {} -'''.format(default_build_id) - ) - parser.add_argument( - '--gem5-build-type', default='opt', - help='gem5 build type, most often used for "debug" builds. Default: %(default)s' - ) - parser.add_argument( - '--gem5-source-dir', - help='''\ -Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`. -''' - ) - parser.add_argument( - '-N', '--gem5-worktree', - help='''\ -Create and use a git worktree of the gem5 submodule. -See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree -''' - ) - parser.add_argument( - '-n', '--run-id', default='0', - help='''\ + ) + + # QEMU. + self.add_argument( + '-Q', '--qemu-build-id', default=consts['default_build_id'], + help='QEMU build ID. Allows you to keep multiple separate QEMU builds.' + ) + + # Userland. + self.add_argument( + '--userland-build-id', default=None + ) + + # Run. + self.add_argument( + '-n', '--run-id', default='0', + help='''\ ID for run outputs such as gem5's m5out. Allows you to do multiple runs, and then inspect separate outputs later in different output directories. -Default: %(default)s ''' - ) - parser.add_argument( - '-P', '--prebuilt', default=False, action='store_true', - help='''\ + ) + self.add_argument( + '-P', '--prebuilt', default=False, + help='''\ Use prebuilt packaged host utilities as much as possible instead of the ones we built ourselves. Saves build time, but decreases the likelihood of incompatibilities. ''' - ) - parser.add_argument( - '--port-offset', type=int, - help='''\ + ) + self.add_argument( + '--port-offset', type=int, + help='''\ Increase the ports to be used such as for GDB by an offset to run multiple -instances in parallel. -Default: the run ID (-n) if that is an integer, otherwise 0. +instances in parallel. Default: the run ID (-n) if that is an integer, otherwise 0. ''' - ) - emulator_group.add_argument( - '--qemu', default=False, action='store_true', - help='''\ + ) + + # Misc. + self.add_argument( + '-g', '--gem5', default=False, + help='Use gem5 instead of QEMU.' + ) + self.add_argument( + '--qemu', default=False, + help='''\ Use QEMU as the emulator. This option exists in addition to --gem5 to allow overriding configs from the CLI. ''' - ) - parser.add_argument( - '-Q', '--qemu-build-id', default=default_build_id, - help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s' - ) - parser.add_argument( - '--userland-build-id', default=None - ) - parser.add_argument( - '-v', '--verbose', default=False, action='store_true', - help='Show full compilation commands when they are not shown by default.' - ) - if hasattr(common, 'configs'): - defaults = common.configs.copy() - else: - defaults = {} - defaults.update(default_args) - # A bit ugly as it actually changes the defaults shown on --help, but we can't do any better - # because it is impossible to check if arguments were given or not... - # - https://stackoverflow.com/questions/30487767/check-if-argparse-optional-argument-is-set-or-not - # - https://stackoverflow.com/questions/3609852/which-is-the-best-way-to-allow-configuration-options-be-overridden-at-the-comman - parser.set_defaults(**defaults) - return parser + ) -def get_elf_entry(elf_file_path): - readelf_header = subprocess.check_output([ - common.get_toolchain_tool('readelf'), - '-h', - elf_file_path - ]) - for line in readelf_header.decode().split('\n'): - split = line.split() - if line.startswith(' Entry point address:'): - addr = line.split()[-1] - break - return int(addr, 0) - -def get_stats(stat_re=None, stats_file=None): - if stat_re is None: - stat_re = '^system.cpu[0-9]*.numCycles$' - if stats_file is None: - stats_file = common.stats_file - stat_re = re.compile(stat_re) - ret = [] - with open(stats_file, 'r') as statfile: - for line in statfile: - if line[0] != '-': - cols = line.split() - if len(cols) > 1 and stat_re.search(cols[0]): - ret.append(cols[1]) - return ret - -def get_toolchain_prefix(tool, allowed_toolchains=None): - buildroot_full_prefix = os.path.join(common.host_bin_dir, common.buildroot_toolchain_prefix) - buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool)) - crosstool_ng_full_prefix = os.path.join(common.crosstool_ng_bin_dir, common.crosstool_ng_toolchain_prefix) - crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool)) - host_tool = '{}-{}'.format(common.ubuntu_toolchain_prefix, tool) - host_path = shutil.which(host_tool) - if host_path is not None: - host_exists = True - host_full_prefix = host_path[:-(len(tool)+1)] - else: - host_exists = False - host_full_prefix = None - known_toolchains = { - 'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix), - 'buildroot': (buildroot_exists, buildroot_full_prefix), - 'host': (host_exists, host_full_prefix), - } - if allowed_toolchains is None: - if common.baremetal is None: - allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] + def _init_env(self, env): + ''' + Update the kwargs from the command line with derived arguments. + ''' + def join(*paths): + return os.path.join(*paths) + if env['qemu'] or not env['gem5']: + env['emulator'] = 'qemu' else: - allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] - tried = [] - for toolchain in allowed_toolchains: - exists, prefix = known_toolchains[toolchain] - tried.append('{}-{}'.format(prefix, tool)) - if exists: - return prefix - raise Exception('Tool not found. Tried:\n' + '\n'.join(tried)) + env['emulator'] = 'gem5' + if env['arch'] in env['arch_short_to_long_dict']: + env['arch'] = env['arch_short_to_long_dict'][env['arch']] + if env['userland_build_id'] is None: + env['userland_build_id'] = env['default_build_id'] + env['userland_build_id_given'] = False + else: + env['userland_build_id_given'] = True + if env['gem5_worktree'] is not None and env['gem5_build_id'] is None: + env['gem5_build_id'] = env['gem5_worktree'] + env['is_arm'] = False + if env['arch'] == 'arm': + env['armv'] = 7 + env['gem5_arch'] = 'ARM' + env['mcpu'] = 'cortex-a15' + env['buildroot_toolchain_prefix'] = 'arm-buildroot-linux-uclibcgnueabihf' + env['crosstool_ng_toolchain_prefix'] = 'arm-unknown-eabi' + env['ubuntu_toolchain_prefix'] = 'arm-linux-gnueabihf' + if env['emulator'] == 'gem5': + if env['machine'] is None: + env['machine'] = 'VExpress_GEM5_V1' + else: + if env['machine'] is None: + env['machine'] = 'virt' + env['is_arm'] = True + elif env['arch'] == 'aarch64': + env['armv'] = 8 + env['gem5_arch'] = 'ARM' + env['mcpu'] = 'cortex-a57' + env['buildroot_toolchain_prefix'] = 'aarch64-buildroot-linux-uclibc' + env['crosstool_ng_toolchain_prefix'] = 'aarch64-unknown-elf' + env['ubuntu_toolchain_prefix'] = 'aarch64-linux-gnu' + if env['emulator'] == 'gem5': + if env['machine'] is None: + env['machine'] = 'VExpress_GEM5_V1' + else: + if env['machine'] is None: + env['machine'] = 'virt' + env['is_arm'] = True + elif env['arch'] == 'x86_64': + env['crosstool_ng_toolchain_prefix'] = 'x86_64-unknown-elf' + env['gem5_arch'] = 'X86' + env['buildroot_toolchain_prefix'] = 'x86_64-buildroot-linux-uclibc' + env['ubuntu_toolchain_prefix'] = 'x86_64-linux-gnu' + if env['emulator'] == 'gem5': + if env['machine'] is None: + env['machine'] = 'TODO' + else: + if env['machine'] is None: + env['machine'] = 'pc' -def get_toolchain_tool(tool, allowed_toolchains=None): - return '{}-{}'.format(common.get_toolchain_prefix(tool, allowed_toolchains), tool) + # Buildroot + env['buildroot_build_dir'] = join(env['buildroot_out_dir'], 'build', env['buildroot_build_id'], env['arch']) + env['buildroot_download_dir'] = join(env['buildroot_out_dir'], 'download') + env['buildroot_config_file'] = join(env['buildroot_build_dir'], '.config') + env['buildroot_build_build_dir'] = join(env['buildroot_build_dir'], 'build') + env['buildroot_linux_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom') + env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], "vmlinux") + env['host_dir'] = join(env['buildroot_build_dir'], 'host') + env['host_bin_dir'] = join(env['host_dir'], 'usr', 'bin') + env['buildroot_pkg_config'] = join(env['host_bin_dir'], 'pkg-config') + env['buildroot_images_dir'] = join(env['buildroot_build_dir'], 'images') + env['buildroot_rootfs_raw_file'] = join(env['buildroot_images_dir'], 'rootfs.ext2') + env['buildroot_qcow2_file'] = env['buildroot_rootfs_raw_file'] + '.qcow2' + env['staging_dir'] = join(env['out_dir'], 'staging', env['arch']) + env['buildroot_staging_dir'] = join(env['buildroot_build_dir'], 'staging') + env['target_dir'] = join(env['buildroot_build_dir'], 'target') + if env['linux_source_dir'] is None: + env['linux_source_dir'] = os.path.join(consts['submodules_dir'], 'linux') + common.extract_vmlinux = os.path.join(env['linux_source_dir'], 'scripts', 'extract-vmlinux') + env['linux_buildroot_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom') -def github_make_request( - authenticate=False, - data=None, - extra_headers=None, - path='', - subdomain='api', - url_params=None, - **extra_request_args - ): - if extra_headers is None: - extra_headers = {} - headers = {'Accept': 'application/vnd.github.v3+json'} - headers.update(extra_headers) - if authenticate: - headers['Authorization'] = 'token ' + os.environ['LKMC_GITHUB_TOKEN'] - if url_params is not None: - path += '?' + urllib.parse.urlencode(url_params) - request = urllib.request.Request( - 'https://' + subdomain + '.github.com/repos/' + github_repo_id + path, - headers=headers, - data=data, - **extra_request_args - ) - response_body = urllib.request.urlopen(request).read().decode() - if response_body: - _json = json.loads(response_body) - else: - _json = {} - return _json + # QEMU + env['qemu_build_dir'] = join(env['out_dir'], 'qemu', env['qemu_build_id']) + env['qemu_executable_basename'] = 'qemu-system-{}'.format(env['arch']) + env['qemu_executable'] = join(env['qemu_build_dir'], '{}-softmmu'.format(env['arch']), env['qemu_executable_basename']) + env['qemu_img_basename'] = 'qemu-img' + env['qemu_img_executable'] = join(env['qemu_build_dir'], env['qemu_img_basename']) -def log_error(msg): - print('error: {}'.format(msg), file=sys.stderr) + # gem5 + if env['gem5_build_dir'] is None: + env['gem5_build_dir'] = join(env['gem5_out_dir'], env['gem5_build_id'], env['gem5_build_type']) + env['gem5_fake_iso'] = join(env['gem5_out_dir'], 'fake.iso') + env['gem5_m5term'] = join(env['gem5_build_dir'], 'm5term') + env['gem5_build_build_dir'] = join(env['gem5_build_dir'], 'build') + env['gem5_executable'] = join(env['gem5_build_build_dir'], env['gem5_arch'], 'gem5.{}'.format(env['gem5_build_type'])) + env['gem5_system_dir'] = join(env['gem5_build_dir'], 'system') -def make_build_dirs(): - os.makedirs(common.buildroot_build_build_dir, exist_ok=True) - os.makedirs(common.gem5_build_dir, exist_ok=True) - os.makedirs(common.out_rootfs_overlay_dir, exist_ok=True) + # gem5 source + if env['gem5_source_dir'] is not None: + assert os.path.exists(env['gem5_source_dir']) + else: + if env['gem5_worktree'] is not None: + env['gem5_source_dir'] = join(env['gem5_non_default_src_root_dir'], env['gem5_worktree']) + else: + env['gem5_source_dir'] = env['gem5_default_src_dir'] + env['gem5_m5_source_dir'] = join(env['gem5_source_dir'], 'util', 'm5') + env['gem5_config_dir'] = join(env['gem5_source_dir'], 'configs') + env['gem5_se_file'] = join(env['gem5_config_dir'], 'example', 'se.py') + env['gem5_fs_file'] = join(env['gem5_config_dir'], 'example', 'fs.py') -def make_run_dirs(): - ''' - Make directories required for the run. - The user could nuke those anytime between runs to try and clean things up. - ''' - os.makedirs(common.gem5_run_dir, exist_ok=True) - os.makedirs(common.p9_dir, exist_ok=True) - os.makedirs(common.qemu_run_dir, exist_ok=True) + # crosstool-ng + env['crosstool_ng_buildid_dir'] = join(env['crosstool_ng_out_dir'], 'build', env['crosstool_ng_build_id']) + env['crosstool_ng_install_dir'] = join(env['crosstool_ng_buildid_dir'], 'install', env['arch']) + env['crosstool_ng_bin_dir'] = join(env['crosstool_ng_install_dir'], 'bin') + env['crosstool_ng_util_dir'] = join(env['crosstool_ng_buildid_dir'], 'util') + env['crosstool_ng_config'] = join(env['crosstool_ng_util_dir'], '.config') + env['crosstool_ng_defconfig'] = join(env['crosstool_ng_util_dir'], 'defconfig') + env['crosstool_ng_executable'] = join(env['crosstool_ng_util_dir'], 'ct-ng') + env['crosstool_ng_build_dir'] = join(env['crosstool_ng_buildid_dir'], 'build') + env['crosstool_ng_download_dir'] = join(env['crosstool_ng_out_dir'], 'download') -def need_rebuild(srcs, dst): - if not os.path.exists(dst): - return True - for src in srcs: - if os.path.getmtime(src) > os.path.getmtime(dst): + # run + env['gem5_run_dir'] = join(env['run_dir_base'], 'gem5', env['arch'], str(env['run_id'])) + env['m5out_dir'] = join(env['gem5_run_dir'], 'm5out') + env['stats_file'] = join(env['m5out_dir'], 'stats.txt') + env['gem5_trace_txt_file'] = join(env['m5out_dir'], 'trace.txt') + env['gem5_guest_terminal_file'] = join(env['m5out_dir'], 'system.terminal') + env['gem5_readfile'] = join(env['gem5_run_dir'], 'readfile') + env['gem5_termout_file'] = join(env['gem5_run_dir'], 'termout.txt') + env['qemu_run_dir'] = join(env['run_dir_base'], 'qemu', env['arch'], str(env['run_id'])) + env['qemu_termout_file'] = join(env['qemu_run_dir'], 'termout.txt') + env['qemu_trace_basename'] = 'trace.bin' + env['qemu_trace_file'] = join(env['qemu_run_dir'], 'trace.bin') + env['qemu_trace_txt_file'] = join(env['qemu_run_dir'], 'trace.txt') + env['qemu_rrfile'] = join(env['qemu_run_dir'], 'rrfile') + env['gem5_out_dir'] = join(env['out_dir'], 'gem5') + + # Ports + if env['port_offset'] is None: + try: + env['port_offset'] = int(env['run_id']) + except ValueError: + env['port_offset'] = 0 + if env['emulator'] == 'gem5': + env['gem5_telnet_port'] = 3456 + env['port_offset'] + env['gdb_port'] = 7000 + env['port_offset'] + else: + env['qemu_base_port'] = 45454 + 10 * env['port_offset'] + env['qemu_monitor_port'] = env['qemu_base_port'] + 0 + env['qemu_hostfwd_generic_port'] = env['qemu_base_port'] + 1 + env['qemu_hostfwd_ssh_port'] = env['qemu_base_port'] + 2 + env['qemu_gdb_port'] = env['qemu_base_port'] + 3 + env['extra_serial_port'] = env['qemu_base_port'] + 4 + env['gdb_port'] = env['qemu_gdb_port'] + env['qemu_background_serial_file'] = join(env['qemu_run_dir'], 'background.log') + + # gem5 QEMU polymorphism. + if env['emulator'] == 'gem5': + env['executable'] = env['gem5_executable'] + env['run_dir'] = env['gem5_run_dir'] + env['termout_file'] = env['gem5_termout_file'] + env['guest_terminal_file'] = env['gem5_guest_terminal_file'] + env['trace_txt_file'] = env['gem5_trace_txt_file'] + else: + env['executable'] = env['qemu_executable'] + env['run_dir'] = env['qemu_run_dir'] + env['termout_file'] = env['qemu_termout_file'] + env['guest_terminal_file'] = env['qemu_termout_file'] + env['trace_txt_file'] = env['qemu_trace_txt_file'] + env['run_cmd_file'] = join(env['run_dir'], 'run.sh') + + # Linux kernl. + if 'linux_build_id' in env: + env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch']) + env['lkmc_vmlinux'] = join(env['linux_build_dir'], "vmlinux") + if env['arch'] == 'arm': + env['linux_arch'] = 'arm' + env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage') + elif env['arch'] == 'aarch64': + env['linux_arch'] = 'arm64' + env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'Image') + elif env['arch'] == 'x86_64': + env['linux_arch'] = 'x86' + env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'bzImage') + env['lkmc_linux_image'] = join(env['linux_build_dir'], env['linux_image_prefix']) + env['buildroot_linux_image'] = join(env['buildroot_linux_build_dir'], env['linux_image_prefix']) + if env['buildroot_linux']: + env['vmlinux'] = env['buildroot_vmlinux'] + env['linux_image'] = env['buildroot_linux_image'] + else: + env['vmlinux'] = env['lkmc_vmlinux'] + env['linux_image'] = env['lkmc_linux_image'] + + # Kernel modules. + env['kernel_modules_build_dir'] = join(env['kernel_modules_build_base_dir'], env['arch']) + env['kernel_modules_build_subdir'] = join(env['kernel_modules_build_dir'], env['kernel_modules_subdir']) + env['kernel_modules_build_host_dir'] = join(env['kernel_modules_build_base_dir'], 'host') + env['kernel_modules_build_host_subdir'] = join(env['kernel_modules_build_host_dir'], env['kernel_modules_subdir']) + env['userland_build_dir'] = join(env['out_dir'], 'userland', env['userland_build_id'], env['arch']) + env['out_rootfs_overlay_dir'] = join(env['out_dir'], 'rootfs_overlay', env['arch']) + env['out_rootfs_overlay_bin_dir'] = join(env['out_rootfs_overlay_dir'], 'bin') + + # Baremetal. + env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal') + env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename']) + if env['emulator'] == 'gem5': + env['simulator_name'] = 'gem5' + else: + env['simulator_name'] = 'qemu' + env['baremetal_build_dir'] = join(env['out_dir'], 'baremetal', env['arch'], env['simulator_name'], env['machine']) + env['baremetal_build_lib_dir'] = join(env['baremetal_build_dir'], env['baremetal_lib_basename']) + env['baremetal_build_ext'] = '.elf' + + # Docker + env['docker_build_dir'] = join(env['out_dir'], 'docker', env['arch']) + env['docker_tar_dir'] = join(env['docker_build_dir'], 'export') + env['docker_tar_file'] = join(env['docker_build_dir'], 'export.tar') + env['docker_rootfs_raw_file'] = join(env['docker_build_dir'], 'export.ext2') + env['docker_qcow2_file'] = join(env['docker_rootfs_raw_file'] + '.qcow2') + if env['docker']: + env['rootfs_raw_file'] = env['docker_rootfs_raw_file'] + env['qcow2_file'] = env['docker_qcow2_file'] + else: + env['rootfs_raw_file'] = env['buildroot_rootfs_raw_file'] + env['qcow2_file'] = env['buildroot_qcow2_file'] + + # Image. + if env['baremetal'] is None: + if env['emulator'] == 'gem5': + env['image'] = env['vmlinux'] + env['disk_image'] = env['rootfs_raw_file'] + else: + env['image'] = env['linux_image'] + env['disk_image'] = env['qcow2_file'] + else: + env['disk_image'] = env['gem5_fake_iso'] + if env['baremetal'] == 'all': + path = env['baremetal'] + else: + path = self.resolve_executable( + env['baremetal'], + env['baremetal_src_dir'], + env['baremetal_build_dir'], + env['baremetal_build_ext'], + ) + source_path_noext = os.path.splitext(join( + env['baremetal_src_dir'], + os.path.relpath(path, env['baremetal_build_dir']) + ))[0] + for ext in [env['c_ext'], env['asm_ext']]: + source_path = source_path_noext + ext + if os.path.exists(source_path): + env['source_path'] = source_path + break + env['image'] = path + self.env = env + + def assert_crosstool_ng_supports_arch(self, arch): + if arch not in self.env['crosstool_ng_supported_archs']: + raise Exception('arch not yet supported: ' + arch) + + @staticmethod + def base64_encode(string): + return base64.b64encode(string.encode()).decode() + + def gem_list_checkpoint_dirs(self): + ''' + List checkpoint directory, oldest first. + ''' + prefix_re = re.compile(self.env['gem5_cpt_prefix']) + files = list(filter(lambda x: os.path.isdir(os.path.join(self.env['m5out_dir'], x)) and prefix_re.search(x), os.listdir(self.env['m5out_dir']))) + files.sort(key=lambda x: os.path.getmtime(os.path.join(self.env['m5out_dir'], x))) + return files + + def get_elf_entry(self, elf_file_path): + readelf_header = subprocess.check_output([ + self.get_toolchain_tool('readelf'), + '-h', + elf_file_path + ]) + for line in readelf_header.decode().split('\n'): + split = line.split() + if line.startswith(' Entry point address:'): + addr = line.split()[-1] + break + return int(addr, 0) + + def get_stats(self, stat_re=None, stats_file=None): + if stat_re is None: + stat_re = '^system.cpu[0-9]*.numCycles$' + if stats_file is None: + stats_file = self.env['stats_file'] + stat_re = re.compile(stat_re) + ret = [] + with open(stats_file, 'r') as statfile: + for line in statfile: + if line[0] != '-': + cols = line.split() + if len(cols) > 1 and stat_re.search(cols[0]): + ret.append(cols[1]) + return ret + + def get_toolchain_prefix(self, tool, allowed_toolchains=None): + buildroot_full_prefix = os.path.join(self.env['host_bin_dir'], self.env['buildroot_toolchain_prefix']) + buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool)) + crosstool_ng_full_prefix = os.path.join(self.env['crosstool_ng_bin_dir'], self.env['crosstool_ng_toolchain_prefix']) + crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool)) + host_tool = '{}-{}'.format(self.env['ubuntu_toolchain_prefix'], tool) + host_path = shutil.which(host_tool) + if host_path is not None: + host_exists = True + host_full_prefix = host_path[:-(len(tool)+1)] + else: + host_exists = False + host_full_prefix = None + known_toolchains = { + 'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix), + 'buildroot': (buildroot_exists, buildroot_full_prefix), + 'host': (host_exists, host_full_prefix), + } + if allowed_toolchains is None: + if self.env['baremetal'] is None: + allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] + else: + allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] + tried = [] + for toolchain in allowed_toolchains: + exists, prefix = known_toolchains[toolchain] + tried.append('{}-{}'.format(prefix, tool)) + if exists: + return prefix + raise Exception('Tool not found. Tried:\n' + '\n'.join(tried)) + + def get_toolchain_tool(self, tool, allowed_toolchains=None): + return '{}-{}'.format(self.get_toolchain_prefix(tool, allowed_toolchains), tool) + + @staticmethod + def github_make_request( + authenticate=False, + data=None, + extra_headers=None, + path='', + subdomain='api', + url_params=None, + **extra_request_args + ): + if extra_headers is None: + extra_headers = {} + headers = {'Accept': 'application/vnd.github.v3+json'} + headers.update(extra_headers) + if authenticate: + headers['Authorization'] = 'token ' + os.environ['LKMC_GITHUB_TOKEN'] + if url_params is not None: + path += '?' + urllib.parse.urlencode(url_params) + request = urllib.request.Request( + 'https://' + subdomain + '.github.com/repos/' + github_repo_id + path, + headers=headers, + data=data, + **extra_request_args + ) + response_body = urllib.request.urlopen(request).read().decode() + if response_body: + _json = json.loads(response_body) + else: + _json = {} + return _json + + @staticmethod + def log_error(msg): + print('error: {}'.format(msg), file=sys.stderr) + + def main(self, **kwargs): + ''' + Time the main of the derived class. + ''' + if not kwargs['dry_run']: + start_time = time.time() + kwargs.update(consts) + self._init_env(kwargs) + self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run']) + ret = self.timed_main() + if not kwargs['dry_run']: + end_time = time.time() + self.print_time(end_time - start_time) + return ret + + def make_build_dirs(self): + os.makedirs(self.env['buildroot_build_build_dir'], exist_ok=True) + os.makedirs(self.env['gem5_build_dir'], exist_ok=True) + os.makedirs(self.env['out_rootfs_overlay_dir'], exist_ok=True) + + def make_run_dirs(self): + ''' + Make directories required for the run. + The user could nuke those anytime between runs to try and clean things up. + ''' + os.makedirs(self.env['gem5_run_dir'], exist_ok=True) + os.makedirs(self.env['p9_dir'], exist_ok=True) + os.makedirs(self.env['qemu_run_dir'], exist_ok=True) + + @staticmethod + def need_rebuild(srcs, dst): + if not os.path.exists(dst): return True - return False + for src in srcs: + if os.path.getmtime(src) > os.path.getmtime(dst): + return True + return False -def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): - ''' - Print cmd_to_string to stdout. + @staticmethod + def print_time(ellapsed_seconds): + hours, rem = divmod(ellapsed_seconds, 3600) + minutes, seconds = divmod(rem, 60) + print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) - Optionally save the command to cmd_file file, and add extra_env - environment variables to the command generated. - - If cmd contains at least one common.Newline, newlines are only added on common.Newline. - Otherwise, newlines are added automatically after every word. - ''' - global dry_run - if type(cmd) is str: - cmd_string = cmd - else: - cmd_string = cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths) - print(common.command_prefix + cmd_string) - if cmd_file is not None: - with open(cmd_file, 'w') as f: - f.write('#!/usr/bin/env bash\n') - f.write(cmd_string) - st = os.stat(cmd_file) - os.chmod(cmd_file, st.st_mode | stat.S_IXUSR) - -def print_time(ellapsed_seconds): - hours, rem = divmod(ellapsed_seconds, 3600) - minutes, seconds = divmod(rem, 60) - print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) - -def raw_to_qcow2(prebuilt=False, reverse=False): - if prebuilt or not os.path.exists(common.qemu_img_executable): - disable_trace = [] - qemu_img_executable = common.qemu_img_basename - else: - # Prevent qemu-img from generating trace files like QEMU. Disgusting. - disable_trace = ['-T', 'pr_manager_run,file=/dev/null', common.Newline,] - qemu_img_executable = common.qemu_img_executable - infmt = 'raw' - outfmt = 'qcow2' - infile = common.rootfs_raw_file - outfile = common.qcow2_file - if reverse: - tmp = infmt - infmt = outfmt - outfmt = tmp - tmp = infile - infile = outfile - outfile = tmp - common.run_cmd( - [ - qemu_img_executable, common.Newline, - ] + - disable_trace + - [ - 'convert', common.Newline, - '-f', infmt, common.Newline, - '-O', outfmt, common.Newline, - infile, common.Newline, - outfile, common.Newline, - ] - ) - -def resolve_args(defaults, args, extra_args): - if extra_args is None: - extra_args = {} - argcopy = copy.copy(args) - argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items())) - return argcopy - -def rmrf(path): - print_cmd(['rm', '-r', '-f', path]) - if not common.dry_run and os.path.exists(path): - shutil.rmtree(path) - -def run_cmd( - cmd, - cmd_file=None, - out_file=None, - show_stdout=True, - show_cmd=True, - extra_env=None, - extra_paths=None, - delete_env=None, - dry_run=False, - raise_on_failure=True, - **kwargs - ): - ''' - Run a command. Write the command to stdout before running it. - - Wait until the command finishes execution. - - :param cmd: command to run. common.Newline entries are magic get skipped. - :type cmd: List[str] - - :param cmd_file: if not None, write the command to be run to that file - :type cmd_file: str - - :param out_file: if not None, write the stdout and stderr of the command the file - :type out_file: str - - :param show_stdout: wether to show stdout and stderr on the terminal or not - :type show_stdout: bool - - :param extra_env: extra environment variables to add when running the command - :type extra_env: Dict[str,str] - - :param dry_run: don't run the commands, just potentially print them. Debug aid. - :type dry_run: Bool - ''' - if out_file is not None: - stdout = subprocess.PIPE - stderr = subprocess.STDOUT - else: - if show_stdout: - stdout = None - stderr = None + def raw_to_qcow2(self, prebuilt=False, reverse=False): + if prebuilt or not os.path.exists(self.env['qemu_img_executable']): + disable_trace = [] + qemu_img_executable = self.env['qemu_img_basename'] else: - stdout = subprocess.DEVNULL - stderr = subprocess.DEVNULL - if extra_env is None: - extra_env = {} - if delete_env is None: - delete_env = [] - if 'cwd' in kwargs: - cwd = kwargs['cwd'] - else: - cwd = None - env = os.environ.copy() - env.update(extra_env) - if extra_paths is not None: - path = ':'.join(extra_paths) - if 'PATH' in os.environ: - path += ':' + os.environ['PATH'] - env['PATH'] = path - for key in delete_env: - if key in env: - del env[key] - if show_cmd: - print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) + # Prevent qemu-img from generating trace files like QEMU. Disgusting. + disable_trace = ['-T', 'pr_manager_run,file=/dev/null', LF] + qemu_img_executable = self.env['qemu_img_executable'] + infmt = 'raw' + outfmt = 'qcow2' + infile = self.env['rootfs_raw_file'] + outfile = self.env['qcow2_file'] + if reverse: + tmp = infmt + infmt = outfmt + outfmt = tmp + tmp = infile + infile = outfile + outfile = tmp + self.sh.run_cmd( + [ + qemu_img_executable, LF, + ] + + disable_trace + + [ + 'convert', LF, + '-f', infmt, LF, + '-O', outfmt, LF, + infile, LF, + outfile, LF, + ] + ) - # Otherwise Ctrl + C gives: - # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). - # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec - sigint_old = signal.getsignal(signal.SIGINT) - signal.signal(signal.SIGINT, signal.SIG_IGN) + @staticmethod + def resolve_args(defaults, args, extra_args): + if extra_args is None: + extra_args = {} + argcopy = copy.copy(args) + argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items())) + return argcopy - # Otherwise BrokenPipeError when piping through | grep - # But if I do this_module, my terminal gets broken at the end. Why, why, why. - # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python - # Ignoring the exception is not enough as it prints a warning anyways. - #sigpipe_old = signal.getsignal(signal.SIGPIPE) - #signal.signal(signal.SIGPIPE, signal.SIG_DFL) + @staticmethod + def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext): + if os.path.isabs(in_path): + return in_path + else: + paths = [ + os.path.join(magic_out_dir, in_path), + os.path.join( + magic_out_dir, + os.path.relpath(in_path, magic_in_dir), + ) + ] + paths[:] = [os.path.splitext(path)[0] + out_ext for path in paths] + for path in paths: + if os.path.exists(path): + return path + raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths)) - cmd = common.strip_newlines(cmd) - if not dry_run and not common.dry_run: - # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 - with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: - if out_file is not None: - os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True) - with open(out_file, 'bw') as logfile: - while True: - byte = proc.stdout.read(1) - if byte: - if show_stdout: - sys.stdout.buffer.write(byte) - try: - sys.stdout.flush() - except BlockingIOError: - # TODO understand. Why, Python, why. - pass - logfile.write(byte) - else: - break - signal.signal(signal.SIGINT, sigint_old) - #signal.signal(signal.SIGPIPE, sigpipe_old) - returncode = proc.returncode - if returncode != 0 and raise_on_failure: - raise Exception('Command exited with status: {}'.format(returncode)) - return returncode - else: - return 0 + def resolve_userland(self, path): + return self.resolve_executable( + path, + self.env['userland_src_dir'], + self.env['userland_build_dir'], + self.env['userland_build_ext'], + ) -def setup(parser): + def timed_main(self): + ''' + Main action of the derived class. + ''' + raise NotImplementedError() + +class BuildCliFunction(LkmcCliFunction): ''' - Parse the command line arguments, and setup several variables based on them. - Typically done after getting inputs from the command line arguments. + A CLI function with common facilities to build stuff, e.g.: + + * `--clean` to clean the build directory + * `--nproc` to set he number of build threads ''' - args = parser.parse_args() - if args.qemu or not args.gem5: - common.emulator = 'qemu' - else: - common.emulator = 'gem5' - if args.arch in common.arch_short_to_long_dict: - args.arch = common.arch_short_to_long_dict[args.arch] - if args.gem5_build_id is None: - args.gem5_build_id = default_build_id - gem5_build_id_given = False - else: - gem5_build_id_given = True - if args.userland_build_id is None: - args.userland_build_id = default_build_id - common.userland_build_id_given = False - else: - common.userland_build_id_given = True - if args.gem5_worktree is not None and not gem5_build_id_given: - args.gem5_build_id = args.gem5_worktree - common.machine = args.machine - common.setup_dry_run_arguments(args) - common.is_arm = False - if args.arch == 'arm': - common.armv = 7 - common.gem5_arch = 'ARM' - common.mcpu = 'cortex-a15' - common.buildroot_toolchain_prefix = 'arm-buildroot-linux-uclibcgnueabihf' - common.crosstool_ng_toolchain_prefix = 'arm-unknown-eabi' - common.ubuntu_toolchain_prefix = 'arm-linux-gnueabihf' - common.is_arm = True - elif args.arch == 'aarch64': - common.armv = 8 - common.gem5_arch = 'ARM' - common.mcpu = 'cortex-a57' - common.buildroot_toolchain_prefix = 'aarch64-buildroot-linux-uclibc' - common.crosstool_ng_toolchain_prefix = 'aarch64-unknown-elf' - common.ubuntu_toolchain_prefix = 'aarch64-linux-gnu' - common.is_arm = True - elif args.arch == 'x86_64': - common.crosstool_ng_toolchain_prefix = 'x86_64-unknown-elf' - common.gem5_arch = 'X86' - common.buildroot_toolchain_prefix = 'x86_64-buildroot-linux-uclibc' - common.ubuntu_toolchain_prefix = 'x86_64-linux-gnu' - if common.emulator == 'gem5': - if common.machine is None: - common.machine = 'TODO' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.add_argument( + '--clean', + default=False, + help='Clean the build instead of building.', + ), + self.add_argument( + '-j', + '--nproc', + default=multiprocessing.cpu_count(), + type=int, + help='Number of processors to use for the build.', + ) + + def clean(self): + build_dir = self.get_build_dir() + if build_dir is not None: + self.sh.rmrf(build_dir) + + def build(self): + ''' + Do the actual main build work. + ''' + raise NotImplementedError() + + def get_build_dir(self): + return None + + def timed_main(self): + ''' + Parse CLI, and to the build based on it. + + The actual build work is done by do_build in implementing classes. + ''' + if self.env['clean']: + return self.clean() else: - if common.machine is None: - common.machine = 'pc' - if is_arm: - if common.emulator == 'gem5': - if common.machine is None: - if args.dp650: - common.machine = 'VExpress_GEM5_V1_DPU' - else: - common.machine = 'VExpress_GEM5_V1' - else: - if common.machine is None: - common.machine = 'virt' - common.buildroot_out_dir = os.path.join(common.out_dir, 'buildroot') - common.buildroot_build_dir = os.path.join(common.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch) - common.buildroot_download_dir = os.path.join(common.buildroot_out_dir, 'download') - common.buildroot_config_file = os.path.join(common.buildroot_build_dir, '.config') - common.buildroot_build_build_dir = os.path.join(common.buildroot_build_dir, 'build') - common.buildroot_linux_build_dir = os.path.join(common.buildroot_build_build_dir, 'linux-custom') - common.buildroot_vmlinux = os.path.join(common.buildroot_linux_build_dir, "vmlinux") - common.qemu_build_dir = os.path.join(common.out_dir, 'qemu', args.qemu_build_id) - common.qemu_executable_basename = 'qemu-system-{}'.format(args.arch) - common.qemu_executable = os.path.join(common.qemu_build_dir, '{}-softmmu'.format(args.arch), common.qemu_executable_basename) - common.qemu_img_basename = 'qemu-img' - common.qemu_img_executable = os.path.join(common.qemu_build_dir, common.qemu_img_basename) - common.host_dir = os.path.join(common.buildroot_build_dir, 'host') - common.host_bin_dir = os.path.join(common.host_dir, 'usr', 'bin') - common.buildroot_pkg_config = os.path.join(common.host_bin_dir, 'pkg-config') - common.buildroot_images_dir = os.path.join(common.buildroot_build_dir, 'images') - common.buildroot_rootfs_raw_file = os.path.join(common.buildroot_images_dir, 'rootfs.ext2') - common.buildroot_qcow2_file = common.buildroot_rootfs_raw_file + '.qcow2' - common.staging_dir = os.path.join(common.out_dir, 'staging', args.arch) - common.buildroot_staging_dir = os.path.join(common.buildroot_build_dir, 'staging') - common.target_dir = os.path.join(common.buildroot_build_dir, 'target') - common.run_dir_base = os.path.join(common.out_dir, 'run') - common.gem5_run_dir = os.path.join(common.run_dir_base, 'gem5', args.arch, str(args.run_id)) - common.m5out_dir = os.path.join(common.gem5_run_dir, 'm5out') - common.stats_file = os.path.join(common.m5out_dir, 'stats.txt') - common.gem5_trace_txt_file = os.path.join(common.m5out_dir, 'trace.txt') - common.gem5_guest_terminal_file = os.path.join(common.m5out_dir, 'system.terminal') - common.gem5_readfile = os.path.join(common.gem5_run_dir, 'readfile') - common.gem5_termout_file = os.path.join(common.gem5_run_dir, 'termout.txt') - common.qemu_run_dir = os.path.join(common.run_dir_base, 'qemu', args.arch, str(args.run_id)) - common.qemu_trace_basename = 'trace.bin' - common.qemu_trace_file = os.path.join(common.qemu_run_dir, 'trace.bin') - common.qemu_trace_txt_file = os.path.join(common.qemu_run_dir, 'trace.txt') - common.qemu_termout_file = os.path.join(common.qemu_run_dir, 'termout.txt') - common.qemu_rrfile = os.path.join(common.qemu_run_dir, 'rrfile') - common.qemu_guest_terminal_file = os.path.join(common.m5out_dir, qemu_termout_file) - common.gem5_out_dir = os.path.join(common.out_dir, 'gem5') - if args.gem5_build_dir is None: - common.gem5_build_dir = os.path.join(common.gem5_out_dir, args.gem5_build_id, args.gem5_build_type) - else: - common.gem5_build_dir = args.gem5_build_dir - common.gem5_fake_iso = os.path.join(common.gem5_out_dir, 'fake.iso') - common.gem5_m5term = os.path.join(common.gem5_build_dir, 'm5term') - common.gem5_build_build_dir = os.path.join(common.gem5_build_dir, 'build') - common.gem5_executable = os.path.join(common.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type)) - common.gem5_system_dir = os.path.join(common.gem5_build_dir, 'system') - common.crosstool_ng_out_dir = os.path.join(common.out_dir, 'crosstool-ng') - common.crosstool_ng_buildid_dir = os.path.join(common.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id) - common.crosstool_ng_install_dir = os.path.join(common.crosstool_ng_buildid_dir, 'install', args.arch) - common.crosstool_ng_bin_dir = os.path.join(common.crosstool_ng_install_dir, 'bin') - common.crosstool_ng_util_dir = os.path.join(common.crosstool_ng_buildid_dir, 'util') - common.crosstool_ng_config = os.path.join(common.crosstool_ng_util_dir, '.config') - common.crosstool_ng_defconfig = os.path.join(common.crosstool_ng_util_dir, 'defconfig') - common.crosstool_ng_executable = os.path.join(common.crosstool_ng_util_dir, 'ct-ng') - common.crosstool_ng_build_dir = os.path.join(common.crosstool_ng_buildid_dir, 'build') - common.crosstool_ng_download_dir = os.path.join(common.crosstool_ng_out_dir, 'download') - common.gem5_default_src_dir = os.path.join(submodules_dir, 'gem5') - if args.gem5_source_dir is not None: - common.gem5_src_dir = args.gem5_source_dir - assert(os.path.exists(args.gem5_source_dir)) - else: - if args.gem5_worktree is not None: - common.gem5_src_dir = os.path.join(common.gem5_non_default_src_root_dir, args.gem5_worktree) - else: - common.gem5_src_dir = common.gem5_default_src_dir - common.gem5_m5_src_dir = os.path.join(common.gem5_src_dir, 'util', 'm5') - common.gem5_m5_build_dir = os.path.join(common.out_dir, 'util', 'm5') - if common.emulator == 'gem5': - common.executable = common.gem5_executable - common.run_dir = common.gem5_run_dir - common.termout_file = common.gem5_termout_file - common.guest_terminal_file = gem5_guest_terminal_file - common.trace_txt_file = gem5_trace_txt_file - else: - common.executable = common.qemu_executable - common.run_dir = common.qemu_run_dir - common.termout_file = common.qemu_termout_file - common.guest_terminal_file = qemu_guest_terminal_file - common.trace_txt_file = qemu_trace_txt_file - common.gem5_config_dir = os.path.join(common.gem5_src_dir, 'configs') - common.gem5_se_file = os.path.join(common.gem5_config_dir, 'example', 'se.py') - common.gem5_fs_file = os.path.join(common.gem5_config_dir, 'example', 'fs.py') - common.run_cmd_file = os.path.join(common.run_dir, 'run.sh') - - # Linux - if args.linux_source_dir is None: - common.linux_source_dir = os.path.join(submodules_dir, 'linux') - else: - common.linux_source_dir = args.linux_source_dir - common.extract_vmlinux = os.path.join(linux_source_dir, 'scripts', 'extract-vmlinux') - common.linux_buildroot_build_dir = os.path.join(common.buildroot_build_build_dir, 'linux-custom') - if args.linux_build_dir is None: - common.linux_build_dir = os.path.join(common.out_dir, 'linux', args.linux_build_id, args.arch) - else: - common.linux_build_dir = args.linux_build_dir - common.lkmc_vmlinux = os.path.join(common.linux_build_dir, 'vmlinux') - if args.arch == 'arm': - common.linux_arch = 'arm' - common.linux_image_prefix = os.path.join('arch', common.linux_arch, 'boot', 'zImage') - elif args.arch == 'aarch64': - common.linux_arch = 'arm64' - common.linux_image_prefix = os.path.join('arch', common.linux_arch, 'boot', 'Image') - elif args.arch == 'x86_64': - common.linux_arch = 'x86' - common.linux_image_prefix = os.path.join('arch', common.linux_arch, 'boot', 'bzImage') - common.lkmc_linux_image = os.path.join(common.linux_build_dir, common.linux_image_prefix) - common.buildroot_linux_image = os.path.join(common.buildroot_linux_build_dir, linux_image_prefix) - if args.buildroot_linux: - common.vmlinux = common.buildroot_vmlinux - common.linux_image = common.buildroot_linux_image - else: - common.vmlinux = common.lkmc_vmlinux - common.linux_image = common.lkmc_linux_image - - # Kernel modules. - common.kernel_modules_build_base_dir = os.path.join(common.out_dir, 'kernel_modules') - common.kernel_modules_build_dir = os.path.join(common.kernel_modules_build_base_dir, args.arch) - common.kernel_modules_build_subdir = os.path.join(common.kernel_modules_build_dir, kernel_modules_subdir) - common.kernel_modules_build_host_dir = os.path.join(common.kernel_modules_build_base_dir, 'host') - common.kernel_modules_build_host_subdir = os.path.join(common.kernel_modules_build_host_dir, kernel_modules_subdir) - common.userland_build_dir = os.path.join(common.out_dir, 'userland', args.userland_build_id, args.arch) - common.out_rootfs_overlay_dir = os.path.join(common.out_dir, 'rootfs_overlay', args.arch) - common.out_rootfs_overlay_bin_dir = os.path.join(common.out_rootfs_overlay_dir, 'bin') - - # Ports - if args.port_offset is None: - try: - args.port_offset = int(args.run_id) - except ValueError: - args.port_offset = 0 - if common.emulator == 'gem5': - common.gem5_telnet_port = 3456 + args.port_offset - common.gdb_port = 7000 + args.port_offset - else: - common.qemu_base_port = 45454 + 10 * args.port_offset - common.qemu_monitor_port = common.qemu_base_port + 0 - common.qemu_hostfwd_generic_port = common.qemu_base_port + 1 - common.qemu_hostfwd_ssh_port = common.qemu_base_port + 2 - common.qemu_gdb_port = common.qemu_base_port + 3 - common.extra_serial_port = common.qemu_base_port + 4 - common.gdb_port = common.qemu_gdb_port - common.qemu_background_serial_file = os.path.join(common.qemu_run_dir, 'background.log') - - # Baremetal. - common.baremetal = args.baremetal - common.baremetal_lib_basename = 'lib' - common.baremetal_src_dir = os.path.join(common.root_dir, 'baremetal') - common.baremetal_src_lib_dir = os.path.join(common.baremetal_src_dir, common.baremetal_lib_basename) - if common.emulator == 'gem5': - common.simulator_name = 'gem5' - else: - common.simulator_name = 'qemu' - common.baremetal_build_dir = os.path.join(out_dir, 'baremetal', args.arch, common.simulator_name, common.machine) - common.baremetal_build_lib_dir = os.path.join(common.baremetal_build_dir, common.baremetal_lib_basename) - common.baremetal_build_ext = '.elf' - - # Docker - common.docker_build_dir = os.path.join(common.out_dir, 'docker', args.arch) - common.docker_tar_dir = os.path.join(common.docker_build_dir, 'export') - common.docker_tar_file = os.path.join(common.docker_build_dir, 'export.tar') - common.docker_rootfs_raw_file = os.path.join(common.docker_build_dir, 'export.ext2') - common.docker_qcow2_file = os.path.join(common.docker_rootfs_raw_file + '.qcow2') - if args.docker: - common.rootfs_raw_file = common.docker_rootfs_raw_file - common.qcow2_file = common.docker_qcow2_file - else: - common.rootfs_raw_file = common.buildroot_rootfs_raw_file - common.qcow2_file = common.buildroot_qcow2_file - - # Image. - if common.baremetal is None: - if common.emulator == 'gem5': - common.image = common.vmlinux - common.disk_image = common.rootfs_raw_file - else: - common.image = common.linux_image - common.disk_image = common.qcow2_file - else: - common.disk_image = common.gem5_fake_iso - if common.baremetal == 'all': - path = common.baremetal - else: - path = common.resolve_executable( - common.baremetal, - common.baremetal_src_dir, - common.baremetal_build_dir, - common.baremetal_build_ext, - ) - source_path_noext = os.path.splitext(os.path.join( - common.baremetal_src_dir, - os.path.relpath(path, common.baremetal_build_dir) - ))[0] - for ext in [c_ext, asm_ext]: - source_path = source_path_noext + ext - if os.path.exists(source_path): - common.source_path = source_path - break - common.image = path - return args - -def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext): - if os.path.isabs(in_path): - return in_path - else: - paths = [ - os.path.join(magic_out_dir, in_path), - os.path.join( - magic_out_dir, - os.path.relpath(in_path, magic_in_dir), - ) - ] - paths[:] = [os.path.splitext(path)[0] + out_ext for path in paths] - for path in paths: - if os.path.exists(path): - return path - raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths)) - -def resolve_userland(path): - return common.resolve_executable( - path, - common.userland_src_dir, - common.userland_build_dir, - common.userland_build_ext, - ) - -def setup_dry_run_arguments(args): - common.dry_run = args.dry_run - -def shlex_split(string): - ''' - shlex_split, but also add Newline after every word. - - Not perfect since it does not group arguments, but I don't see a solution. - ''' - return common.add_newlines(shlex.split(string)) - -def strip_newlines(cmd): - return [x for x in cmd if x != common.Newline] - -def write_configs(config_path, configs, config_fragments=None): - """ - Write extra configs into the Buildroot config file. - TODO Can't get rid of these for now with nice fragments: - http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config - """ - if config_fragments is None: - config_fragments = [] - with open(config_path, 'a') as config_file: - for config_fragment in config_fragments: - with open(config_fragment, 'r') as config_fragment_file: - print_cmd(['cat', config_fragment, '>>', config_path]) - if not common.dry_run: - for line in config_fragment_file: - config_file.write(line) - write_string_to_file(config_path, '\n'.join(configs), mode='a') + return self.build() diff --git a/copy-overlay b/copy-overlay index bbf3470..115d92a 100755 --- a/copy-overlay +++ b/copy-overlay @@ -5,25 +5,26 @@ import os import shutil import common +from shell_helpers import LF -class CopyOverlayComponent(common.Component): - def do_build(self, args): - distutils.dir_util.copy_tree( - common.rootfs_overlay_dir, - common.out_rootfs_overlay_dir, - update=1, - ) - - def get_argparse_args(self): - return { - 'description': '''\ +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ Copy our git tracked rootfs_overlay to the final generated rootfs_overlay that also contains generated build outputs. This has the following advantages over just adding that to BR2_ROOTFS_OVERLAY: - also works for non Buildroot root filesystesms - places everything in one place for a nice 9P mount -''', - } +''') + + def build(self): + # TODO: print rsync equivalent, move into shell_helpers. + distutils.dir_util.copy_tree( + self.env['rootfs_overlay_dir'], + self.env['out_rootfs_overlay_dir'], + update=1, + ) if __name__ == '__main__': - CopyOverlayComponent().build() + Main().cli() diff --git a/gem5-shell b/gem5-shell index f669b57..66eca3f 100755 --- a/gem5-shell +++ b/gem5-shell @@ -3,14 +3,15 @@ import sys import common +from shell_helpers import LF -parser = common.get_argparse( +parser = self.get_argparse( default_args={'gem5':True}, argparse_args={'description':'Connect a terminal to a running gem5 instance'} ) -args = common.setup(parser) -sys.exit(common.run_cmd([ - common.gem5_m5term, common.Newline, - 'localhost', common.Newline, - str(common.gem5_telnet_port), common.Newline, +args = self.setup(parser) +sys.exit(self.sh.run_cmd([ + kwargs['gem5_m5term'], LF, + 'localhost', LF, + str(kwargs['gem5_telnet_port']), LF, ])) diff --git a/gem5-stat b/gem5-stat index 9b19b4c..0a42619 100755 --- a/gem5-stat +++ b/gem5-stat @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import common -parser = common.get_argparse( +parser = self.get_argparse( argparse_args={'description':'Get the value of a gem5 stat from the stats.txt file.'} ) parser.add_argument( @@ -9,6 +9,6 @@ parser.add_argument( help='Python regexp matching the full stat name of interest', nargs='?', ) -args = common.setup(parser) -stats = common.get_stats(args.stat) +args = self.setup(parser) +stats = self.get_stats(kwargs['stat']) print('\n'.join(stats)) diff --git a/getvar b/getvar index 431d4f9..0f6c04f 100755 --- a/getvar +++ b/getvar @@ -3,14 +3,15 @@ import types import common +from shell_helpers import LF -parser = common.get_argparse(argparse_args={ - 'description': '''Print the value of a common.py variable. +parser = self.get_argparse(argparse_args={ + 'description': '''Print the value of a kwargs['py'] variable. This is useful to: * give dry commands on the README that don't change when we refactor directory structure -* create simple bash scripts that call use common.py variables +* create simple bash scripts that call use kwargs['py'] variables For example, to get the Buildroot output directory for an ARM build, use: @@ -27,9 +28,9 @@ List all available variables: ''' }) parser.add_argument('variable', nargs='?') -args = common.setup(parser) -if args.variable: - print(getattr(common, args.variable)) +args = self.setup(parser) +if kwargs['variable']: + print(getattr(common, kwargs['variable'])) else: for attr in dir(common): if not attr.startswith('__'): diff --git a/qemu-monitor b/qemu-monitor index a2bcd9d..50e8f51 100755 --- a/qemu-monitor +++ b/qemu-monitor @@ -5,10 +5,11 @@ import sys import telnetlib import common +from shell_helpers import LF prompt = b'\n(qemu) ' -parser = common.get_argparse({ +parser = self.get_argparse({ 'description': '''\ Run a command on the QEMU monitor of a running QEMU instance @@ -21,24 +22,24 @@ parser.add_argument( help='If given, run this command and quit', nargs='*', ) -args = common.setup(parser) +args = self.setup(parser) def write_and_read(tn, cmd, prompt): tn.write(cmd.encode('utf-8')) return '\n'.join(tn.read_until(prompt).decode('utf-8').splitlines()[1:])[:-len(prompt)] -with telnetlib.Telnet('localhost', common.qemu_monitor_port) as tn: +with telnetlib.Telnet('localhost', kwargs['qemu_monitor_port']) as tn: # Couldn't disable server echo, so just removing the write for now. # https://stackoverflow.com/questions/12421799/how-to-disable-telnet-echo-in-python-telnetlib # sock = tn.get_socket() # sock.send(telnetlib.IAC + telnetlib.WILL + telnetlib.ECHO) if os.isatty(sys.stdin.fileno()): - if args.command == []: + if kwargs['command'] == []: print(tn.read_until(prompt).decode('utf-8'), end='') tn.interact() else: tn.read_until(prompt) - print(write_and_read(tn, ' '.join(args.command) + '\n', prompt)) + print(write_and_read(tn, ' '.join(kwargs['command']) + '\n', prompt)) else: tn.read_until(prompt) print(write_and_read(tn, sys.stdin.read() + '\n', prompt)) diff --git a/qemu-trace2txt b/qemu-trace2txt index fa4ab1f..c35c8b4 100755 --- a/qemu-trace2txt +++ b/qemu-trace2txt @@ -5,22 +5,23 @@ import subprocess import sys import common +from shell_helpers import LF def main(): - return common.run_cmd( + return self.sh.run_cmd( [ - os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), common.Newline, - os.path.join(common.qemu_build_dir, 'trace-events-all'), common.Newline, - os.path.join(common.qemu_trace_file), common.Newline, + os.path.join(kwargs['qemu_src_dir'], 'scripts/simpletrace.py'), LF, + os.path.join(kwargs['qemu_build_dir'], 'trace-events-all'), LF, + os.path.join(kwargs['qemu_trace_file']), LF, ], - cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'), - out_file=common.qemu_trace_txt_file, + cmd_file=os.path.join(kwargs['run_dir'], 'qemu-trace2txt'), + out_file=kwargs['qemu_trace_txt_file'], show_stdout=False, ) if __name__ == '__main__': - parser = common.get_argparse(argparse_args={ + parser = self.get_argparse(argparse_args={ 'description': 'Convert a QEMU `-trace exec_tb` to text form.' }) - args = common.setup(parser) + args = self.setup(parser) sys.exit(main()) diff --git a/release b/release index dc8cc54..50ab913 100755 --- a/release +++ b/release @@ -10,23 +10,23 @@ import subprocess import time import common -release_zip = imp.load_source('release_zip', os.path.join(common.root_dir, 'release-zip')) -release_upload = imp.load_source('release_upload', os.path.join(common.root_dir, 'release-upload')) +release_zip = imp.load_source('release_zip', os.path.join(kwargs['root_dir'], 'release-zip')) +release_upload = imp.load_source('release_upload', os.path.join(kwargs['root_dir'], 'release-upload')) start_time = time.time() # TODO factor those out so we don't redo the same thing multiple times. -# subprocess.check_call([os.path.join(common.root_dir, 'test')]) -# subprocess.check_call([os.path.join(common.root_dir, ''bench-all', '-A', '-u']) +# subprocess.check_call([os.path.join(kwargs['root_dir'], 'test')]) +# subprocess.check_call([os.path.join(kwargs['root_dir'], ''bench-all', '-A', '-u']) # A clean release requires a full rebuild unless we hack it :-( # We can't just use our current build as it contains packages we've # installed in random experiments. And with EXT2: we can't easily # know what the smallest root filesystem size is and use it either... # https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot -subprocess.check_call([os.path.join(common.root_dir, 'configure'), '--all']) -subprocess.check_call([os.path.join(common.root_dir, 'build'), '--all-archs', 'release']) +subprocess.check_call([os.path.join(kwargs['root_dir'], 'configure'), '--all']) +subprocess.check_call([os.path.join(kwargs['root_dir'], 'build'), '--all-archs', 'release']) release_zip.main() subprocess.check_call(['git', 'push']) release_upload.main() end_time = time.time() -common.print_time(end_time - start_time) +self.print_time(end_time - start_time) diff --git a/release-download-latest b/release-download-latest index d30ad8c..ee23a11 100755 --- a/release-download-latest +++ b/release-download-latest @@ -10,7 +10,8 @@ https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downl import urllib.request import common +from shell_helpers import LF -_json = common.github_make_request(path='/releases') +_json = self.github_make_request(path='/releases') asset = _json[0]['assets'][0] urllib.request.urlretrieve(asset['browser_download_url'], asset['name']) diff --git a/release-upload b/release-upload index 5dc6581..7149bdb 100755 --- a/release-upload +++ b/release-upload @@ -16,15 +16,16 @@ import sys import urllib.error import common +from shell_helpers import LF def main(): - repo = common.github_repo_id - tag = 'sha-{}'.format(common.sha) - upload_path = common.release_zip_file + repo = kwargs['github_repo_id'] + tag = 'sha-{}'.format(kwargs['sha']) + upload_path = kwargs['release_zip_file'] # Check the release already exists. try: - _json = common.github_make_request(path='/releases/tags/' + tag) + _json = self.github_make_request(path='/releases/tags/' + tag) except urllib.error.HTTPError as e: if e.code == 404: release_exists = False @@ -36,7 +37,7 @@ def main(): # Create release if not yet created. if not release_exists: - _json = common.github_make_request( + _json = self.github_make_request( authenticate=True, data=json.dumps({ 'tag_name': tag, @@ -50,12 +51,12 @@ def main(): asset_name = os.path.split(upload_path)[1] # Clear the prebuilts for a upload. - _json = common.github_make_request( + _json = self.github_make_request( path=('/releases/' + str(release_id) + '/assets'), ) for asset in _json: if asset['name'] == asset_name: - _json = common.github_make_request( + _json = self.github_make_request( authenticate=True, path=('/releases/assets/' + str(asset['id'])), method='DELETE', @@ -65,7 +66,7 @@ def main(): # Upload the prebuilt. with open(upload_path, 'br') as myfile: content = myfile.read() - _json = common.github_make_request( + _json = self.github_make_request( authenticate=True, data=content, extra_headers={'Content-Type': 'application/zip'}, diff --git a/release-zip b/release-zip index 05b2390..55de3c7 100755 --- a/release-zip +++ b/release-zip @@ -9,16 +9,17 @@ import subprocess import zipfile import common +from shell_helpers import LF def main(): - os.makedirs(common.release_dir, exist_ok=True) - if os.path.exists(common.release_zip_file): - os.unlink(common.release_zip_file) - zipf = zipfile.ZipFile(common.release_zip_file, 'w', zipfile.ZIP_DEFLATED) - for arch in common.all_archs: - common.setup(common.get_argparse(default_args={'arch': arch})) - zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir)) - zipf.write(common.linux_image, arcname=os.path.relpath(common.linux_image, common.root_dir)) + os.makedirs(kwargs['release_dir'], exist_ok=True) + if os.path.exists(kwargs['release_zip_file']): + os.unlink(kwargs['release_zip_file']) + zipf = zipfile.ZipFile(kwargs['release_zip_file'], 'w', zipfile.ZIP_DEFLATED) + for arch in kwargs['all_archs']: + self.setup(common.get_argparse(default_args={'arch': arch})) + zipf.write(kwargs['qcow2_file'], arcname=os.path.relpath(kwargs['qcow2_file'], kwargs['root_dir'])) + zipf.write(kwargs['linux_image'], arcname=os.path.relpath(kwargs['linux_image'], kwargs['root_dir'])) zipf.close() if __name__ == '__main__': diff --git a/run b/run index e28dbd4..e2c53c7 100755 --- a/run +++ b/run @@ -8,451 +8,18 @@ import sys import time import common +from shell_helpers import LF -defaults = { - 'background': False, - 'cpus': 1, - 'wait_gdb': False, - 'debug_vm': None, - 'eval': None, - 'extra_emulator_args': [], - 'gem5_exe_args': '', - 'gem5_script': 'fs', - 'gem5_readfile': '', - 'gem5_restore': None, - 'graphic': False, - 'initramfs': False, - 'initrd': False, - 'kernel_cli': None, - 'kernel_cli_after_dash': None, - 'eval_after': None, - 'kgdb': False, - 'kdb': False, - 'kvm': False, - 'memory': '256M', - 'record': False, - 'replay': False, - 'terminal': False, - 'tmux': None, - 'trace': None, - 'trace_stdout': False, - 'userland': None, - 'userland_before': '', - 'vnc': False, -} - -def main(args, extra_args=None): - global defaults - args = common.resolve_args(defaults, args, extra_args) - # Common qemu / gem5 logic. - # nokaslr: - # * https://unix.stackexchange.com/questions/397939/turning-off-kaslr-to-debug-linux-kernel-using-qemu-and-gdb - # * https://stackoverflow.com/questions/44612822/unable-to-debug-kernel-with-qemu-gdb/49840927#49840927 - # Turned on by default since v4.12 - kernel_cli = 'console_msg_format=syslog nokaslr norandmaps panic=-1 printk.devkmsg=on printk.time=y rw' - if args.kernel_cli is not None: - kernel_cli += ' {}'.format(args.kernel_cli) - kernel_cli_after_dash = '' - extra_emulator_args = [] - extra_qemu_args = [] - if args.debug_vm is not None: - debug_vm = ['gdb', common.Newline, '-q', common.Newline] + common.shlex_split(args.debug_vm) + ['--args', common.Newline] - else: - debug_vm = [] - if args.wait_gdb: - extra_qemu_args.extend(['-S', common.Newline]) - if args.eval_after is not None: - kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(common.base64_encode(args.eval_after)) - if args.kernel_cli_after_dash is not None: - kernel_cli_after_dash += ' {}'.format(args.kernel_cli_after_dash) - if args.vnc: - vnc = ['-vnc', ':0', common.Newline] - else: - vnc = [] - if args.initrd or args.initramfs: - ramfs = True - else: - ramfs = False - if args.eval is not None: - if ramfs: - initarg = 'rdinit' - else: - initarg = 'init' - kernel_cli += ' {}=/eval_base64.sh'.format(initarg) - kernel_cli_after_dash += ' lkmc_eval="{}"'.format(common.base64_encode(args.eval)) - if not args.graphic: - extra_qemu_args.extend(['-nographic', common.Newline]) - console = None - console_type = None - console_count = 0 - if args.arch == 'x86_64': - console_type = 'ttyS' - elif common.is_arm: - console_type = 'ttyAMA' - console = '{}{}'.format(console_type, console_count) - console_count += 1 - if not (args.arch == 'x86_64' and args.graphic): - kernel_cli += ' console={}'.format(console) - extra_console = '{}{}'.format(console_type, console_count) - console_count += 1 - if args.kdb or args.kgdb: - kernel_cli += ' kgdbwait' - if args.kdb: - if args.graphic: - kdb_cmd = 'kbd,' - else: - kdb_cmd = '' - kernel_cli += ' kgdboc={}{},115200'.format(kdb_cmd, console) - if args.kgdb: - kernel_cli += ' kgdboc={},115200'.format(extra_console) - if kernel_cli_after_dash: - kernel_cli += " -{}".format(kernel_cli_after_dash) - extra_env = {} - if args.trace is None: - do_trace = False - # A dummy value that is already turned on by default and does not produce large output, - # just to prevent QEMU from emitting a warning that '' is not valid. - trace_type = 'load_file' - else: - do_trace = True - trace_type = args.trace - - def raise_rootfs_not_found(): - if not args.dry_run: - raise Exception('Root filesystem not found. Did you build it?\n' \ - 'Tried to use: ' + common.disk_image) - def raise_image_not_found(): - if not args.dry_run: - raise Exception('Executable image not found. Did you build it?\n' \ - 'Tried to use: ' + common.image) - if common.image is None: - raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) - cmd = debug_vm.copy() - if common.emulator == 'gem5': - if common.baremetal is None: - if not os.path.exists(common.rootfs_raw_file): - if not os.path.exists(common.qcow2_file): - raise_rootfs_not_found() - common.raw_to_qcow2(prebuilt=args.prebuilt, reverse=True) - else: - if not os.path.exists(common.gem5_fake_iso): - os.makedirs(os.path.dirname(common.gem5_fake_iso), exist_ok=True) - common.write_string_to_file(common.gem5_fake_iso, 'a' * 512) - if not os.path.exists(common.image): - # This is to run gem5 from a prebuilt download. - if (not common.baremetal is None) or (not os.path.exists(common.linux_image)): - raise_image_not_found() - common.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) - os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True) - common.write_string_to_file(common.gem5_readfile, args.gem5_readfile) - memory = '{}B'.format(args.memory) - gem5_exe_args = common.shlex_split(args.gem5_exe_args) - if do_trace: - gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), common.Newline]) - extra_env['M5_PATH'] = common.gem5_system_dir - # https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071 - extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true' - if args.trace_stdout: - debug_file = 'cout' - else: - debug_file = 'trace.txt' - cmd.extend( - [ - common.executable, common.Newline, - '--debug-file', debug_file, common.Newline, - '--listener-mode', 'on', common.Newline, - '--outdir', common.m5out_dir, common.Newline, - ] + - gem5_exe_args +class Main(common.LkmcCliFunction): + def __init__(self): + super().__init__( + description='''\ +Run some content on an emulator. +''' ) - if args.userland is not None: - cmd.extend([ - common.gem5_se_file, common.Newline, - '-c', common.resolve_userland(args.userland), common.Newline, - ]) - else: - if args.gem5_script == 'fs': - # TODO port - if args.gem5_restore is not None: - cpt_dirs = common.gem_list_checkpoint_dirs() - cpt_dir = cpt_dirs[-args.gem5_restore] - extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) - cmd.extend([ - common.gem5_fs_file, common.Newline, - '--disk-image', common.disk_image, common.Newline, - '--kernel', common.image, common.Newline, - '--mem-size', memory, common.Newline, - '--num-cpus', str(args.cpus), common.Newline, - '--script', common.gem5_readfile, common.Newline, - ]) - if args.arch == 'x86_64': - if args.kvm: - cmd.extend(['--cpu-type', 'X86KvmCPU', common.Newline]) - cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), common.Newline]) - elif common.is_arm: - if args.kvm: - cmd.extend(['--cpu-type', 'ArmV8KvmCPU', common.Newline]) - if args.dp650: - dp650_cmd = 'dpu_' - else: - dp650_cmd = '' - cmd.extend([ - # TODO why is it mandatory to pass mem= here? Not true for QEMU. - # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? - '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), common.Newline, - '--machine-type', common.machine, common.Newline, - ]) - dtb = None - if args.dtb is not None: - dtb = args.dtb - elif args.dp650: - dtb = os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}{}cpu.dtb'.format(common.armv, dp650_cmd, args.cpus)), common.Newline, - if dtb is None: - cmd.extend(['--generate-dtb', common.Newline]) - else: - cmd.extend(['--dtb-filename', dtb, common.Newline]) - if common.baremetal is None: - cmd.extend([ - '--param', 'system.panic_on_panic = True', common.Newline - ]) - else: - cmd.extend([ - '--bare-metal', common.Newline, - '--param', 'system.auto_reset_addr = True', common.Newline, - ]) - if args.arch == 'aarch64': - # https://stackoverflow.com/questions/43682311/uart-communication-in-gem5-with-arm-bare-metal/50983650#50983650 - cmd.extend(['--param', 'system.highest_el_is_64 = True', common.Newline]) - elif args.gem5_script == 'biglittle': - if args.kvm: - cpu_type = 'kvm' - else: - cpu_type = 'atomic' - if args.gem5_restore is not None: - cpt_dir = common.gem_list_checkpoint_dirs()[-args.gem5_restore] - extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)]) - cmd.extend([ - os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), common.Newline, - '--big-cpus', '2', common.Newline, - '--cpu-type', cpu_type, common.Newline, - '--disk', common.disk_image, common.Newline, - '--kernel', common.image, common.Newline, - '--little-cpus', '2', common.Newline, - ]) - if args.dtb: - cmd.extend(['--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), common.Newline,]) - if args.wait_gdb: - # https://stackoverflow.com/questions/49296092/how-to-make-gem5-wait-for-gdb-to-connect-to-reliably-break-at-start-kernel-of-th - cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', common.Newline]) - else: - qemu_user_and_system_options = [ - '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), common.Newline, - ] - if args.userland is not None: - if args.wait_gdb: - debug_args = ['-g', str(common.gdb_port), common.Newline] - else: - debug_args = [] - cmd.extend( - [ - os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)), common.Newline, - '-L', common.target_dir, common.Newline - ] + - qemu_user_and_system_options + - common.shlex_split(args.userland_before) + - debug_args + - [ - common.resolve_userland(args.userland), common.Newline - ] - ) - else: - if not os.path.exists(common.image): - raise_image_not_found() - extra_emulator_args.extend(extra_qemu_args) - common.make_run_dirs() - if args.prebuilt or not os.path.exists(common.qemu_executable): - qemu_executable = common.qemu_executable_basename - qemu_executable_prebuilt = True - else: - qemu_executable = common.qemu_executable - qemu_executable_prebuilt = False - qemu_executable = shutil.which(qemu_executable) - if qemu_executable is None: - raise Exception('QEMU executable not found, did you forget to build or install it?\n' \ - 'Tried to use: ' + qemu_executable) - if args.debug_vm: - serial_monitor = [] - else: - if args.background: - serial_monitor = ['-serial', 'file:{}'.format(common.qemu_background_serial_file), common.Newline] - else: - serial_monitor = ['-serial', 'mon:stdio', common.Newline] - if args.kvm: - extra_emulator_args.extend(['-enable-kvm', common.Newline]) - extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port), common.Newline]) - virtfs_data = [ - (common.p9_dir, 'host_data'), - (common.out_dir, 'host_out'), - (common.out_rootfs_overlay_dir, 'host_out_rootfs_overlay'), - (common.rootfs_overlay_dir, 'host_rootfs_overlay'), - ] - virtfs_cmd = [] - for virtfs_dir, virtfs_tag in virtfs_data: - if os.path.exists(virtfs_dir): - virtfs_cmd.extend([ - '-virtfs', - 'local,path={virtfs_dir},mount_tag={virtfs_tag},security_model=mapped,id={virtfs_tag}' \ - .format(virtfs_dir=virtfs_dir, virtfs_tag=virtfs_tag), - common.Newline, - ]) - cmd.extend( - [ - qemu_executable, common.Newline, - '-device', 'rtl8139,netdev=net0', common.Newline, - '-gdb', 'tcp::{}'.format(common.gdb_port), common.Newline, - '-kernel', common.image, common.Newline, - '-m', args.memory, common.Newline, - '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), common.Newline, - '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), common.Newline, - '-no-reboot', common.Newline, - '-smp', str(args.cpus), common.Newline, - ] + - virtfs_cmd + - serial_monitor + - vnc - ) - if args.dtb is not None: - cmd.extend(['-dtb', args.dtb, common.Newline]) - if not qemu_executable_prebuilt: - cmd.extend(qemu_user_and_system_options) - if args.initrd: - extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')]) - rr = args.record or args.replay - if ramfs: - # TODO why is this needed, and why any string works. - root = 'root=/dev/anything' - else: - if rr: - driveif = 'none' - rrid = ',id=img-direct' - root = 'root=/dev/sda' - snapshot = '' - else: - driveif = 'virtio' - root = 'root=/dev/vda' - rrid = '' - snapshot = ',snapshot' - if common.baremetal is None: - if not os.path.exists(common.qcow2_file): - if not os.path.exists(common.rootfs_raw_file): - raise_rootfs_not_found() - common.raw_to_qcow2(prebuilt=args.prebuilt) - extra_emulator_args.extend([ - '-drive', - 'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid), - common.Newline, - ]) - if rr: - extra_emulator_args.extend([ - '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', common.Newline, - '-device', 'ide-hd,drive=img-blkreplay', common.Newline, - ]) - if rr: - extra_emulator_args.extend([ - '-object', 'filter-replay,id=replay,netdev=net0', - '-icount', 'shift=7,rr={},rrfile={}'.format('record' if args.record else 'replay', common.qemu_rrfile), - ]) - virtio_gpu_pci = [] - else: - virtio_gpu_pci = ['-device', 'virtio-gpu-pci', common.Newline] - if args.arch == 'x86_64': - append = ['-append', '{} nopat {}'.format(root, kernel_cli), common.Newline] - cmd.extend([ - '-M', common.machine, common.Newline, - '-device', 'edu', common.Newline, - ]) - elif common.is_arm: - extra_emulator_args.extend(['-semihosting', common.Newline]) - if args.arch == 'arm': - cpu = 'cortex-a15' - else: - cpu = 'cortex-a57' - append = ['-append', '{} {}'.format(root, kernel_cli), common.Newline] - cmd.extend( - [ - # highmem=off needed since v3.0.0 due to: - # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html - '-M', '{},highmem=off'.format(common.machine), common.Newline, - '-cpu', cpu, common.Newline, - ] + - virtio_gpu_pci - ) - if common.baremetal is None: - cmd.extend(append) - if args.tmux is not None: - tmux_args = '--run-id {}'.format(args.run_id) - if common.emulator == 'gem5': - tmux_cmd = './gem5-shell' - elif args.wait_gdb: - tmux_cmd = './run-gdb' - # TODO find a nicer way to forward all those args automatically. - # Part of me wants to: https://github.com/jonathanslenders/pymux - # but it cannot be used as a library properly it seems, and it is - # slower than tmux. - tmux_args += " --arch {} --linux-build-id '{}' --run-id '{}'".format( - args.arch, - args.linux_build_id, - args.run_id, - ) - if common.baremetal: - tmux_args += " --baremetal '{}'".format(common.baremetal) - if args.userland: - tmux_args += " --userland '{}'".format(args.userland) - tmux_args += ' {}'.format(args.tmux) - subprocess.Popen([ - os.path.join(common.root_dir, 'tmu'), - "sleep 2;{} {}".format(tmux_cmd, tmux_args) - ]) - cmd.extend(extra_emulator_args) - cmd.extend(args.extra_emulator_args) - if debug_vm or args.terminal: - out_file = None - else: - out_file = common.termout_file - common.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) - # Check if guest panicked. - if common.emulator == 'gem5': - # We have to do some parsing here because gem5 exits with status 0 even when panic happens. - # Grepping for '^panic: ' does not work because some errors don't show that message. - panic_msg = b'--- BEGIN LIBC BACKTRACE ---$' - else: - panic_msg = b'Kernel panic - not syncing' - panic_re = re.compile(panic_msg) - error_string_found = False - if out_file is not None and not args.dry_run: - with open(common.termout_file, 'br') as logfile: - for line in logfile: - if panic_re.search(line): - error_string_found = True - if os.path.exists(common.guest_terminal_file): - with open(common.guest_terminal_file, 'br') as logfile: - lines = logfile.readlines() - if lines: - last_line = lines[-1] - if last_line.rstrip() == common.magic_fail_string: - error_string_found = True - if error_string_found: - common.log_error('simulation error detected by parsing logs') - return 1 - return 0 - -def get_argparse(): - parser = common.get_argparse(argparse_args={'description':'Run Linux on an emulator'}) - init_group = parser.add_mutually_exclusive_group() - kvm_group = parser.add_mutually_exclusive_group() - parser.add_argument( - '--background', default=defaults['background'], action='store_true', - help='''\ + self.add_argument( + '--background', default=False, + help='''\ Send QEMU output to a file instead of the terminal so it does not require a terminal attached to run on the background. Interactive input cannot be given. TODO: use a port instead. If only there was a way to redirect a serial to multiple @@ -460,49 +27,53 @@ places, both to a port and a file? We use the file currently to be able to have any output at all. https://superuser.com/questions/1373226/how-to-redirect-qemu-serial-output-to-both-a-file-and-the-terminal-or-a-port ''' - ) - parser.add_argument( - '-c', '--cpus', default=defaults['cpus'], type=int, - help='Number of guest CPUs to emulate. Default: %(default)s' - ) - parser.add_argument( - '-D', '--debug-vm', default=defaults['debug_vm'], nargs='?', action='store', const='', - help='Run GDB on the emulator itself.' - ) - parser.add_argument( - '--dtb', - help='''\ + ) + self.add_argument( + '-c', '--cpus', default=1, type=int, + help='Number of guest CPUs to emulate. Default: %(default)s' + ) + self.add_argument( + '-D', '--debug-vm', default=False, + help='Run GDB on the emulator itself.' + ) + self.add_argument( + '--debug-vm-args', default='', + help='Pass arguments to GDB.' + ) + self.add_argument( + '--dtb', + help='''\ Use the specified DTB file. If not given, let the emulator generate a DTB for us, which is what you usually want. ''' - ) - parser.add_argument( - '-E', '--eval', - help='''\ + ) + self.add_argument( + '-E', '--eval', + help='''\ Replace the normal init with a minimal init that just evals the given string. See: https://github.com/cirosantilli/linux-kernel-module-cheat#replace-init ''' - ) - parser.add_argument( - '-e', '--kernel-cli', - help='''\ + ) + self.add_argument( + '-e', '--kernel-cli', + help='''\ Pass an extra Linux kernel command line options, and place them before the dash separator `-`. Only options that come before the `-`, i.e. "standard" options, should be passed with this option. Example: `./run -a arm -e 'init=/poweroff.out'` ''' - ) - parser.add_argument( - '-F', '--eval-after', - help='''\ + ) + self.add_argument( + '-F', '--eval-after', + help='''\ Pass a base64 encoded command line parameter that gets evalled at the end of the normal init. See: https://github.com/cirosantilli/linux-kernel-module-cheat#init-busybox ''' - ) - parser.add_argument( - '-f', '--kernel-cli-after-dash', - help='''\ + ) + self.add_argument( + '-f', '--kernel-cli-after-dash', + help='''\ Pass an extra Linux kernel command line options, add a dash `-` separator, and place the options after the dash. Intended for custom options understood by our `init` scripts, most of which are prefixed @@ -510,10 +81,10 @@ by `lkmc_`. Example: `./run --kernel-cli-after-dash 'lkmc_eval="wget google.com" lkmc_lala=y'` Mnenomic: `-f` comes after `-e`. ''' - ) - parser.add_argument( - '-G', '--gem5-exe-args', default=defaults['gem5_exe_args'], - help='''\ + ) + self.add_argument( + '-G', '--gem5-exe-args', default='', + help='''\ Pass extra options to the gem5 executable. Do not confuse with the arguments passed to config scripts, like `fs.py`. Example: @@ -521,140 +92,525 @@ like `fs.py`. Example: will run: gem.op5 --debug-flags=Exec fs.py --cpu-type=HPI --caches ''' - ) - parser.add_argument( - '--gem5-script', default=defaults['gem5_script'], choices=['fs', 'biglittle'], - help='Which gem5 script to use' - ) - parser.add_argument( - '--gem5-readfile', default=defaults['gem5_readfile'], - help='Set the contents of m5 readfile to this string.' - ) - init_group.add_argument( - '-I', '--initramfs', default=defaults['initramfs'], action='store_true', - help='Use initramfs instead of a root filesystem' - ) - init_group.add_argument( - '-i', '--initrd', default=defaults['initrd'], action='store_true', - help='Use initrd instead of a root filesystem' - ) - kvm_group.add_argument( - '-K', '--kvm', default=defaults['kvm'], action='store_true', - help='Use KVM. Only works if guest arch == host arch' - ) - parser.add_argument( - '--kgdb', default=defaults['kgdb'], action='store_true' - ) - parser.add_argument( - '--kdb', default=defaults['kdb'], action='store_true' - ) - parser.add_argument( - '-l', '--gem5-restore', type=int, - help='''\ + ) + self.add_argument( + '--gem5-script', default='fs', choices=['fs', 'biglittle'], + help='Which gem5 script to use' + ) + self.add_argument( + '--gem5-readfile', default='', + help='Set the contents of m5 readfile to this string.' + ) + self.add_argument( + '-K', '--kvm', default=False, + help='Use KVM. Only works if guest arch == host arch' + ) + self.add_argument( + '--kgdb', default=False, + ) + self.add_argument( + '--kdb', default=False, + ) + self.add_argument( + '-l', '--gem5-restore', type=int, + help='''\ Restore the nth most recently taken gem5 checkpoint according to directory timestamps. ''' - ) - parser.add_argument( - '-m', '--memory', default=defaults['memory'], - help='''\ + ) + self.add_argument( + '-m', '--memory', default='256M', + help='''\ Set the memory size of the guest. E.g.: `-m 512M`. We try to keep the default at the minimal ammount amount that boots all archs. Anything lower could lead some arch to fail to boot. Default: %(default)s ''' - ) - group = parser.add_mutually_exclusive_group() - group.add_argument( - '-R', '--replay', default=defaults['replay'], action='store_true', - help='Replay a QEMU run record deterministically' - ) - group.add_argument( - '-r', '--record', default=defaults['record'], action='store_true', - help='Record a QEMU run record for later replay with `-R`' - ) - parser.add_argument( - '-T', '--trace', - help='''\ + ) + self.add_argument( + '-R', '--replay', default=False, + help='Replay a QEMU run record deterministically' + ) + self.add_argument( + '-r', '--record', default=False, + help='Record a QEMU run record for later replay with `-R`' + ) + self.add_argument( + '-T', '--trace', + help='''\ Set trace events to be enabled. If not given, gem5 tracing is completely disabled, while QEMU tracing is enabled but uses default traces that are very rare and don't affect performance, because `./configure --enable-trace-backends=simple` seems to enable some traces by default, e.g. `pr_manager_run`, and I don't know how to get rid of them. ''' - ) - parser.add_argument( - '--trace-stdout', default=defaults['trace_stdout'], action='store_true', - help='''\ + ) + self.add_argument( + '--trace-stdout', default=False, + help='''\ Output trace to stdout instead of a file. Only works for gem5 currently. ''' - ) - init_group.add_argument( - '--terminal', default=defaults['terminal'], action='store_true', - help='''\ + ) + self.add_argument( + '--terminal', default=False, + help='''\ Output to the terminal, don't pipe to tee as the default. Does not save the output to a file, but allows you to use debuggers. Set automatically by --debug-vm, but you still need this option to debug gem5 Python scripts. ''' - ) - parser.add_argument( - '-t', '--tmux', default=defaults['tmux'], nargs='?', action='store', const='', - help='''\ + ) + self.add_argument( + '-t', '--tmux', default=False, + help='''\ Create a tmux split the window. You must already be inside of a `tmux` session to use this option: * on the main window, run the emulator as usual * on the split: ** if on QEMU and `-d` is given, GDB ** if on gem5, the gem5 terminal -If values are given to this option, pass those as parameters -to the program running on the split. ''' - ) - parser.add_argument( - '-u', '--userland', default=defaults['userland'], - help='''\ + ) + self.add_argument( + '--tmux-args', + help='''\ +Parameters to pass to the program running on the tmux split. +''' + ) + self.add_argument( + '-u', '--userland', + help='''\ Run the given userland executable in user mode instead of booting the Linux kernel in full system mode. In gem5, user mode is called Syscall Emulation (SE) mode and uses se.py. - Path resolution is similar to --baremetal. ''' - ) - parser.add_argument( - '--userland-before', default=defaults['userland_before'], - help='''\ + ) + self.add_argument( + '--userland-before', default='', + help='''\ Pass these arguments to the QEMU user mode CLI before the program to execute. This is required with --userland since arguments that come at the end are interpreted as command line arguments to that executable. ''' - ) - kvm_group.add_argument( - '-w', '--wait-gdb', default=defaults['wait_gdb'], action='store_true', - help='Wait for GDB to connect before starting execution' - ) - parser.add_argument( - '-x', '--graphic', default=defaults['graphic'], action='store_true', - help='Run in graphic mode. Mnemonic: X11' - ) - parser.add_argument( - '-V', '--vnc', default=defaults['vnc'], action='store_true', - help='''\ + ) + self.add_argument( + '-w', '--wait-gdb', default=False, + help='Wait for GDB to connect before starting execution' + ) + self.add_argument( + '-x', '--graphic', default=False, + help='Run in graphic mode. Mnemonic: X11' + ) + self.add_argument( + '-V', '--vnc', default=False, + help='''\ Run QEMU with VNC instead of the default SDL. Connect to it with: `vinagre localhost:5900`. ''' - ) - parser.add_argument( - 'extra_emulator_args', nargs='*', default=defaults['extra_emulator_args'], - help='Extra options to append at the end of the emulator command line' - ) - return parser + ) + self.add_argument( + 'extra_emulator_args', nargs='*', default=[], + help='Extra options to append at the end of the emulator command line' + ) + + def timed_main(self): + # Common qemu / gem5 logic. + # nokaslr: + # * https://unix.stackexchange.com/questions/397939/turning-off-kaslr-to-debug-linux-kernel-using-qemu-and-gdb + # * https://stackoverflow.com/questions/44612822/unable-to-debug-kernel-with-qemu-gdb/49840927#49840927 + # Turned on by default since v4.12 + kernel_cli = 'console_msg_format=syslog nokaslr norandmaps panic=-1 printk.devkmsg=on printk.time=y rw' + if self.env['kernel_cli'] is not None: + kernel_cli += ' {}'.format(self.env['kernel_cli']) + kernel_cli_after_dash = '' + extra_emulator_args = [] + extra_qemu_args = [] + if self.env['debug_vm']: + debug_vm = ['gdb', LF, '-q', LF] + self.sh.shlex_split(self.env['debug_vm_args']) + ['--args', LF] + else: + debug_vm = [] + if self.env['wait_gdb']: + extra_qemu_args.extend(['-S', LF]) + if self.env['eval_after'] is not None: + kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(self.base64_encode(self.env['eval_after'])) + if self.env['kernel_cli_after_dash'] is not None: + kernel_cli_after_dash += ' {}'.format(self.env['kernel_cli_after_dash']) + if self.env['vnc']: + vnc = ['-vnc', ':0', LF] + else: + vnc = [] + if self.env['initrd'] or self.env['initramfs']: + ramfs = True + else: + ramfs = False + if self.env['eval'] is not None: + if ramfs: + initarg = 'rdinit' + else: + initarg = 'init' + kernel_cli += ' {}=/eval_base64.sh'.format(initarg) + kernel_cli_after_dash += ' lkmc_eval="{}"'.format(self.base64_encode(self.env['eval'])) + if not self.env['graphic']: + extra_qemu_args.extend(['-nographic', LF]) + console = None + console_type = None + console_count = 0 + if self.env['arch'] == 'x86_64': + console_type = 'ttyS' + elif self.env['is_arm']: + console_type = 'ttyAMA' + console = '{}{}'.format(console_type, console_count) + console_count += 1 + if not (self.env['arch'] == 'x86_64' and self.env['graphic']): + kernel_cli += ' console={}'.format(console) + extra_console = '{}{}'.format(console_type, console_count) + console_count += 1 + if self.env['kdb'] or self.env['kgdb']: + kernel_cli += ' kgdbwait' + if self.env['kdb']: + if self.env['graphic']: + kdb_cmd = 'kbd,' + else: + kdb_cmd = '' + kernel_cli += ' kgdboc={}{},115200'.format(kdb_cmd, console) + if self.env['kgdb']: + kernel_cli += ' kgdboc={},115200'.format(extra_console) + if kernel_cli_after_dash: + kernel_cli += " -{}".format(kernel_cli_after_dash) + extra_env = {} + if self.env['trace'] is None: + do_trace = False + # A dummy value that is already turned on by default and does not produce large output, + # just to prevent QEMU from emitting a warning that '' is not valid. + trace_type = 'load_file' + else: + do_trace = True + trace_type = self.env['trace'] + + def raise_rootfs_not_found(): + if not self.env['dry_run']: + raise Exception('Root filesystem not found. Did you build it?\n' \ + 'Tried to use: ' + self.env['disk_image']) + def raise_image_not_found(): + if not self.env['dry_run']: + raise Exception('Executable image not found. Did you build it?\n' \ + 'Tried to use: ' + self.env['image']) + if self.env['image'] is None: + raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) + cmd = debug_vm.copy() + if self.env['emulator'] == 'gem5': + if self.env['baremetal'] is None: + if not os.path.exists(self.env['rootfs_raw_file']): + if not os.path.exists(self.env['qcow2_file']): + raise_rootfs_not_found() + self.raw_to_qcow2(prebuilt=self.env['prebuilt'], reverse=True) + else: + if not os.path.exists(self.env['gem5_fake_iso']): + os.makedirs(os.path.dirname(self.env['gem5_fake_iso']), exist_ok=True) + self.sh.write_string_to_file(self.env['gem5_fake_iso'], 'a' * 512) + if not os.path.exists(self.env['image']): + # This is to run gem5 from a prebuilt download. + if (not self.env['baremetal'] is None) or (not os.path.exists(self.env['linux_image'])): + raise_image_not_found() + self.sh.run_cmd([os.path.join(self.env['extract_vmlinux'], self.env['linux_image'])]) + os.makedirs(os.path.dirname(self.env['gem5_readfile']), exist_ok=True) + self.sh.write_string_to_file(self.env['gem5_readfile'], self.env['gem5_readfile']) + memory = '{}B'.format(self.env['memory']) + gem5_exe_args = self.sh.shlex_split(self.env['gem5_exe_args']) + if do_trace: + gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), LF]) + extra_env['M5_PATH'] = self.env['gem5_system_dir'] + # https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071 + extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true' + if self.env['trace_stdout']: + debug_file = 'cout' + else: + debug_file = 'trace.txt' + cmd.extend( + [ + self.env['executable'], LF, + '--debug-file', debug_file, LF, + '--listener-mode', 'on', LF, + '--outdir', self.env['m5out_dir'], LF, + ] + + gem5_exe_args + ) + if self.env['userland'] is not None: + cmd.extend([ + self.env['gem5_se_file'], LF, + '-c', self.resolve_userland(self.env['userland']), LF, + ]) + else: + if self.env['gem5_script'] == 'fs': + # TODO port + if self.env['gem5_restore'] is not None: + cpt_dirs = self.gem_list_checkpoint_dirs() + cpt_dir = cpt_dirs[-self.env['gem5_restore']] + extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) + cmd.extend([ + self.env['gem5_fs_file'], LF, + '--disk-image', self.env['disk_image'], LF, + '--kernel', self.env['image'], LF, + '--mem-size', memory, LF, + '--num-cpus', str(self.env['cpus']), LF, + '--script', self.env['gem5_readfile'], LF, + ]) + if self.env['arch'] == 'x86_64': + if self.env['kvm']: + cmd.extend(['--cpu-type', 'X86KvmCPU', LF]) + cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), LF]) + elif self.env['is_arm']: + if self.env['kvm']: + cmd.extend(['--cpu-type', 'ArmV8KvmCPU', LF]) + cmd.extend([ + # TODO why is it mandatory to pass mem= here? Not true for QEMU. + # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? + '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), LF, + '--machine-type', self.env['machine'], LF, + ]) + dtb = None + if self.env['dtb'] is not None: + dtb = self.env['dtb'] + elif args.dp650: + dtb = os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}{}cpu.dtb'.format(common.armv, dp650_cmd, args.cpus)), common.Newline, + if dtb is None: + cmd.extend(['--generate-dtb', common.Newline]) + else: + cmd.extend(['--dtb-filename', dtb, common.Newline]) + if self.env['baremetal'] is None: + cmd.extend([ + '--param', 'system.panic_on_panic = True', LF]) + else: + cmd.extend([ + '--bare-metal', LF, + '--param', 'system.auto_reset_addr = True', LF, + ]) + if self.env['arch'] == 'aarch64': + # https://stackoverflow.com/questions/43682311/uart-communication-in-gem5-with-arm-bare-metal/50983650#50983650 + cmd.extend(['--param', 'system.highest_el_is_64 = True', LF]) + elif self.env['gem5_script'] == 'biglittle': + if self.env['kvm']: + cpu_type = 'kvm' + else: + cpu_type = 'atomic' + if self.env['gem5_restore'] is not None: + cpt_dir = self.gem_list_checkpoint_dirs()[-self.env['gem5_restore']] + extra_emulator_args.extend(['--restore-from', os.path.join(self.env['m5out_dir'], cpt_dir)]) + cmd.extend([ + os.path.join(self.env['gem5_source_dir'], 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF, + '--big-cpus', '2', LF, + '--cpu-type', cpu_type, LF, + '--disk', self.env['disk_image'], LF, + '--kernel', self.env['image'], LF, + '--little-cpus', '2', LF, + ]) + if self.env['dtb']: + cmd.extend(['--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), NL]) + if self.env['wait_gdb']: + # https://stackoverflow.com/questions/49296092/how-to-make-gem5-wait-for-gdb-to-connect-to-reliably-break-at-start-kernel-of-th + cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', LF]) + else: + qemu_user_and_system_options = [ + '-trace', 'enable={},file={}'.format(trace_type, self.env['qemu_trace_file']), LF, + ] + if self.env['userland'] is not None: + if self.env['wait_gdb']: + debug_args = ['-g', str(self.env['gdb_port']), LF] + else: + debug_args = [] + cmd.extend( + [ + os.path.join(self.env['qemu_build_dir'], '{}-linux-user'.format(self.env['arch']), 'qemu-{}'.format(self.env['arch'])), LF, + '-L', self.env['target_dir'], LF + ] + + qemu_user_and_system_options + + self.sh.shlex_split(self.env['userland_before']) + + debug_args + + [ + self.resolve_userland(self.env['userland']), LF + ] + ) + else: + if not os.path.exists(self.env['image']): + raise_image_not_found() + extra_emulator_args.extend(extra_qemu_args) + self.make_run_dirs() + if self.env['prebuilt'] or not os.path.exists(self.env['qemu_executable']): + qemu_executable = self.env['qemu_executable_basename'] + qemu_executable_prebuilt = True + else: + qemu_executable = self.env['qemu_executable'] + qemu_executable_prebuilt = False + qemu_executable = shutil.which(qemu_executable) + if qemu_executable is None: + raise Exception('QEMU executable not found, did you forget to build or install it?\n' \ + 'Tried to use: ' + qemu_executable) + if self.env['debug_vm']: + serial_monitor = [] + else: + if self.env['background']: + serial_monitor = ['-serial', 'file:{}'.format(self.env['qemu_background_serial_file']), LF] + else: + serial_monitor = ['-serial', 'mon:stdio', LF] + if self.env['kvm']: + extra_emulator_args.extend(['-enable-kvm', LF]) + extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(self.env['extra_serial_port']), LF]) + virtfs_data = [ + (self.env['p9_dir'], 'host_data'), + (self.env['out_dir'], 'host_out'), + (self.env['out_rootfs_overlay_dir'], 'host_out_rootfs_overlay'), + (self.env['rootfs_overlay_dir'], 'host_rootfs_overlay'), + ] + virtfs_cmd = [] + for virtfs_dir, virtfs_tag in virtfs_data: + if os.path.exists(virtfs_dir): + virtfs_cmd.extend([ + '-virtfs', + 'local,path={virtfs_dir},mount_tag={virtfs_tag},security_model=mapped,id={virtfs_tag}' \ + .format(virtfs_dir=virtfs_dir, virtfs_tag=virtfs_tag), + LF, + ]) + cmd.extend( + [ + qemu_executable, LF, + '-device', 'rtl8139,netdev=net0', LF, + '-gdb', 'tcp::{}'.format(self.env['gdb_port']), LF, + '-kernel', self.env['image'], LF, + '-m', self.env['memory'], LF, + '-monitor', 'telnet::{},server,nowait'.format(self.env['qemu_monitor_port']), LF, + '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(self.env['qemu_hostfwd_generic_port'], self.env['qemu_hostfwd_generic_port'], self.env['qemu_hostfwd_ssh_port']), LF, + '-no-reboot', LF, + '-smp', str(self.env['cpus']), LF, + ] + + virtfs_cmd + + serial_monitor + + vnc + ) + if self.env['dtb'] is not None: + cmd.extend(['-dtb', self.env['dtb'], common.Newline]) + if not qemu_executable_prebuilt: + cmd.extend(qemu_user_and_system_options) + if self.env['initrd']: + extra_emulator_args.extend(['-initrd', os.path.join(self.env['buildroot_images_dir'], 'rootfs.cpio')]) + rr = self.env['record'] or self.env['replay'] + if ramfs: + # TODO why is this needed, and why any string works. + root = 'root=/dev/anything' + else: + if rr: + driveif = 'none' + rrid = ',id=img-direct' + root = 'root=/dev/sda' + snapshot = '' + else: + driveif = 'virtio' + root = 'root=/dev/vda' + rrid = '' + snapshot = ',snapshot' + if self.env['baremetal'] is None: + if not os.path.exists(self.env['qcow2_file']): + if not os.path.exists(self.env['rootfs_raw_file']): + raise_rootfs_not_found() + self.raw_to_qcow2(prebuilt=self.env['prebuilt']) + extra_emulator_args.extend([ + '-drive', + 'file={},format=qcow2,if={}{}{}'.format(self.env['disk_image'], driveif, snapshot, rrid), + LF, + ]) + if rr: + extra_emulator_args.extend([ + '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', LF, + '-device', 'ide-hd,drive=img-blkreplay', LF, + ]) + if rr: + extra_emulator_args.extend([ + '-object', 'filter-replay,id=replay,netdev=net0', + '-icount', 'shift=7,rr={},rrfile={}'.format('record' if self.env['record'] else 'replay', self.env['qemu_rrfile']), + ]) + virtio_gpu_pci = [] + else: + virtio_gpu_pci = ['-device', 'virtio-gpu-pci', LF] + if self.env['arch'] == 'x86_64': + append = ['-append', '{} nopat {}'.format(root, kernel_cli), LF] + cmd.extend([ + '-M', self.env['machine'], LF, + '-device', 'edu', LF, + ]) + elif self.env['is_arm']: + extra_emulator_args.extend(['-semihosting', LF]) + if self.env['arch'] == 'arm': + cpu = 'cortex-a15' + else: + cpu = 'cortex-a57' + append = ['-append', '{} {}'.format(root, kernel_cli), LF] + cmd.extend( + [ + # highmem=off needed since v3.0.0 due to: + # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html + '-M', '{},highmem=off'.format(self.env['machine']), LF, + '-cpu', cpu, LF, + ] + + virtio_gpu_pci + ) + if self.env['baremetal'] is None: + cmd.extend(append) + if self.env['tmux']: + tmux_args = '--run-id {}'.format(self.env['run_id']) + if self.env['emulator'] == 'gem5': + tmux_cmd = './gem5-shell' + else: + tmux_cmd = './run-gdb' + # TODO find a nicer way to forward all those args automatically. + # Part of me wants to: https://github.com/jonathanslenders/pymux + # but it cannot be used as a library properly it seems, and it is + # slower than tmux. + tmux_args += " --arch {} --linux-build-id '{}' --run-id '{}'".format( + self.env['arch'], + self.env['linux_build_id'], + self.env['run_id'], + ) + if self.env['baremetal']: + tmux_args += " --baremetal '{}'".format(self.env['baremetal']) + if self.env['userland']: + tmux_args += " --userland '{}'".format(self.env['userland']) + tmux_args += ' {}'.format(self.env['tmux']) + subprocess.Popen([ + os.path.join(self.env['root_dir'], 'tmu'), + "sleep 2;{} {}".format(tmux_cmd, tmux_args) + ]) + cmd.extend(extra_emulator_args) + cmd.extend(self.env['extra_emulator_args']) + if debug_vm or self.env['terminal']: + out_file = None + else: + out_file = self.env['termout_file'] + self.sh.run_cmd(cmd, cmd_file=self.env['run_cmd_file'], out_file=out_file, extra_env=extra_env) + # Check if guest panicked. + if self.env['emulator'] == 'gem5': + # We have to do some parsing here because gem5 exits with status 0 even when panic happens. + # Grepping for '^panic: ' does not work because some errors don't show that message. + panic_msg = b'--- BEGIN LIBC BACKTRACE ---$' + else: + panic_msg = b'Kernel panic - not syncing' + panic_re = re.compile(panic_msg) + error_string_found = False + if out_file is not None and not self.env['dry_run']: + with open(self.env['termout_file'], 'br') as logfile: + for line in logfile: + if panic_re.search(line): + error_string_found = True + if os.path.exists(self.env['guest_terminal_file']): + with open(self.env['guest_terminal_file'], 'br') as logfile: + lines = logfile.readlines() + if lines: + last_line = lines[-1] + if last_line.rstrip() == self.env['magic_fail_string']: + error_string_found = True + if error_string_found: + self.log_error('simulation error detected by parsing logs') + return 1 + return 0 if __name__ == '__main__': - parser = get_argparse() - args = common.setup(parser) - start_time = time.time() - exit_status = main(args) - end_time = time.time() - common.print_time(end_time - start_time) - sys.exit(exit_status) + Main().cli() diff --git a/run-docker b/run-docker index 56e471b..b3411c9 100755 --- a/run-docker +++ b/run-docker @@ -4,50 +4,51 @@ import argparse import os import common +from shell_helpers import LF -container_name = common.repo_short_id -container_hostname = common.repo_short_id -image_name = common.repo_short_id -target_dir = '/root/{}'.format(common.repo_short_id) +container_name = kwargs['repo_short_id'] +container_hostname = kwargs['repo_short_id'] +image_name = kwargs['repo_short_id'] +target_dir = '/root/{}'.format(kwargs['repo_short_id']) docker = ['sudo', 'docker'] def create(args): - common.run_cmd(docker + ['build', '-t', image_name, '.', common.Newline]) + self.sh.run_cmd(docker + ['build', '-t', image_name, '.', LF]) # --privileged for KVM: # https://stackoverflow.com/questions/48422001/launching-qemu-kvm-from-inside-docker-container - common.run_cmd( + self.sh.run_cmd( docker + [ - 'create', common.Newline, - '--hostname', container_hostname, common.Newline, - '-i', common.Newline, - '--name', container_name, common.Newline, - '--net', 'host', common.Newline, - '--privileged', common.Newline, - '-t', common.Newline, - '-w', target_dir, common.Newline, - '-v', '{}:{}'.format(os.getcwd(), target_dir), common.Newline, + 'create', LF, + '--hostname', container_hostname, LF, + '-i', LF, + '--name', container_name, LF, + '--net', 'host', LF, + '--privileged', LF, + '-t', LF, + '-w', target_dir, LF, + '-v', '{}:{}'.format(os.getcwd(), target_dir), LF, image_name, ] ) def destroy(args): stop(args) - common.run_cmd(docker + ['rm', container_name, common.Newline]) - common.run_cmd(docker + ['rmi', image_name, common.Newline]) + self.sh.run_cmd(docker + ['rm', container_name, LF]) + self.sh.run_cmd(docker + ['rmi', image_name, LF]) def sh(args): start(args) if args: sh_args = args else: sh_args = ['bash'] - common.run_cmd( + self.sh.run_cmd( docker + ['exec', '-i', '-t', container_name] + sh_args + - [common.Newline], + [LF], ) def start(args): - common.run_cmd(docker + ['start', container_name, common.Newline]) + self.sh.run_cmd(docker + ['start', container_name, LF]) def stop(args): - common.run_cmd(docker + ['stop', container_name, common.Newline]) + self.sh.run_cmd(docker + ['stop', container_name, LF]) cmd_action_map = { 'create': lambda args: create(args), 'DESTROY': lambda args: destroy(args), @@ -58,7 +59,7 @@ cmd_action_map = { parser = argparse.ArgumentParser() parser.add_argument('cmd', choices=cmd_action_map) parser.add_argument('args', nargs='*') -common.add_dry_run_argument(parser) +self.add_dry_run_argument(parser) args = parser.parse_args() -common.setup_dry_run_arguments(args) -cmd_action_map[args.cmd](args.args) +self.setup_dry_run_arguments(args) +cmd_action_map[kwargs['cmd']](kwargs['args']) diff --git a/run-gdb b/run-gdb index 55fc02f..379b406 100755 --- a/run-gdb +++ b/run-gdb @@ -7,18 +7,7 @@ import subprocess import sys import common - -defaults = { - 'after': '', - 'before': '', - 'break_at': None, - 'kgdb': False, - 'no_continue': False, - 'no_lxsymbols': False, - 'test': False, - 'sim': False, - 'userland': None, -} +from shell_helpers import LF class GdbTestcase: def __init__( @@ -34,8 +23,8 @@ class GdbTestcase: ''' self.prompt = '\(gdb\) ' self.source_path = source_path - common.print_cmd(cmd) - cmd = common.strip_newlines(cmd) + self.print_cmd(cmd) + cmd = self.strip_newlines(cmd) import pexpect self.child = pexpect.spawn( cmd[0], @@ -84,155 +73,146 @@ class GdbTestcase: self.child.sendline(line) self.child.expect(self.prompt) -def main(args, extra_args=None): - ''' - :param args: argparse parse_argument() output. Must contain all the common options, - but does not need GDB specific ones. - :type args: argparse.Namespace - - :param extra_args: extra arguments to be added to args - :type extra_args: Dict[str,Any] - - :return: GDB exit status - :rtype: int - ''' - global defaults - args = common.resolve_args(defaults, args, extra_args) - after = common.shlex_split(args.after) - before = common.shlex_split(args.before) - no_continue = args.no_continue - if args.test: - no_continue = True - before.extend([ - '-q', common.Newline, - '-nh', common.Newline, - '-ex', 'set confirm off', common.Newline - ]) - elif args.verbose: - # The output of this would affect the tests. - # https://stackoverflow.com/questions/13496389/gdb-remote-protocol-how-to-analyse-packets - # Also be opinionated and set remotetimeout to allow you to step debug the emulator at the same time. - before.extend([ - '-ex', 'set debug remote 1', common.Newline, - '-ex', 'set remotetimeout 99999', common.Newline, - ]) - if args.break_at is not None: - break_at = ['-ex', 'break {}'.format(args.break_at), common.Newline] - else: - break_at = [] - linux_full_system = (common.baremetal is None and args.userland is None) - if args.userland: - image = common.resolve_userland(args.userland) - elif common.baremetal: - image = common.image - test_script_path = os.path.splitext(common.source_path)[0] + '.py' - else: - image = common.vmlinux - if common.baremetal: - allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] - else: - allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] - cmd = ( - [common.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains), common.Newline] + - before + - ['-q', common.Newline] - ) - if linux_full_system: - cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), common.Newline]) - if args.sim: - target = 'sim' - else: - if args.kgdb: - port = common.extra_serial_port - else: - port = common.gdb_port - target = 'remote localhost:{}'.format(port) - cmd.extend([ - '-ex', 'file {}'.format(image), common.Newline, - '-ex', 'target {}'.format(target), common.Newline, - ]) - if not args.kgdb: - cmd.extend(break_at) - if not no_continue: - # ## lx-symbols - # - # ### lx-symbols after continue - # - # lx symbols must be run after continue. - # - # running it immediately after the connect on the bootloader leads to failure, - # likely because kernel structure on which it depends are not yet available. - # - # With this setup, continue runs, and lx-symbols only runs when a break happens, - # either by hitting the breakpoint, or by entering Ctrl + C. - # - # Sure, if the user sets a break on a raw address of the bootloader, - # problems will still arise, but let's think about that some other time. - # - # ### lx-symbols autoload - # - # The lx-symbols commands gets loaded through the file vmlinux-gdb.py - # which gets put on the kernel build root when python debugging scripts are enabled. - cmd.extend(['-ex', 'continue', common.Newline]) - if not args.no_lxsymbols and linux_full_system: - cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), common.Newline]) - cmd.extend(after) - if args.test: - GdbTestcase( - common.source_path, - test_script_path, - cmd, - verbose=args.verbose, +class Main(common.LkmcCliFunction): + def __init__(self): + super().__init__(description='''\ +Connect with GDB to an emulator to debug Linux itself +''') + self.add_argument( + '-A', '--after', default='', + help='Pass extra arguments to GDB, to be appended after all other arguments' ) - else: - # I would rather have cwd be out_rootfs_overlay_dir, - # but then lx-symbols cannot fine the vmlinux and fails with: - # vmlinux: No such file or directory. - return common.run_cmd( - cmd, - cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'), - cwd=common.linux_build_dir + self.add_argument( + '--before', default='', + help='Pass extra arguments to GDB to be prepended before any of the arguments passed by this script' ) - -if __name__ == '__main__': - parser = common.get_argparse(argparse_args={'description': 'Connect with GDB to an emulator to debug Linux itself'}) - parser.add_argument( - '-A', '--after', default=defaults['after'], - help='Pass extra arguments to GDB, to be appended after all other arguments' - ) - parser.add_argument( - '--before', default=defaults['before'], - help='Pass extra arguments to GDB to be prepended before any of the arguments passed by this script' - ) - parser.add_argument( - '-C', '--no-continue', default=defaults['no_continue'], action='store_true', - help="Don't run continue after connecting" - ) - parser.add_argument( - '-k', '--kgdb', default=defaults['kgdb'], action='store_true' - ) - parser.add_argument( - '--sim', default=defaults['sim'], action='store_true', - help='''Use the built-in GDB CPU simulator -See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-simulator -''' - ) - parser.add_argument( - '-X', '--no-lxsymbols', default=defaults['no_lxsymbols'], action='store_true' - ) - parser.add_argument( - '--test', default=defaults['test'], action='store_true', - help='''\ + self.add_argument( + 'break_at', nargs='?', + help='Extra options to append at the end of the emulator command line' + ) + self.add_argument( + '-k', '--kgdb', default=False, + ) + self.add_argument( + '-C', '--no-continue', default=False, + help="Don't run continue after connecting" + ) + self.add_argument( + '-X', '--no-lxsymbols', default=False, + ) + self.add_argument( + '--test', default=False, + help='''\ Run an expect test case instead of interactive usage. For baremetal and userland, the script is a .py file next to the source code. ''' - ) - parser.add_argument( - '-u', '--userland', default=defaults['userland'], - ) - parser.add_argument( - 'break_at', nargs='?', - help='Extra options to append at the end of the emulator command line' - ) - args = common.setup(parser) - sys.exit(main(args)) + ) + self.add_argument( + '--sim', default=False, + help='''Use the built-in GDB CPU simulator +See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-simulator +''' + ) + self.add_argument( + '-u', '--userland', + ) + + def timed_main(self): + after = self.sh.shlex_split(self.env['after']) + before = self.sh.shlex_split(self.env['before']) + no_continue = self.env['no_continue'] + if self.env['test']: + no_continue = True + before.extend([ + '-q', LF, + '-nh', LF, + '-ex', 'set confirm off', LF + ]) + elif self.env['verbose']: + # The output of this would affect the tests. + # https://stackoverflow.com/questions/13496389/gdb-remote-protocol-how-to-analyse-packets + # Also be opinionated and set remotetimeout to allow you to step debug the emulator at the same time. + before.extend([ + '-ex', 'set debug remote 1', LF, + '-ex', 'set remotetimeout 99999', LF, + ]) + if self.env['break_at'] is not None: + break_at = ['-ex', 'break {}'.format(self.env['break_at']), LF] + else: + break_at = [] + linux_full_system = (self.env['baremetal'] is None and self.env['userland'] is None) + if self.env['userland']: + image = self.resolve_userland(self.env['userland']) + elif self.env['baremetal']: + image = self.env['image'] + test_script_path = os.path.splitext(self.env['source_path'])[0] + '.py' + else: + image = self.env['vmlinux'] + if self.env['baremetal']: + allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] + else: + allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] + cmd = ( + [self.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains), LF] + + before + + ['-q', LF] + ) + if linux_full_system: + cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(self.env['linux_build_dir']), LF]) + if self.env['sim']: + target = 'sim' + else: + if self.env['kgdb']: + port = self.env['extra_serial_port'] + else: + port = self.env['gdb_port'] + target = 'remote localhost:{}'.format(port) + cmd.extend([ + '-ex', 'file {}'.format(image), LF, + '-ex', 'target {}'.format(target), LF, + ]) + if not self.env['kgdb']: + cmd.extend(break_at) + if not no_continue: + # ## lx-symbols + # + # ### lx-symbols after continue + # + # lx symbols must be run after continue. + # + # running it immediately after the connect on the bootloader leads to failure, + # likely because kernel structure on which it depends are not yet available. + # + # With this setup, continue runs, and lx-symbols only runs when a break happens, + # either by hitting the breakpoint, or by entering Ctrl + C. + # + # Sure, if the user sets a break on a raw address of the bootloader, + # problems will still arise, but let's think about that some other time. + # + # ### lx-symbols autoload + # + # The lx-symbols commands gets loaded through the file vmlinux-gdb.py + # which gets put on the kernel build root when python debugging scripts are enabled. + cmd.extend(['-ex', 'continue', LF]) + if not self.env['no_lxsymbols'] and linux_full_system: + cmd.extend(['-ex', 'lx-symbols {}'.format(self.env['kernel_modules_build_subdir']), LF]) + cmd.extend(after) + if self.env['test']: + GdbTestcase( + self.env['source_path'], + test_script_path, + cmd, + verbose=self.env['verbose'], + ) + else: + # I would rather have cwd be out_rootfs_overlay_dir, + # but then lx-symbols cannot fine the vmlinux and fails with: + # vmlinux: No such file or directory. + return self.sh.run_cmd( + cmd, + cmd_file=os.path.join(self.env['run_dir'], 'run-gdb.sh'), + cwd=self.env['linux_build_dir'] + ) + +if __name__ == '__main__': + Main().cli() diff --git a/run-gdb-user b/run-gdb-user index 6756701..44884d1 100755 --- a/run-gdb-user +++ b/run-gdb-user @@ -5,9 +5,9 @@ import os import sys import common -rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'run-gdb')) +rungdb = imp.load_source('rungdb', os.path.join(kwargs['root_dir'], 'run-gdb')) -parser = common.get_argparse(argparse_args={ +parser = self.get_argparse(argparse_args={ 'description': '''GDB step debug guest userland processes without gdbserver. More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-step-debug-userland-processes @@ -23,9 +23,9 @@ parser.add_argument( help='Break at this point, e.g. main.', nargs='?' ) -args = common.setup(parser) -executable = common.resolve_userland(args.executable) -addr = common.get_elf_entry(os.path.join(common.buildroot_build_build_dir, executable)) +args = self.setup(parser) +executable = self.resolve_userland(kwargs['executable']) +addr = self.get_elf_entry(os.path.join(kwargs['buildroot_build_build_dir'], executable)) extra_args = {} extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(addr)) # Or else lx-symbols throws for arm: @@ -33,5 +33,5 @@ extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(ad # TODO understand better. # Also, lx-symbols overrides the add-symbol-file commands. extra_args['no_lxsymbols'] = True -extra_args['break_at'] = args.break_at +extra_args['break_at'] = kwargs['break_at'] sys.exit(rungdb.main(args, extra_args)) diff --git a/run-gdbserver b/run-gdbserver index e34656f..f67002b 100755 --- a/run-gdbserver +++ b/run-gdbserver @@ -5,8 +5,9 @@ import subprocess import sys import common +from shell_helpers import LF -parser = common.get_argparse(argparse_args={ +parser = self.get_argparse(argparse_args={ 'description':'Connect to gdbserver running on the guest.' }) parser.add_argument( @@ -16,13 +17,13 @@ parser.add_argument( parser.add_argument( 'break_at', default='main', nargs='?' ) -args = common.setup(parser) +args = self.setup(parser) sys.exit(subprocess.Popen([ - common.get_toolchain_tool('gdb'), + self.get_toolchain_tool('gdb'), '-q', - '-ex', 'set sysroot {}'.format(common.buildroot_staging_dir), - '-ex', 'target remote localhost:{}'.format(common.qemu_hostfwd_generic_port), - '-ex', 'tbreak {}'.format(args.break_at), + '-ex', 'set sysroot {}'.format(kwargs['buildroot_staging_dir']), + '-ex', 'target remote localhost:{}'.format(kwargs['qemu_hostfwd_generic_port']), + '-ex', 'tbreak {}'.format(kwargs['break_at']), '-ex', 'continue', - os.path.join(common.buildroot_build_build_dir, common.resolve_userland(args.executable)), + os.path.join(kwargs['buildroot_build_build_dir'], self.resolve_userland(kwargs['executable'])), ]).wait()) diff --git a/run-toolchain b/run-toolchain index 041e1b5..e84a06c 100755 --- a/run-toolchain +++ b/run-toolchain @@ -4,8 +4,9 @@ import os import sys import common +from shell_helpers import LF -parser = common.get_argparse(argparse_args={ +parser = self.get_argparse(argparse_args={ 'description': '''Run a Buildroot ToolChain tool like readelf or objdump. For example, to get some information about the arm vmlinux: @@ -24,7 +25,6 @@ ls "$(./getvar -a arm host_bin_dir)" parser.add_argument( '--dry', help='Just output the tool path to stdout but actually run it', - action='store_true', ) parser.add_argument('tool', help='Which tool to run.') parser.add_argument( @@ -34,17 +34,17 @@ parser.add_argument( metavar='extra-args', nargs='*' ) -args = common.setup(parser) -if common.baremetal is None: - image = common.vmlinux +args = self.setup(parser) +if kwargs['baremetal'] is None: + image = kwargs['vmlinux'] else: - image = common.image -tool= common.get_toolchain_tool(args.tool) -if args.dry: + image = kwargs['image'] +tool= self.get_toolchain_tool(kwargs['tool']) +if kwargs['dry']: print(tool) else: - sys.exit(common.run_cmd( - [tool, common.Newline] - + common.add_newlines(args.extra_args), - cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'), + sys.exit(self.sh.run_cmd( + [tool, LF] + + self.sh.add_newlines(kwargs['extra_args']), + cmd_file=os.path.join(kwargs['run_dir'], 'run-toolchain.sh'), )) diff --git a/shell_helpers.py b/shell_helpers.py new file mode 100644 index 0000000..040b175 --- /dev/null +++ b/shell_helpers.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 + +import distutils.file_util +import itertools +import os +import shlex +import shutil +import signal +import stat +import subprocess +import sys + +class LF: + ''' + LineFeed (AKA newline). + + Singleton class. Can be used in print_cmd to print out nicer command lines + with --key on the same line as "--key value". + ''' + pass + +class ShellHelpers: + ''' + Helpers to do things which are easy from the shell, + usually filesystem, process or pipe operations. + + Attempt to print shell equivalents of all commands to make things + easy to debug and understand what is going on. + ''' + def __init__(self, dry_run=False): + ''' + :param dry_run: don't run the commands, just potentially print them. Debug aid. + :type dry_run: Bool + ''' + self.dry_run = dry_run + + def add_newlines(self, cmd): + out = [] + for arg in cmd: + out.extend([arg, LF]) + return out + + def cp(self, src, dest, **kwargs): + self.print_cmd(['cp', src, dest]) + if not self.dry_run: + shutil.copy2(src, dest) + + def cmd_to_string(self, cmd, cwd=None, extra_env=None, extra_paths=None): + ''' + Format a command given as a list of strings so that it can + be viewed nicely and executed by bash directly and print it to stdout. + ''' + last_newline = ' \\\n' + newline_separator = last_newline + ' ' + out = [] + if extra_env is None: + extra_env = {} + if cwd is not None: + out.append('cd {} &&'.format(shlex.quote(cwd))) + if extra_paths is not None: + out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths))) + for key in extra_env: + out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key]))) + cmd_quote = [] + newline_count = 0 + for arg in cmd: + if arg == LF: + cmd_quote.append(arg) + newline_count += 1 + else: + cmd_quote.append(shlex.quote(arg)) + if newline_count > 0: + cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == LF) if not x] + out.extend(cmd_quote) + if newline_count == 1 and cmd[-1] == LF: + ending = '' + else: + ending = last_newline + ';' + return newline_separator.join(out) + ending + + def copy_dir_if_update_non_recursive(self, srcdir, destdir, filter_ext=None): + # TODO print rsync equivalent. + os.makedirs(destdir, exist_ok=True) + for basename in os.listdir(srcdir): + src = os.path.join(srcdir, basename) + if os.path.isfile(src): + noext, ext = os.path.splitext(basename) + if filter_ext is not None and ext == filter_ext: + distutils.file_util.copy_file( + src, + os.path.join(destdir, basename), + update=1, + ) + + def print_cmd(self, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): + ''' + Print cmd_to_string to stdout. + + Optionally save the command to cmd_file file, and add extra_env + environment variables to the command generated. + + If cmd contains at least one LF, newlines are only added on LF. + Otherwise, newlines are added automatically after every word. + ''' + if type(cmd) is str: + cmd_string = cmd + else: + cmd_string = self.cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths) + print('+ ' + cmd_string) + if cmd_file is not None: + with open(cmd_file, 'w') as f: + f.write('#!/usr/bin/env bash\n') + f.write(cmd_string) + st = os.stat(cmd_file) + os.chmod(cmd_file, st.st_mode | stat.S_IXUSR) + + def run_cmd( + self, + cmd, + cmd_file=None, + out_file=None, + show_stdout=True, + show_cmd=True, + extra_env=None, + extra_paths=None, + delete_env=None, + raise_on_failure=True, + **kwargs + ): + ''' + Run a command. Write the command to stdout before running it. + + Wait until the command finishes execution. + + :param cmd: command to run. LF entries are magic get skipped. + :type cmd: List[str] + + :param cmd_file: if not None, write the command to be run to that file + :type cmd_file: str + + :param out_file: if not None, write the stdout and stderr of the command the file + :type out_file: str + + :param show_stdout: wether to show stdout and stderr on the terminal or not + :type show_stdout: bool + + :param extra_env: extra environment variables to add when running the command + :type extra_env: Dict[str,str] + ''' + if out_file is not None: + stdout = subprocess.PIPE + stderr = subprocess.STDOUT + else: + if show_stdout: + stdout = None + stderr = None + else: + stdout = subprocess.DEVNULL + stderr = subprocess.DEVNULL + if extra_env is None: + extra_env = {} + if delete_env is None: + delete_env = [] + if 'cwd' in kwargs: + cwd = kwargs['cwd'] + else: + cwd = None + env = os.environ.copy() + env.update(extra_env) + if extra_paths is not None: + path = ':'.join(extra_paths) + if 'PATH' in os.environ: + path += ':' + os.environ['PATH'] + env['PATH'] = path + for key in delete_env: + if key in env: + del env[key] + if show_cmd: + self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) + + # Otherwise Ctrl + C gives: + # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). + # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec + sigint_old = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, signal.SIG_IGN) + + # Otherwise BrokenPipeError when piping through | grep + # But if I do this_module, my terminal gets broken at the end. Why, why, why. + # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python + # Ignoring the exception is not enough as it prints a warning anyways. + #sigpipe_old = signal.getsignal(signal.SIGPIPE) + #signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + cmd = self.strip_newlines(cmd) + if not self.dry_run: + # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 + with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: + if out_file is not None: + os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True) + with open(out_file, 'bw') as logfile: + while True: + byte = proc.stdout.read(1) + if byte: + if show_stdout: + sys.stdout.buffer.write(byte) + try: + sys.stdout.flush() + except BlockingIOError: + # TODO understand. Why, Python, why. + pass + logfile.write(byte) + else: + break + signal.signal(signal.SIGINT, sigint_old) + #signal.signal(signal.SIGPIPE, sigpipe_old) + returncode = proc.returncode + if returncode != 0 and raise_on_failure: + raise Exception('Command exited with status: {}'.format(returncode)) + return returncode + else: + return 0 + + def shlex_split(self, string): + ''' + shlex_split, but also add Newline after every word. + + Not perfect since it does not group arguments, but I don't see a solution. + ''' + return self.add_newlines(shlex.split(string)) + + def strip_newlines(self, cmd): + return [x for x in cmd if x != LF] + + def rmrf(self, path): + self.print_cmd(['rm', '-r', '-f', path, LF]) + if not self.dry_run and os.path.exists(path): + shutil.rmtree(path) + + def write_configs(self, config_path, configs, config_fragments=None): + ''' + Write extra KEY=val configs into the given config file. + ''' + if config_fragments is None: + config_fragments = [] + with open(config_path, 'a') as config_file: + for config_fragment in config_fragments: + with open(config_fragment, 'r') as config_fragment_file: + self.print_cmd(['cat', config_fragment, '>>', config_path]) + if not self.dry_run: + for line in config_fragment_file: + config_file.write(line) + self.write_string_to_file(config_path, '\n'.join(configs), mode='a') + + def write_string_to_file(self, path, string, mode='w'): + if mode == 'a': + redirect = '>>' + else: + redirect = '>' + self.print_cmd("cat << 'EOF' {} {}\n{}\nEOF".format(redirect, path, string)) + if not self.dry_run: + with open(path, 'a') as f: + f.write(string) diff --git a/trace-boot b/trace-boot index 2daac4b..6b6a8f7 100755 --- a/trace-boot +++ b/trace-boot @@ -6,10 +6,10 @@ import subprocess import re import common -run = imp.load_source('run', os.path.join(common.root_dir, 'run')) -qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(common.root_dir, 'qemu-trace2txt')) +run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run')) +qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(kwargs['root_dir'], 'qemu-trace2txt')) -parser = common.get_argparse(argparse_args={ +parser = self.get_argparse(argparse_args={ 'description': '''Trace the PIC addresses executed on a Linux kernel boot. More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#tracing @@ -19,11 +19,11 @@ parser.add_argument( 'extra_emulator_args', nargs='*', help='Extra options to append at the end of the emulator command line' ) -args = common.setup(parser) +args = self.setup(parser) extra_args = { - 'extra_emulator_args': args.extra_emulator_args, + 'extra_emulator_args': kwargs['extra_emulator_args'], } -if common.emulator == 'gem5': +if kwargs['emulator'] == 'gem5': extra_args.update({ 'eval': 'm5 exit', 'trace': 'Exec,-ExecSymbol,-ExecMicro', @@ -39,10 +39,10 @@ else: # Instruction count. # We could put this on a separate script, but it just adds more arch boilerplate to a new script. # So let's just leave it here for now since it did not add a significant processing time. - kernel_entry_addr = hex(common.get_elf_entry(common.vmlinux)) + kernel_entry_addr = hex(self.get_elf_entry(kwargs['vmlinux'])) nlines = 0 nlines_firmware = 0 - with open(common.qemu_trace_txt_file, 'r') as trace_file: + with open(kwargs['qemu_trace_txt_file'], 'r') as trace_file: in_firmware = True for line in trace_file: line = line.rstrip() diff --git a/trace2line b/trace2line index 451a50d..cd53d98 100755 --- a/trace2line +++ b/trace2line @@ -13,23 +13,24 @@ import subprocess import sys import common +from shell_helpers import LF -parser = common.get_argparse(argparse_args={ +parser = self.get_argparse(argparse_args={ 'description': 'Convert an execution trace containing PC values into the Linux kernel linex executed' }) -args = common.setup(parser) +args = self.setup(parser) sys.exit(subprocess.Popen([ - os.path.join(common.root_dir, 'trace2line.sh'), - 'true' if common.emulator == 'gem5' else 'false', - common.trace_txt_file, - common.get_toolchain_tool('addr2line'), - common.vmlinux, - common.run_dir, + os.path.join(kwargs['root_dir'], 'trace2line.sh'), + 'true' if kwargs['emulator'] == 'gem5' else 'false', + kwargs['trace_txt_file'], + self.get_toolchain_tool('addr2line'), + kwargs['vmlinux'], + kwargs['run_dir'], ]).wait()) # This was the full conversion attempt. -# if common.emulator == 'gem5': +# if kwargs['emulator'] == 'gem5': # def get_pc(line): # # TODO # # stdin = sed -r 's/^.* (0x[^. ]*)[. ].*/\1/' "$common_trace_txt_file") @@ -40,17 +41,17 @@ sys.exit(subprocess.Popen([ # with \ # subprocess.Popen( # [ -# common.get_toolchain_tool('addr2line'), +# self.get_toolchain_tool('addr2line'), # '-e', -# common.vmlinux, +# kwargs['vmlinux'], # '-f', # '-p', # ], # stdout=subprocess.PIPE, # stdin=subprocess.PIPE, # ) as proc, \ -# open(common.trace_txt_file, 'r') as infile, \ -# open(os.path.join(common.run_dir, 'trace-lines.txt'), 'w') as outfile \ +# open(kwargs['trace_txt_file'], 'r') as infile, \ +# open(os.path.join(kwargs['run_dir'], 'trace-lines.txt'), 'w') as outfile \ # : # for in_line in infile: # proc.stdin.write(get_pc(in_line).encode()) @@ -58,5 +59,5 @@ sys.exit(subprocess.Popen([ # stdout = proc.stdout.read() # outfile.write(stdout.decode()) # # TODO -# # sed -E "s|at ${common.linux_build_dir}/(\./\|)||" +# # sed -E "s|at ${kwargs['linux_build_dir']}/(\./\|)||" # # uniq -c diff --git a/userland/Makefile b/userland/Makefile index a02e229..bb0bee3 100644 --- a/userland/Makefile +++ b/userland/Makefile @@ -46,7 +46,9 @@ OUTS := $(addprefix $(OUT_DIR)/,$(OUTS)) all: mkdir $(OUTS) for subdir in $(SUBDIRS); do \ - $(MAKE) -C $${subdir} OUT_DIR="$(OUT_DIR)/$$subdir"; \ + if [ -d "$${subdir}" ]; then \ + $(MAKE) -C "$${subdir}" OUT_DIR="$(OUT_DIR)/$$subdir"; \ + fi \ done $(COMMON_OBJ): $(COMMON_DIR)/$(COMMON_BASENAME)$(IN_EXT_C) @@ -64,7 +66,9 @@ $(OUT_DIR)/%$(OUT_EXT): %$(IN_EXT_CXX) $(COMMON_OBJ) clean: rm -f *'$(OBJ_EXT)' *'$(OUT_EXT)' for subdir in $(SUBDIRS); do \ - $(MAKE) -C $${subdir} clean; \ + if [ -d "$${subdir}" ]; then \ + $(MAKE) -C $${subdir} clean; \ + fi \ done mkdir: