From 1768421dbd2b29192fc38329a14acd13c18cf508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sat, 8 Dec 2018 00:00:01 +0000 Subject: [PATCH] args -> kwargs --- build | 36 +++---- build-baremetal | 58 +++++----- build-buildroot | 56 +++++----- build-crosstool-ng | 24 ++--- build-linux | 51 ++++----- build-m5 | 18 ++-- build-modules | 32 +++--- build-qemu | 26 ++--- build-userland | 28 ++--- common.py | 18 ++-- gem5-shell | 6 +- gem5-stat | 2 +- getvar | 4 +- qemu-monitor | 4 +- qemu-trace2txt | 6 +- run | 262 ++++++++++++++++++++++----------------------- run-docker | 32 +++--- run-gdb | 56 +++++----- run-gdb-user | 4 +- run-gdbserver | 4 +- run-toolchain | 8 +- shell_helpers.py | 2 +- trace-boot | 2 +- 23 files changed, 370 insertions(+), 369 deletions(-) diff --git a/build b/build index 400a8d0..a0c810c 100755 --- a/build +++ b/build @@ -51,9 +51,9 @@ def run_cmd(cmd, arch): cmd_abs = cmd.copy() cmd_abs[0] = os.path.join(common.root_dir, cmd[0]) cmd_abs.extend(['--arch', arch]) - if args.extra_args: - cmd_abs.append(args.extra_args) - self.sh.run_cmd(cmd_abs, dry_run=args.dry_run) + if kwargs['extra_args']: + cmd_abs.append(kwargs['extra_args']) + self.sh.run_cmd(cmd_abs, dry_run=kwargs['dry_run']) buildroot_component = Component( lambda arch: run_cmd(['build-buildroot'], arch), @@ -320,21 +320,21 @@ args = parser.parse_args() common.setup_dry_run_arguments(args) # Decide archs. -if args.arch == []: - if args.all or args.all_archs: +if kwargs['arch'] == []: + if kwargs['all'] or kwargs['all_archs']: archs = common.all_archs.copy() else: archs = set([common.default_arch]) else: archs = set() - for arch in args.arch: + for arch in kwargs['arch']: if arch in common.arch_short_to_long_dict: arch = common.arch_short_to_long_dict[arch] archs.add(arch) # Decide components. -components = args.components -if args.all: +components = kwargs['components'] +if kwargs['all']: components = ['all'] elif components == []: components = ['qemu-buildroot'] @@ -350,7 +350,7 @@ for component_name in components: selected_components.append(component) todo.extend(component.dependencies) -if args.download_dependencies: +if kwargs['download_dependencies']: apt_get_pkgs = { # Core requirements for this repo. 'git', @@ -388,7 +388,7 @@ if args.download_dependencies: python2_pkgs.update(component.python2_pkgs) python3_pkgs.update(component.python3_pkgs) if apt_get_pkgs or apt_build_deps: - if args.travis: + if kwargs['travis']: interacive_pkgs = { 'libsdl2-dev', } @@ -406,34 +406,34 @@ if args.download_dependencies: f.write(sources_txt) else: sudo = ['sudo'] - if common.in_docker or args.travis: + if common.in_docker or kwargs['travis']: y = ['-y'] else: y = [] self.sh.run_cmd( - sudo + ['apt-get', 'update', common.Newline] + sudo + ['apt-get', 'update', LF] ) if apt_get_pkgs: self.sh.run_cmd( - sudo + ['apt-get', 'install'] + y + [common.Newline] + + sudo + ['apt-get', 'install'] + y + [LF] + common.add_newlines(sorted(apt_get_pkgs)) ) if apt_build_deps: self.sh.run_cmd( sudo + - ['apt-get', 'build-dep'] + y + [common.Newline] + + ['apt-get', 'build-dep'] + y + [LF] + common.add_newlines(sorted(apt_build_deps)) ) if python2_pkgs: self.sh.run_cmd( - ['python', '-m', 'pip', 'install', '--user', common.Newline] + + ['python', '-m', 'pip', 'install', '--user', LF] + common.add_newlines(sorted(python2_pkgs)) ) if python3_pkgs: # Not with pip executable directly: # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 self.sh.run_cmd( - ['python3', '-m', 'pip', 'install', '--user', common.Newline] + + ['python3', '-m', 'pip', 'install', '--user', LF] + common.add_newlines(sorted(python3_pkgs)) ) git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive'] @@ -449,7 +449,7 @@ if args.download_dependencies: # # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 self.sh.run_cmd( - git_cmd_common + ['--', common.Newline] + + git_cmd_common + ['--', LF] + common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules)]) ) if submodules_shallow: @@ -473,7 +473,7 @@ if args.download_dependencies: # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac # self.sh.run_cmd( - git_cmd_common + ['--depth', '1', '--', common.Newline] + + git_cmd_common + ['--depth', '1', '--', LF] + common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules_shallow)]) ) diff --git a/build-baremetal b/build-baremetal index 75c76c7..7383118 100755 --- a/build-baremetal +++ b/build-baremetal @@ -6,7 +6,7 @@ import common class BaremetalComponent(common.Component): def do_build(self, args): - common.assert_crosstool_ng_supports_arch(args.arch) + common.assert_crosstool_ng_supports_arch(kwargs['arch']) build_dir = self.get_build_dir(args) bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext)) common_basename_noext = 'common' @@ -17,14 +17,14 @@ class BaremetalComponent(common.Component): syscalls_obj = os.path.join(common.baremetal_build_lib_dir, syscalls_basename_noext + common.obj_ext) common_objs = [common_obj, syscalls_obj] cflags = [ - '-I', common.baremetal_src_lib_dir, common.Newline, - '-I', common.root_dir, common.Newline, - '-O0', common.Newline, - '-ggdb3', common.Newline, - '-mcpu={}'.format(common.mcpu), common.Newline, - '-nostartfiles', common.Newline, + '-I', common.baremetal_src_lib_dir, LF, + '-I', common.root_dir, LF, + '-O0', LF, + '-ggdb3', LF, + '-mcpu={}'.format(common.mcpu), LF, + '-nostartfiles', LF, ] - if args.prebuilt: + if kwargs['prebuilt']: gcc = 'arm-none-eabi-gcc' else: os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH'] @@ -38,21 +38,21 @@ class BaremetalComponent(common.Component): uart_address = 0x10009000 else: raise Exception('unknown machine: ' + common.machine) - cflags.extend(['-D', 'GEM5'.format(uart_address), common.Newline]) + cflags.extend(['-D', 'GEM5'.format(uart_address), LF]) else: entry_address = 0x40000000 uart_address = 0x09000000 os.makedirs(build_dir, exist_ok=True) os.makedirs(common.baremetal_build_lib_dir, exist_ok=True) - src = os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)) + src = os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(kwargs['arch'], common.asm_ext)) if common.need_rebuild([src], bootloader_obj): self.sh.run_cmd( - [gcc, common.Newline] + + [gcc, LF] + cflags + [ - '-c', common.Newline, - '-o', bootloader_obj, common.Newline, - src, common.Newline, + '-c', LF, + '-o', bootloader_obj, LF, + src, LF, ] ) for src, obj in [ @@ -61,13 +61,13 @@ class BaremetalComponent(common.Component): ]: if common.need_rebuild([src], obj): self.sh.run_cmd( - [gcc, common.Newline] + + [gcc, LF] + cflags + [ - '-c', common.Newline, - '-D', 'UART0_ADDR={:#x}'.format(uart_address), common.Newline, - '-o', obj, common.Newline, - src, common.Newline, + '-c', LF, + '-D', 'UART0_ADDR={:#x}'.format(uart_address), LF, + '-o', obj, LF, + src, LF, ] ) self._build_dir( @@ -86,7 +86,7 @@ class BaremetalComponent(common.Component): bootloader_obj=bootloader_obj, common_objs=common_objs, ) - arch_dir = os.path.join('arch', args.arch) + arch_dir = os.path.join('arch', kwargs['arch']) if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)): self._build_dir( arch_dir, @@ -96,7 +96,7 @@ class BaremetalComponent(common.Component): bootloader_obj=bootloader_obj, common_objs=common_objs, ) - arch_dir = os.path.join('arch', args.arch, 'no_bootloader') + arch_dir = os.path.join('arch', kwargs['arch'], 'no_bootloader') if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)): self._build_dir( arch_dir, @@ -151,12 +151,12 @@ Build the baremetal examples with crosstool-NG. src = os.path.join(common.baremetal_src_dir, in_path) if common.need_rebuild([src], main_obj): self.sh.run_cmd( - [gcc, common.Newline] + + [gcc, LF] + cflags + [ - '-c', common.Newline, - '-o', main_obj, common.Newline, - src, common.Newline, + '-c', LF, + '-o', main_obj, LF, + src, LF, ] ) objs = common_objs + [main_obj] @@ -164,12 +164,12 @@ Build the baremetal examples with crosstool-NG. link_script = os.path.join(common.baremetal_src_dir, 'link.ld') if common.need_rebuild(objs + [link_script], out): self.sh.run_cmd( - [gcc, common.Newline] + + [gcc, LF] + cflags + [ - '-Wl,--section-start=.text={:#x}'.format(entry_address), common.Newline, - '-o', out, common.Newline, - '-T', link_script, common.Newline, + '-Wl,--section-start=.text={:#x}'.format(entry_address), LF, + '-o', out, LF, + '-T', link_script, LF, ] + common.add_newlines(objs) ) diff --git a/build-buildroot b/build-buildroot index 6b2ea87..9b955fb 100755 --- a/build-buildroot +++ b/build-buildroot @@ -68,16 +68,16 @@ usually extra Buildroot targets. def do_build(self, args): build_dir = self.get_build_dir(args) os.makedirs(common.out_dir, exist_ok=True) - extra_make_args = common.add_newlines(args.extra_make_args) - if args.build_linux: - extra_make_args.extend(['linux-reconfigure', common.Newline]) + extra_make_args = common.add_newlines(kwargs['extra_make_args']) + if kwargs['build_linux']: + extra_make_args.extend(['linux-reconfigure', LF]) if common.emulator == 'gem5': - extra_make_args.extend(['gem5-reconfigure', common.Newline]) - if args.arch == 'x86_64': + extra_make_args.extend(['gem5-reconfigure', LF]) + if kwargs['arch'] == 'x86_64': defconfig = 'qemu_x86_64_defconfig' - elif args.arch == 'arm': + elif kwargs['arch'] == 'arm': defconfig = 'qemu_arm_vexpress_defconfig' - elif args.arch == 'aarch64': + elif kwargs['arch'] == 'aarch64': defconfig = 'qemu_aarch64_virt_defconfig' br2_external_dirs = [] for package_dir in os.listdir(common.packages_dir): @@ -87,24 +87,24 @@ usually extra Buildroot targets. br2_external_str = ':'.join(br2_external_dirs) self.sh.run_cmd( [ - 'make', common.Newline, - 'O={}'.format(common.buildroot_build_dir), common.Newline, - 'BR2_EXTERNAL={}'.format(br2_external_str), common.Newline, - defconfig, common.Newline, + 'make', LF, + 'O={}'.format(common.buildroot_build_dir), LF, + 'BR2_EXTERNAL={}'.format(br2_external_str), LF, + defconfig, LF, ], cwd=common.buildroot_src_dir, ) - configs = args.config + configs = kwargs['config'] configs.extend([ - 'BR2_JLEVEL={}'.format(args.nproc), + 'BR2_JLEVEL={}'.format(kwargs['nproc']), 'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir), ]) - if not args.build_linux: + if not kwargs['build_linux']: configs.extend([ '# BR2_LINUX_KERNEL is not set', ]) config_fragments = [] - if not args.baseline: + if not kwargs['baseline']: configs.extend([ 'BR2_GLOBAL_PATCH_DIR="{}"'.format( self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global')) @@ -122,34 +122,34 @@ usually extra Buildroot targets. self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table')) ), ]) - if not args.no_overlay: + if not kwargs['no_overlay']: configs.append('BR2_ROOTFS_OVERLAY="{}"'.format( self._path_relative_to_buildroot(common.out_rootfs_overlay_dir) )) config_fragments = [ os.path.join(common.root_dir, 'buildroot_config', 'default') - ] + args.config_fragment + ] + kwargs['config_fragment'] # TODO Can't get rid of these for now with nice fragments on Buildroot: # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config self.sh.write_configs(common.buildroot_config_file, configs, config_fragments) self.sh.run_cmd( [ - 'make', common.Newline, - 'O={}'.format(common.buildroot_build_dir), common.Newline, - 'olddefconfig', common.Newline, + 'make', LF, + 'O={}'.format(common.buildroot_build_dir), LF, + 'olddefconfig', LF, ], cwd=common.buildroot_src_dir, ) common.make_build_dirs() - if not args.no_all: - extra_make_args.extend(['all', common.Newline]) + if not kwargs['no_all']: + extra_make_args.extend(['all', LF]) self.sh.run_cmd( [ - 'make', common.Newline, - 'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_source_dir), common.Newline, - 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), common.Newline, - 'O={}'.format(common.buildroot_build_dir), common.Newline, - 'V={}'.format(int(args.verbose)), common.Newline, + 'make', LF, + 'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_source_dir), LF, + 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), LF, + 'O={}'.format(common.buildroot_build_dir), LF, + 'V={}'.format(int(kwargs['verbose'])), LF, ] + extra_make_args , @@ -160,7 +160,7 @@ usually extra Buildroot targets. # Create the qcow2 from ext2. # Skip if qemu is not present, because gem5 does not need the qcow2. # so we don't force a QEMU build for gem5. - if not args.no_all and os.path.exists(common.qemu_img_executable): + if not kwargs['no_all'] and os.path.exists(common.qemu_img_executable): common.raw_to_qcow2() def get_argparse_args(self): diff --git a/build-crosstool-ng b/build-crosstool-ng index 57799a0..b497d55 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -6,7 +6,7 @@ import common class CrosstoolNgComponent(common.Component): def do_build(self, args): - common.assert_crosstool_ng_supports_arch(args.arch) + common.assert_crosstool_ng_supports_arch(kwargs['arch']) build_dir = self.get_build_dir(args) defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig') os.makedirs(common.crosstool_ng_util_dir, exist_ok=True) @@ -16,25 +16,25 @@ class CrosstoolNgComponent(common.Component): # https://github.com/crosstool-ng/crosstool-ng/issues/1021 os.chdir(common.crosstool_ng_src_dir) self.sh.run_cmd( - [os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), common.Newline], + [os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), LF], ) os.chdir(common.crosstool_ng_util_dir) self.sh.run_cmd( [ - os.path.join(common.crosstool_ng_src_dir, 'configure'), common.Newline, - '--enable-local', common.Newline, + os.path.join(common.crosstool_ng_src_dir, 'configure'), LF, + '--enable-local', LF, ], ) self.sh.run_cmd( [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, + 'make', LF, + '-j', str(kwargs['nproc']), LF, ], ) # Build the toolchain. self.sh.cp( - os.path.join(common.root_dir, 'crosstool_ng_config', args.arch), + os.path.join(common.root_dir, 'crosstool_ng_config', kwargs['arch']), defconfig_dest ) common.write_configs( @@ -47,16 +47,16 @@ class CrosstoolNgComponent(common.Component): ) self.sh.run_cmd( [ - common.crosstool_ng_executable, common.Newline, - 'defconfig', common.Newline, + common.crosstool_ng_executable, LF, + 'defconfig', LF, ], ) os.unlink(defconfig_dest) self.sh.run_cmd( [ - common.crosstool_ng_executable, common.Newline, - 'build', common.Newline, - 'CT_JOBS={}'.format(str(args.nproc)), common.Newline, + common.crosstool_ng_executable, LF, + 'build', LF, + 'CT_JOBS={}'.format(str(kwargs['nproc'])), LF, ], out_file=os.path.join(build_dir, 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], diff --git a/build-linux b/build-linux index 90a8a85..a3902ff 100755 --- a/build-linux +++ b/build-linux @@ -4,6 +4,7 @@ import os import shutil import common +from shell_helpers import LF class LinuxComponent(common.Component): def __init__(self, parser): @@ -45,7 +46,7 @@ Configure the kernel, but don't build it. def build(self, **kwargs): build_dir = self.get_build_dir(**kwargs) - if args.initrd or args.initramfs: + if kwargs['initrd'] or kwargs['initramfs']: raise Exception('just trolling, --initrd and --initramfs are broken for now') os.makedirs(build_dir, exist_ok=True) tool = 'gcc' @@ -59,32 +60,32 @@ Configure the kernel, but don't build it. cc = '{} {}'.format(ccache, gcc) else: cc = gcc - if args.verbose: + if kwargs['verbose']: verbose = ['V=1'] else: verbose = [] common_make_args = [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, - 'ARCH={}'.format(common.linux_arch), common.Newline, - 'CROSS_COMPILE={}'.format(prefix), common.Newline, - 'CC={}'.format(cc), common.Newline, - 'O={}'.format(build_dir), common.Newline, + 'make', LF, + '-j', str(kwargs['nproc']), LF, + 'ARCH={}'.format(common.linux_arch), LF, + 'CROSS_COMPILE={}'.format(prefix), LF, + 'CC={}'.format(cc), LF, + 'O={}'.format(build_dir), LF, ] + verbose - if args.custom_config_file is not None: - if not os.path.exists(args.custom_config_file): - raise Exception('config fragment file does not exist: {}'.format(args.custom_config_file)) - base_config_file = args.custom_config_file + if kwargs['custom_config_file'] is not None: + if not os.path.exists(kwargs['custom_config_file']): + raise Exception('config fragment file does not exist: {}'.format(kwargs['custom_config_file'])) + base_config_file = kwargs['custom_config_file'] config_fragments = [] else: - base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch)) + base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(kwargs['arch'])) config_fragments = ['min', 'default'] for i, config_fragment in enumerate(config_fragments): config_fragments[i] = os.path.join(common.linux_config_dir, config_fragment) - config_fragments.extend(args.config_fragment) - if args.config != []: + config_fragments.extend(kwargs['config_fragment']) + if kwargs['config'] != []: cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment') - cli_config_str = '\n'.join(args.config) + cli_config_str = '\n'.join(kwargs['config']) common.write_string_to_file(cli_config_fragment_path, cli_config_str) config_fragments.append(cli_config_fragment_path) self.sh.cp( @@ -93,25 +94,25 @@ Configure the kernel, but don't build it. ) self.sh.run_cmd( [ - os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), common.Newline, - '-m', common.Newline, - '-O', build_dir, common.Newline, - os.path.join(build_dir, '.config'), common.Newline, + os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), LF, + '-m', LF, + '-O', build_dir, LF, + os.path.join(build_dir, '.config'), LF, ] + common.add_newlines(config_fragments) ) self.sh.run_cmd( ( common_make_args + - ['olddefconfig', common.Newline] + ['olddefconfig', LF] ), **common_args ) - if not args.config_only: + if not kwargs['config_only']: self.sh.run_cmd( ( common_make_args + - common.add_newlines(args.extra_make_args) + common.add_newlines(kwargs['extra_make_args']) ), **common_args ) @@ -119,8 +120,8 @@ Configure the kernel, but don't build it. ( common_make_args + [ - 'INSTALL_MOD_PATH={}'.format(common.out_rootfs_overlay_dir), common.Newline, - 'modules_install', common.Newline, + 'INSTALL_MOD_PATH={}'.format(common.out_rootfs_overlay_dir), LF, + 'modules_install', LF, ] ), **common_args diff --git a/build-m5 b/build-m5 index c7f022a..86983eb 100755 --- a/build-m5 +++ b/build-m5 @@ -9,17 +9,17 @@ class M5Component(common.Component): allowed_toolchains = ['buildroot'] cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) ld = common.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains) - if args.arch == 'x86_64': + if kwargs['arch'] == 'x86_64': arch = 'x86' else: - arch = args.arch + arch = kwargs['arch'] return [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, - '-f', 'Makefile.{}'.format(arch), common.Newline, - 'CC={}'.format(cc), common.Newline, - 'LD={}'.format(ld), common.Newline, - 'PWD={}'.format(common.gem5_m5_source_dir), common.Newline, + 'make', LF, + '-j', str(kwargs['nproc']), LF, + '-f', 'Makefile.{}'.format(arch), LF, + 'CC={}'.format(cc), LF, + 'LD={}'.format(ld), LF, + 'PWD={}'.format(common.gem5_m5_source_dir), LF, ] def do_build(self, args): @@ -36,7 +36,7 @@ class M5Component(common.Component): def clean(self, args): self.sh.run_cmd( - self.get_make_cmd(args) + ['clean', common.Newline], + self.get_make_cmd(args) + ['clean', LF], cwd=common.gem5_m5_source_dir, ) diff --git a/build-modules b/build-modules index 5fe1a08..0bb2f5d 100755 --- a/build-modules +++ b/build-modules @@ -58,13 +58,13 @@ Use the host packaged cross toolchain. noext, ext = os.path.splitext(basename) if ext == common.c_ext: all_kernel_modules.append(noext) - if args.kernel_modules == []: + if kwargs['kernel_modules'] == []: kernel_modules = all_kernel_modules else: - kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], args.kernel_modules) + kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], kwargs['kernel_modules']) object_files = map(lambda x: x + common.obj_ext, kernel_modules) tool = 'gcc' - if args.host: + if kwargs['host']: allowed_toolchains = ['host'] build_subdir = common.kernel_modules_build_host_subdir else: @@ -77,32 +77,32 @@ Use the host packaged cross toolchain. cc = '{} {}'.format(ccache, gcc) else: cc = gcc - if args.verbose: + if kwargs['verbose']: verbose = ['V=1'] else: verbose = [] - if args.host: + if kwargs['host']: linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build') else: linux_dir = common.linux_build_dir self.sh.run_cmd( ( [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, - 'ARCH={}'.format(common.linux_arch), common.Newline, - 'CC={}'.format(cc), common.Newline, - 'CROSS_COMPILE={}'.format(prefix), common.Newline, - 'LINUX_DIR={}'.format(linux_dir), common.Newline, - 'M={}'.format(build_subdir), common.Newline, - 'OBJECT_FILES={}'.format(' '.join(object_files)), common.Newline, + 'make', LF, + '-j', str(kwargs['nproc']), LF, + 'ARCH={}'.format(common.linux_arch), LF, + 'CC={}'.format(cc), LF, + 'CROSS_COMPILE={}'.format(prefix), LF, + 'LINUX_DIR={}'.format(linux_dir), LF, + 'M={}'.format(build_subdir), LF, + 'OBJECT_FILES={}'.format(' '.join(object_files)), LF, ] + - common.shlex_split(args.make_args) + + common.shlex_split(kwargs['make_args']) + verbose ), cwd=os.path.join(common.kernel_modules_build_subdir), ) - if not args.host: + if not kwargs['host']: common.copy_dir_if_update_non_recursive( srcdir=common.kernel_modules_build_subdir, destdir=common.out_rootfs_overlay_dir, @@ -119,7 +119,7 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host } def get_build_dir(self, args): - if args.host: + if kwargs['host']: return os.path.join(common.kernel_modules_build_host_dir) else: return os.path.join(common.kernel_modules_build_dir) diff --git a/build-qemu b/build-qemu index f867052..b633d9a 100755 --- a/build-qemu +++ b/build-qemu @@ -22,32 +22,32 @@ class QemuComponent(common.Component): def do_build(self, args): build_dir = self.get_build_dir(args) os.makedirs(build_dir, exist_ok=True) - if args.verbose: + if kwargs['verbose']: verbose = ['V=1'] else: verbose = [] - if args.userland: - target_list = '{}-linux-user'.format(args.arch) + if kwargs['userland']: + target_list = '{}-linux-user'.format(kwargs['arch']) else: - target_list = '{}-softmmu'.format(args.arch) + target_list = '{}-softmmu'.format(kwargs['arch']) self.sh.run_cmd( [ - os.path.join(common.qemu_src_dir, 'configure'), common.Newline, - '--enable-debug', common.Newline, - '--enable-trace-backends=simple', common.Newline, - '--target-list={}'.format(target_list), common.Newline, - '--enable-sdl', common.Newline, - '--with-sdlabi=2.0', common.Newline, + os.path.join(common.qemu_src_dir, 'configure'), LF, + '--enable-debug', LF, + '--enable-trace-backends=simple', LF, + '--target-list={}'.format(target_list), LF, + '--enable-sdl', LF, + '--with-sdlabi=2.0', LF, ] + - common.add_newlines(args.extra_config_args), + common.add_newlines(kwargs['extra_config_args']), extra_paths=[common.ccache_dir], cwd=build_dir ) self.sh.run_cmd( ( [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, + 'make', LF, + '-j', str(kwargs['nproc']), LF, ] + verbose diff --git a/build-userland b/build-userland index be39c51..3e7274c 100755 --- a/build-userland +++ b/build-userland @@ -47,7 +47,7 @@ has the OpenBLAS libraries and headers installed. def do_build(self, args): build_dir = self.get_build_dir(args) os.makedirs(build_dir, exist_ok=True) - if args.host: + if kwargs['host']: allowed_toolchains = ['host'] else: allowed_toolchains = ['buildroot'] @@ -56,20 +56,20 @@ has the OpenBLAS libraries and headers installed. self.sh.run_cmd( ( [ - 'make', common.Newline, - '-j', str(args.nproc), common.Newline, - 'ARCH={}'.format(args.arch), common.Newline, - 'CCFLAGS_SCRIPT={} {}'.format('-I', common.userland_src_dir), common.Newline, - 'COMMON_DIR={}'.format(common.root_dir), common.Newline, - 'CC={}'.format(cc), common.Newline, - 'CXX={}'.format(cxx), common.Newline, - 'PKG_CONFIG={}'.format(common.buildroot_pkg_config), common.Newline, - 'STAGING_DIR={}'.format(common.buildroot_staging_dir), common.Newline, - 'OUT_DIR={}'.format(build_dir), common.Newline, + 'make', LF, + '-j', str(kwargs['nproc']), LF, + 'ARCH={}'.format(kwargs['arch']), LF, + 'CCFLAGS_SCRIPT={} {}'.format('-I', common.userland_src_dir), LF, + 'COMMON_DIR={}'.format(common.root_dir), LF, + 'CC={}'.format(cc), LF, + 'CXX={}'.format(cxx), LF, + 'PKG_CONFIG={}'.format(common.buildroot_pkg_config), LF, + 'STAGING_DIR={}'.format(common.buildroot_staging_dir), LF, + 'OUT_DIR={}'.format(build_dir), LF, ] + - common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in args.has_package]) + - shlex.split(args.make_args) + - common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets]) + common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in kwargs['has_package']]) + + shlex.split(kwargs['make_args']) + + common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in kwargs['targets']]) ), cwd=common.userland_src_dir, extra_paths=[common.ccache_dir], diff --git a/common.py b/common.py index 195ed56..b703398 100644 --- a/common.py +++ b/common.py @@ -162,10 +162,10 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree Linux build ID. Allows you to keep multiple separate Linux builds. ''' ) - parser.add_argument( + self.add_argument( '--initramfs', default=False, action='store_true', ) - parser.add_argument( + self.add_argument( '--initrd', default=False, action='store_true', ) @@ -473,7 +473,7 @@ def raw_to_qcow2(prebuilt=False, reverse=False): qemu_img_executable = common.qemu_img_basename else: # Prevent qemu-img from generating trace files like QEMU. Disgusting. - disable_trace = ['-T', 'pr_manager_run,file=/dev/null', common.Newline,] + disable_trace = ['-T', 'pr_manager_run,file=/dev/null', LF,] qemu_img_executable = common.qemu_img_executable infmt = 'raw' outfmt = 'qcow2' @@ -488,15 +488,15 @@ def raw_to_qcow2(prebuilt=False, reverse=False): outfile = tmp self.sh.run_cmd( [ - qemu_img_executable, common.Newline, + qemu_img_executable, LF, ] + disable_trace + [ - 'convert', common.Newline, - '-f', infmt, common.Newline, - '-O', outfmt, common.Newline, - infile, common.Newline, - outfile, common.Newline, + 'convert', LF, + '-f', infmt, LF, + '-O', outfmt, LF, + infile, LF, + outfile, LF, ] ) diff --git a/gem5-shell b/gem5-shell index 699181e..7ba405a 100755 --- a/gem5-shell +++ b/gem5-shell @@ -10,7 +10,7 @@ parser = common.get_argparse( ) args = common.setup(parser) sys.exit(self.sh.run_cmd([ - common.gem5_m5term, common.Newline, - 'localhost', common.Newline, - str(common.gem5_telnet_port), common.Newline, + common.gem5_m5term, LF, + 'localhost', LF, + str(common.gem5_telnet_port), LF, ])) diff --git a/gem5-stat b/gem5-stat index 9b19b4c..c0e7b84 100755 --- a/gem5-stat +++ b/gem5-stat @@ -10,5 +10,5 @@ parser.add_argument( nargs='?', ) args = common.setup(parser) -stats = common.get_stats(args.stat) +stats = common.get_stats(kwargs['stat']) print('\n'.join(stats)) diff --git a/getvar b/getvar index 431d4f9..e706967 100755 --- a/getvar +++ b/getvar @@ -28,8 +28,8 @@ List all available variables: }) parser.add_argument('variable', nargs='?') args = common.setup(parser) -if args.variable: - print(getattr(common, args.variable)) +if kwargs['variable']: + print(getattr(common, kwargs['variable'])) else: for attr in dir(common): if not attr.startswith('__'): diff --git a/qemu-monitor b/qemu-monitor index a2bcd9d..9c0ad1d 100755 --- a/qemu-monitor +++ b/qemu-monitor @@ -33,12 +33,12 @@ with telnetlib.Telnet('localhost', common.qemu_monitor_port) as tn: # sock = tn.get_socket() # sock.send(telnetlib.IAC + telnetlib.WILL + telnetlib.ECHO) if os.isatty(sys.stdin.fileno()): - if args.command == []: + if kwargs['command'] == []: print(tn.read_until(prompt).decode('utf-8'), end='') tn.interact() else: tn.read_until(prompt) - print(write_and_read(tn, ' '.join(args.command) + '\n', prompt)) + print(write_and_read(tn, ' '.join(kwargs['command']) + '\n', prompt)) else: tn.read_until(prompt) print(write_and_read(tn, sys.stdin.read() + '\n', prompt)) diff --git a/qemu-trace2txt b/qemu-trace2txt index 89d634c..8de5815 100755 --- a/qemu-trace2txt +++ b/qemu-trace2txt @@ -9,9 +9,9 @@ import common def main(): return self.sh.run_cmd( [ - os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), common.Newline, - os.path.join(common.qemu_build_dir, 'trace-events-all'), common.Newline, - os.path.join(common.qemu_trace_file), common.Newline, + os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), LF, + os.path.join(common.qemu_build_dir, 'trace-events-all'), LF, + os.path.join(common.qemu_trace_file), LF, ], cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'), out_file=common.qemu_trace_txt_file, diff --git a/run b/run index 1654279..bedfe60 100755 --- a/run +++ b/run @@ -48,79 +48,79 @@ def main(args, extra_args=None): # * https://stackoverflow.com/questions/44612822/unable-to-debug-kernel-with-qemu-gdb/49840927#49840927 # Turned on by default since v4.12 kernel_cli = 'console_msg_format=syslog nokaslr norandmaps panic=-1 printk.devkmsg=on printk.time=y rw' - if args.kernel_cli is not None: - kernel_cli += ' {}'.format(args.kernel_cli) + if kwargs['kernel_cli'] is not None: + kernel_cli += ' {}'.format(kwargs['kernel_cli']) kernel_cli_after_dash = '' extra_emulator_args = [] extra_qemu_args = [] - if args.debug_vm is not None: - debug_vm = ['gdb', common.Newline, '-q', common.Newline] + common.shlex_split(args.debug_vm) + ['--args', common.Newline] + if kwargs['debug_vm'] is not None: + debug_vm = ['gdb', LF, '-q', common.Newline] + common.shlex_split(kwargs['debug_vm']) + ['--args', common.Newline] else: debug_vm = [] - if args.wait_gdb: - extra_qemu_args.extend(['-S', common.Newline]) - if args.eval_after is not None: - kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(common.base64_encode(args.eval_after)) - if args.kernel_cli_after_dash is not None: - kernel_cli_after_dash += ' {}'.format(args.kernel_cli_after_dash) - if args.vnc: - vnc = ['-vnc', ':0', common.Newline] + if kwargs['wait_gdb']: + extra_qemu_args.extend(['-S', LF]) + if kwargs['eval_after'] is not None: + kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(common.base64_encode(kwargs['eval_after'])) + if kwargs['kernel_cli_after_dash'] is not None: + kernel_cli_after_dash += ' {}'.format(kwargs['kernel_cli_after_dash']) + if kwargs['vnc']: + vnc = ['-vnc', ':0', LF] else: vnc = [] - if args.initrd or args.initramfs: + if kwargs['initrd'] or kwargs['initramfs']: ramfs = True else: ramfs = False - if args.eval is not None: + if kwargs['eval'] is not None: if ramfs: initarg = 'rdinit' else: initarg = 'init' kernel_cli += ' {}=/eval_base64.sh'.format(initarg) - kernel_cli_after_dash += ' lkmc_eval="{}"'.format(common.base64_encode(args.eval)) - if not args.graphic: - extra_qemu_args.extend(['-nographic', common.Newline]) + kernel_cli_after_dash += ' lkmc_eval="{}"'.format(common.base64_encode(kwargs['eval'])) + if not kwargs['graphic']: + extra_qemu_args.extend(['-nographic', LF]) console = None console_type = None console_count = 0 - if args.arch == 'x86_64': + if kwargs['arch'] == 'x86_64': console_type = 'ttyS' elif common.is_arm: console_type = 'ttyAMA' console = '{}{}'.format(console_type, console_count) console_count += 1 - if not (args.arch == 'x86_64' and args.graphic): + if not (kwargs['arch'] == 'x86_64' and kwargs['graphic']): kernel_cli += ' console={}'.format(console) extra_console = '{}{}'.format(console_type, console_count) console_count += 1 - if args.kdb or args.kgdb: + if kwargs['kdb'] or kwargs['kgdb']: kernel_cli += ' kgdbwait' - if args.kdb: - if args.graphic: + if kwargs['kdb']: + if kwargs['graphic']: kdb_cmd = 'kbd,' else: kdb_cmd = '' kernel_cli += ' kgdboc={}{},115200'.format(kdb_cmd, console) - if args.kgdb: + if kwargs['kgdb']: kernel_cli += ' kgdboc={},115200'.format(extra_console) if kernel_cli_after_dash: kernel_cli += " -{}".format(kernel_cli_after_dash) extra_env = {} - if args.trace is None: + if kwargs['trace'] is None: do_trace = False # A dummy value that is already turned on by default and does not produce large output, # just to prevent QEMU from emitting a warning that '' is not valid. trace_type = 'load_file' else: do_trace = True - trace_type = args.trace + trace_type = kwargs['trace'] def raise_rootfs_not_found(): - if not args.dry_run: + if not kwargs['dry_run']: raise Exception('Root filesystem not found. Did you build it?\n' \ 'Tried to use: ' + common.disk_image) def raise_image_not_found(): - if not args.dry_run: + if not kwargs['dry_run']: raise Exception('Executable image not found. Did you build it?\n' \ 'Tried to use: ' + common.image) if common.image is None: @@ -131,7 +131,7 @@ def main(args, extra_args=None): if not os.path.exists(common.rootfs_raw_file): if not os.path.exists(common.qcow2_file): raise_rootfs_not_found() - common.raw_to_qcow2(prebuilt=args.prebuilt, reverse=True) + common.raw_to_qcow2(prebuilt=kwargs['prebuilt'], reverse=True) else: if not os.path.exists(common.gem5_fake_iso): os.makedirs(os.path.dirname(common.gem5_fake_iso), exist_ok=True) @@ -142,111 +142,111 @@ def main(args, extra_args=None): raise_image_not_found() self.sh.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True) - common.write_string_to_file(common.gem5_readfile, args.gem5_readfile) - memory = '{}B'.format(args.memory) - gem5_exe_args = common.shlex_split(args.gem5_exe_args) + common.write_string_to_file(common.gem5_readfile, kwargs['gem5_readfile']) + memory = '{}B'.format(kwargs['memory']) + gem5_exe_args = common.shlex_split(kwargs['gem5_exe_args']) if do_trace: - gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), common.Newline]) + gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), LF]) extra_env['M5_PATH'] = common.gem5_system_dir # https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071 extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true' - if args.trace_stdout: + if kwargs['trace_stdout']: debug_file = 'cout' else: debug_file = 'trace.txt' cmd.extend( [ - common.executable, common.Newline, - '--debug-file', debug_file, common.Newline, - '--listener-mode', 'on', common.Newline, - '--outdir', common.m5out_dir, common.Newline, + common.executable, LF, + '--debug-file', debug_file, LF, + '--listener-mode', 'on', LF, + '--outdir', common.m5out_dir, LF, ] + gem5_exe_args ) - if args.userland is not None: + if kwargs['userland'] is not None: cmd.extend([ - common.gem5_se_file, common.Newline, - '-c', common.resolve_userland(args.userland), common.Newline, + common.gem5_se_file, LF, + '-c', common.resolve_userland(kwargs['userland']), LF, ]) else: - if args.gem5_script == 'fs': + if kwargs['gem5_script'] == 'fs': # TODO port - if args.gem5_restore is not None: + if kwargs['gem5_restore'] is not None: cpt_dirs = common.gem_list_checkpoint_dirs() - cpt_dir = cpt_dirs[-args.gem5_restore] + cpt_dir = cpt_dirs[-kwargs['gem5_restore']] extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) cmd.extend([ - common.gem5_fs_file, common.Newline, - '--disk-image', common.disk_image, common.Newline, - '--kernel', common.image, common.Newline, - '--mem-size', memory, common.Newline, - '--num-cpus', str(args.cpus), common.Newline, - '--script', common.gem5_readfile, common.Newline, + common.gem5_fs_file, LF, + '--disk-image', common.disk_image, LF, + '--kernel', common.image, LF, + '--mem-size', memory, LF, + '--num-cpus', str(kwargs['cpus']), LF, + '--script', common.gem5_readfile, LF, ]) - if args.arch == 'x86_64': - if args.kvm: - cmd.extend(['--cpu-type', 'X86KvmCPU', common.Newline]) - cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), common.Newline]) + if kwargs['arch'] == 'x86_64': + if kwargs['kvm']: + cmd.extend(['--cpu-type', 'X86KvmCPU', LF]) + cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), LF]) elif common.is_arm: - if args.kvm: - cmd.extend(['--cpu-type', 'ArmV8KvmCPU', common.Newline]) + if kwargs['kvm']: + cmd.extend(['--cpu-type', 'ArmV8KvmCPU', LF]) cmd.extend([ # TODO why is it mandatory to pass mem= here? Not true for QEMU. # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? - '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), common.Newline, - '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)), common.Newline, - '--machine-type', common.machine, common.Newline, + '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), LF, + '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, kwargs['cpus'])), LF, + '--machine-type', common.machine, LF, ]) if common.baremetal is None: cmd.extend([ - '--param', 'system.panic_on_panic = True', common.Newline]) + '--param', 'system.panic_on_panic = True', LF]) else: cmd.extend([ - '--bare-metal', common.Newline, - '--param', 'system.auto_reset_addr = True', common.Newline, + '--bare-metal', LF, + '--param', 'system.auto_reset_addr = True', LF, ]) - if args.arch == 'aarch64': + if kwargs['arch'] == 'aarch64': # https://stackoverflow.com/questions/43682311/uart-communication-in-gem5-with-arm-bare-metal/50983650#50983650 - cmd.extend(['--param', 'system.highest_el_is_64 = True', common.Newline]) - elif args.gem5_script == 'biglittle': - if args.kvm: + cmd.extend(['--param', 'system.highest_el_is_64 = True', LF]) + elif kwargs['gem5_script'] == 'biglittle': + if kwargs['kvm']: cpu_type = 'kvm' else: cpu_type = 'atomic' - if args.gem5_restore is not None: - cpt_dir = common.gem_list_checkpoint_dirs()[-args.gem5_restore] + if kwargs['gem5_restore'] is not None: + cpt_dir = common.gem_list_checkpoint_dirs()[-kwargs['gem5_restore']] extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)]) cmd.extend([ - os.path.join(common.gem5_source_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), common.Newline, - '--big-cpus', '2', common.Newline, - '--cpu-type', cpu_type, common.Newline, - '--disk', common.disk_image, common.Newline, - '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), common.Newline, - '--kernel', common.image, common.Newline, - '--little-cpus', '2', common.Newline, + os.path.join(common.gem5_source_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF, + '--big-cpus', '2', LF, + '--cpu-type', cpu_type, LF, + '--disk', common.disk_image, LF, + '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), LF, + '--kernel', common.image, LF, + '--little-cpus', '2', LF, ]) - if args.wait_gdb: + if kwargs['wait_gdb']: # https://stackoverflow.com/questions/49296092/how-to-make-gem5-wait-for-gdb-to-connect-to-reliably-break-at-start-kernel-of-th - cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', common.Newline]) + cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', LF]) else: qemu_user_and_system_options = [ - '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), common.Newline, + '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), LF, ] - if args.userland is not None: - if args.wait_gdb: - debug_args = ['-g', str(common.gdb_port), common.Newline] + if kwargs['userland'] is not None: + if kwargs['wait_gdb']: + debug_args = ['-g', str(common.gdb_port), LF] else: debug_args = [] cmd.extend( [ - os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)), common.Newline, - '-L', common.target_dir, common.Newline + os.path.join(common.qemu_build_dir, '{}-linux-user'.format(kwargs['arch']), 'qemu-{}'.format(kwargs['arch'])), LF, + '-L', common.target_dir, LF ] + qemu_user_and_system_options + - common.shlex_split(args.userland_before) + + common.shlex_split(kwargs['userland_before']) + debug_args + [ - common.resolve_userland(args.userland), common.Newline + common.resolve_userland(kwargs['userland']), LF ] ) else: @@ -254,7 +254,7 @@ def main(args, extra_args=None): raise_image_not_found() extra_emulator_args.extend(extra_qemu_args) common.make_run_dirs() - if args.prebuilt or not os.path.exists(common.qemu_executable): + if kwargs['prebuilt'] or not os.path.exists(common.qemu_executable): qemu_executable = common.qemu_executable_basename qemu_executable_prebuilt = True else: @@ -264,16 +264,16 @@ def main(args, extra_args=None): if qemu_executable is None: raise Exception('QEMU executable not found, did you forget to build or install it?\n' \ 'Tried to use: ' + qemu_executable) - if args.debug_vm: + if kwargs['debug_vm']: serial_monitor = [] else: - if args.background: - serial_monitor = ['-serial', 'file:{}'.format(common.qemu_background_serial_file), common.Newline] + if kwargs['background']: + serial_monitor = ['-serial', 'file:{}'.format(common.qemu_background_serial_file), LF] else: - serial_monitor = ['-serial', 'mon:stdio', common.Newline] - if args.kvm: - extra_emulator_args.extend(['-enable-kvm', common.Newline]) - extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port), common.Newline]) + serial_monitor = ['-serial', 'mon:stdio', LF] + if kwargs['kvm']: + extra_emulator_args.extend(['-enable-kvm', LF]) + extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port), LF]) virtfs_data = [ (common.p9_dir, 'host_data'), (common.out_dir, 'host_out'), @@ -287,19 +287,19 @@ def main(args, extra_args=None): '-virtfs', 'local,path={virtfs_dir},mount_tag={virtfs_tag},security_model=mapped,id={virtfs_tag}' \ .format(virtfs_dir=virtfs_dir, virtfs_tag=virtfs_tag), - common.Newline, + LF, ]) cmd.extend( [ - qemu_executable, common.Newline, - '-device', 'rtl8139,netdev=net0', common.Newline, - '-gdb', 'tcp::{}'.format(common.gdb_port), common.Newline, - '-kernel', common.image, common.Newline, - '-m', args.memory, common.Newline, - '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), common.Newline, - '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), common.Newline, - '-no-reboot', common.Newline, - '-smp', str(args.cpus), common.Newline, + qemu_executable, LF, + '-device', 'rtl8139,netdev=net0', LF, + '-gdb', 'tcp::{}'.format(common.gdb_port), LF, + '-kernel', common.image, LF, + '-m', kwargs['memory'], LF, + '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), LF, + '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), LF, + '-no-reboot', LF, + '-smp', str(kwargs['cpus']), LF, ] + virtfs_cmd + serial_monitor + @@ -307,9 +307,9 @@ def main(args, extra_args=None): ) if not qemu_executable_prebuilt: cmd.extend(qemu_user_and_system_options) - if args.initrd: + if kwargs['initrd']: extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')]) - rr = args.record or args.replay + rr = kwargs['record'] or kwargs['replay'] if ramfs: # TODO why is this needed, and why any string works. root = 'root=/dev/anything' @@ -328,76 +328,76 @@ def main(args, extra_args=None): if not os.path.exists(common.qcow2_file): if not os.path.exists(common.rootfs_raw_file): raise_rootfs_not_found() - common.raw_to_qcow2(prebuilt=args.prebuilt) + common.raw_to_qcow2(prebuilt=kwargs['prebuilt']) extra_emulator_args.extend([ '-drive', 'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid), - common.Newline, + LF, ]) if rr: extra_emulator_args.extend([ - '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', common.Newline, - '-device', 'ide-hd,drive=img-blkreplay', common.Newline, + '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', LF, + '-device', 'ide-hd,drive=img-blkreplay', LF, ]) if rr: extra_emulator_args.extend([ '-object', 'filter-replay,id=replay,netdev=net0', - '-icount', 'shift=7,rr={},rrfile={}'.format('record' if args.record else 'replay', common.qemu_rrfile), + '-icount', 'shift=7,rr={},rrfile={}'.format('record' if kwargs['record'] else 'replay', common.qemu_rrfile), ]) virtio_gpu_pci = [] else: - virtio_gpu_pci = ['-device', 'virtio-gpu-pci', common.Newline] - if args.arch == 'x86_64': - append = ['-append', '{} nopat {}'.format(root, kernel_cli), common.Newline] + virtio_gpu_pci = ['-device', 'virtio-gpu-pci', LF] + if kwargs['arch'] == 'x86_64': + append = ['-append', '{} nopat {}'.format(root, kernel_cli), LF] cmd.extend([ - '-M', common.machine, common.Newline, - '-device', 'edu', common.Newline, + '-M', common.machine, LF, + '-device', 'edu', LF, ]) elif common.is_arm: - extra_emulator_args.extend(['-semihosting', common.Newline]) - if args.arch == 'arm': + extra_emulator_args.extend(['-semihosting', LF]) + if kwargs['arch'] == 'arm': cpu = 'cortex-a15' else: cpu = 'cortex-a57' - append = ['-append', '{} {}'.format(root, kernel_cli), common.Newline] + append = ['-append', '{} {}'.format(root, kernel_cli), LF] cmd.extend( [ # highmem=off needed since v3.0.0 due to: # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html - '-M', '{},highmem=off'.format(common.machine), common.Newline, - '-cpu', cpu, common.Newline, + '-M', '{},highmem=off'.format(common.machine), LF, + '-cpu', cpu, LF, ] + virtio_gpu_pci ) if common.baremetal is None: cmd.extend(append) - if args.tmux is not None: - tmux_args = '--run-id {}'.format(args.run_id) + if kwargs['tmux'] is not None: + tmux_args = '--run-id {}'.format(kwargs['run_id']) if common.emulator == 'gem5': tmux_cmd = './gem5-shell' - elif args.wait_gdb: + elif kwargs['wait_gdb']: tmux_cmd = './run-gdb' # TODO find a nicer way to forward all those args automatically. # Part of me wants to: https://github.com/jonathanslenders/pymux # but it cannot be used as a library properly it seems, and it is # slower than tmux. tmux_args += " --arch {} --linux-build-id '{}' --run-id '{}'".format( - args.arch, - args.linux_build_id, - args.run_id, + kwargs['arch'], + kwargs['linux_build_id'], + kwargs['run_id'], ) if common.baremetal: tmux_args += " --baremetal '{}'".format(common.baremetal) - if args.userland: - tmux_args += " --userland '{}'".format(args.userland) - tmux_args += ' {}'.format(args.tmux) + if kwargs['userland']: + tmux_args += " --userland '{}'".format(kwargs['userland']) + tmux_args += ' {}'.format(kwargs['tmux']) subprocess.Popen([ os.path.join(common.root_dir, 'tmu'), "sleep 2;{} {}".format(tmux_cmd, tmux_args) ]) cmd.extend(extra_emulator_args) - cmd.extend(args.extra_emulator_args) - if debug_vm or args.terminal: + cmd.extend(kwargs['extra_emulator_args']) + if debug_vm or kwargs['terminal']: out_file = None else: out_file = common.termout_file @@ -411,7 +411,7 @@ def main(args, extra_args=None): panic_msg = b'Kernel panic - not syncing' panic_re = re.compile(panic_msg) error_string_found = False - if out_file is not None and not args.dry_run: + if out_file is not None and not kwargs['dry_run']: with open(common.termout_file, 'br') as logfile: for line in logfile: if panic_re.search(line): diff --git a/run-docker b/run-docker index bfc3cba..d27302e 100755 --- a/run-docker +++ b/run-docker @@ -11,28 +11,28 @@ image_name = common.repo_short_id target_dir = '/root/{}'.format(common.repo_short_id) docker = ['sudo', 'docker'] def create(args): - self.sh.run_cmd(docker + ['build', '-t', image_name, '.', common.Newline]) + self.sh.run_cmd(docker + ['build', '-t', image_name, '.', LF]) # --privileged for KVM: # https://stackoverflow.com/questions/48422001/launching-qemu-kvm-from-inside-docker-container self.sh.run_cmd( docker + [ - 'create', common.Newline, - '--hostname', container_hostname, common.Newline, - '-i', common.Newline, - '--name', container_name, common.Newline, - '--net', 'host', common.Newline, - '--privileged', common.Newline, - '-t', common.Newline, - '-w', target_dir, common.Newline, - '-v', '{}:{}'.format(os.getcwd(), target_dir), common.Newline, + 'create', LF, + '--hostname', container_hostname, LF, + '-i', LF, + '--name', container_name, LF, + '--net', 'host', LF, + '--privileged', LF, + '-t', LF, + '-w', target_dir, LF, + '-v', '{}:{}'.format(os.getcwd(), target_dir), LF, image_name, ] ) def destroy(args): stop(args) - self.sh.run_cmd(docker + ['rm', container_name, common.Newline]) - self.sh.run_cmd(docker + ['rmi', image_name, common.Newline]) + self.sh.run_cmd(docker + ['rm', container_name, LF]) + self.sh.run_cmd(docker + ['rmi', image_name, LF]) def sh(args): start(args) if args: @@ -42,12 +42,12 @@ def sh(args): self.sh.run_cmd( docker + ['exec', '-i', '-t', container_name] + sh_args + - [common.Newline], + [LF], ) def start(args): - self.sh.run_cmd(docker + ['start', container_name, common.Newline]) + self.sh.run_cmd(docker + ['start', container_name, LF]) def stop(args): - self.sh.run_cmd(docker + ['stop', container_name, common.Newline]) + self.sh.run_cmd(docker + ['stop', container_name, LF]) cmd_action_map = { 'create': lambda args: create(args), 'DESTROY': lambda args: destroy(args), @@ -61,4 +61,4 @@ parser.add_argument('args', nargs='*') common.add_dry_run_argument(parser) args = parser.parse_args() common.setup_dry_run_arguments(args) -cmd_action_map[args.cmd](args.args) +cmd_action_map[kwargs['cmd']](kwargs['args']) diff --git a/run-gdb b/run-gdb index e0a80b6..3f6a037 100755 --- a/run-gdb +++ b/run-gdb @@ -98,31 +98,31 @@ def main(args, extra_args=None): ''' global defaults args = common.resolve_args(defaults, args, extra_args) - after = common.shlex_split(args.after) - before = common.shlex_split(args.before) - no_continue = args.no_continue - if args.test: + after = common.shlex_split(kwargs['after']) + before = common.shlex_split(kwargs['before']) + no_continue = kwargs['no_continue'] + if kwargs['test']: no_continue = True before.extend([ - '-q', common.Newline, - '-nh', common.Newline, - '-ex', 'set confirm off', common.Newline + '-q', LF, + '-nh', LF, + '-ex', 'set confirm off', LF ]) - elif args.verbose: + elif kwargs['verbose']: # The output of this would affect the tests. # https://stackoverflow.com/questions/13496389/gdb-remote-protocol-how-to-analyse-packets # Also be opinionated and set remotetimeout to allow you to step debug the emulator at the same time. before.extend([ - '-ex', 'set debug remote 1', common.Newline, - '-ex', 'set remotetimeout 99999', common.Newline, + '-ex', 'set debug remote 1', LF, + '-ex', 'set remotetimeout 99999', LF, ]) - if args.break_at is not None: - break_at = ['-ex', 'break {}'.format(args.break_at), common.Newline] + if kwargs['break_at'] is not None: + break_at = ['-ex', 'break {}'.format(kwargs['break_at']), LF] else: break_at = [] - linux_full_system = (common.baremetal is None and args.userland is None) - if args.userland: - image = common.resolve_userland(args.userland) + linux_full_system = (common.baremetal is None and kwargs['userland'] is None) + if kwargs['userland']: + image = common.resolve_userland(kwargs['userland']) elif common.baremetal: image = common.image test_script_path = os.path.splitext(common.source_path)[0] + '.py' @@ -133,25 +133,25 @@ def main(args, extra_args=None): else: allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] cmd = ( - [common.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains), common.Newline] + + [common.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains), LF] + before + - ['-q', common.Newline] + ['-q', LF] ) if linux_full_system: - cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), common.Newline]) - if args.sim: + cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), LF]) + if kwargs['sim']: target = 'sim' else: - if args.kgdb: + if kwargs['kgdb']: port = common.extra_serial_port else: port = common.gdb_port target = 'remote localhost:{}'.format(port) cmd.extend([ - '-ex', 'file {}'.format(image), common.Newline, - '-ex', 'target {}'.format(target), common.Newline, + '-ex', 'file {}'.format(image), LF, + '-ex', 'target {}'.format(target), LF, ]) - if not args.kgdb: + if not kwargs['kgdb']: cmd.extend(break_at) if not no_continue: # ## lx-symbols @@ -173,16 +173,16 @@ def main(args, extra_args=None): # # The lx-symbols commands gets loaded through the file vmlinux-gdb.py # which gets put on the kernel build root when python debugging scripts are enabled. - cmd.extend(['-ex', 'continue', common.Newline]) - if not args.no_lxsymbols and linux_full_system: - cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), common.Newline]) + cmd.extend(['-ex', 'continue', LF]) + if not kwargs['no_lxsymbols'] and linux_full_system: + cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), LF]) cmd.extend(after) - if args.test: + if kwargs['test']: GdbTestcase( common.source_path, test_script_path, cmd, - verbose=args.verbose, + verbose=kwargs['verbose'], ) else: # I would rather have cwd be out_rootfs_overlay_dir, diff --git a/run-gdb-user b/run-gdb-user index 6756701..1ad643a 100755 --- a/run-gdb-user +++ b/run-gdb-user @@ -24,7 +24,7 @@ parser.add_argument( nargs='?' ) args = common.setup(parser) -executable = common.resolve_userland(args.executable) +executable = common.resolve_userland(kwargs['executable']) addr = common.get_elf_entry(os.path.join(common.buildroot_build_build_dir, executable)) extra_args = {} extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(addr)) @@ -33,5 +33,5 @@ extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(ad # TODO understand better. # Also, lx-symbols overrides the add-symbol-file commands. extra_args['no_lxsymbols'] = True -extra_args['break_at'] = args.break_at +extra_args['break_at'] = kwargs['break_at'] sys.exit(rungdb.main(args, extra_args)) diff --git a/run-gdbserver b/run-gdbserver index e34656f..623e7fe 100755 --- a/run-gdbserver +++ b/run-gdbserver @@ -22,7 +22,7 @@ sys.exit(subprocess.Popen([ '-q', '-ex', 'set sysroot {}'.format(common.buildroot_staging_dir), '-ex', 'target remote localhost:{}'.format(common.qemu_hostfwd_generic_port), - '-ex', 'tbreak {}'.format(args.break_at), + '-ex', 'tbreak {}'.format(kwargs['break_at']), '-ex', 'continue', - os.path.join(common.buildroot_build_build_dir, common.resolve_userland(args.executable)), + os.path.join(common.buildroot_build_build_dir, common.resolve_userland(kwargs['executable'])), ]).wait()) diff --git a/run-toolchain b/run-toolchain index 1ac8724..07096ac 100755 --- a/run-toolchain +++ b/run-toolchain @@ -39,12 +39,12 @@ if common.baremetal is None: image = common.vmlinux else: image = common.image -tool= common.get_toolchain_tool(args.tool) -if args.dry: +tool= common.get_toolchain_tool(kwargs['tool']) +if kwargs['dry']: print(tool) else: sys.exit(self.sh.run_cmd( - [tool, common.Newline] - + common.add_newlines(args.extra_args), + [tool, LF] + + common.add_newlines(kwargs['extra_args']), cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'), )) diff --git a/shell_helpers.py b/shell_helpers.py index da08627..2496d58 100644 --- a/shell_helpers.py +++ b/shell_helpers.py @@ -95,7 +95,7 @@ class ShellHelpers: Optionally save the command to cmd_file file, and add extra_env environment variables to the command generated. - If cmd contains at least one LF, newlines are only added on common.Newline. + If cmd contains at least one LF, newlines are only added on LF. Otherwise, newlines are added automatically after every word. ''' if type(cmd) is str: diff --git a/trace-boot b/trace-boot index 2daac4b..66e44c5 100755 --- a/trace-boot +++ b/trace-boot @@ -21,7 +21,7 @@ parser.add_argument( ) args = common.setup(parser) extra_args = { - 'extra_emulator_args': args.extra_emulator_args, + 'extra_emulator_args': kwargs['extra_emulator_args'], } if common.emulator == 'gem5': extra_args.update({