From 5e20ba833be1e9ef39960b8d58091877892b324a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sun, 9 Dec 2018 00:00:00 +0000 Subject: [PATCH] build-linux and build-gem5 seem to work --- README.adoc | 4 +- arm | 3 - bisect-linux-boot-gem5 | 6 +- build | 42 +-- build-baremetal | 52 ++-- build-buildroot | 52 ++-- build-crosstool-ng | 30 +- build-docker | 14 +- build-gem5 | 68 ++-- build-linux | 66 ++-- build-m5 | 12 +- build-modules | 36 +-- build-qemu | 10 +- build-userland | 22 +- cli_function.py | 18 +- common.py | 684 +++++++++++++++++++++-------------------- copy-overlay | 4 +- gem5-shell | 4 +- getvar | 4 +- qemu-monitor | 2 +- qemu-trace2txt | 10 +- release | 12 +- release-upload | 6 +- release-zip | 14 +- run | 144 ++++----- run-docker | 8 +- run-gdb | 26 +- run-gdb-user | 4 +- run-gdbserver | 6 +- run-toolchain | 10 +- shell_helpers.py | 6 +- trace-boot | 10 +- trace2line | 20 +- 33 files changed, 702 insertions(+), 707 deletions(-) delete mode 100755 arm diff --git a/README.adoc b/README.adoc index cff5dce..9fc0f8d 100644 --- a/README.adoc +++ b/README.adoc @@ -10301,7 +10301,7 @@ The most important things that we setup in the bootloaders are: The C functions that become available as a result are: * Newlib functions implemented at link:baremetal/lib/syscalls.c[] -* non-Newlib functions implemented at link:common.c[] +* non-Newlib functions implemented at link:kwargs['c'][] It is not possible to call those C functions from the examples that don't use a bootloader. @@ -10333,7 +10333,7 @@ svc 0x00123456 and we can see from the docs that `0x18` stands for the `SYS_EXIT` command. -This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/common.c[]. +This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/kwargs['c'][]. Other magic operations we can do with semihosting besides exiting the on the host include: diff --git a/arm b/arm deleted file mode 100755 index c54105c..0000000 --- a/arm +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -build-crosstool-ng \ -; \ No newline at end of file diff --git a/bisect-linux-boot-gem5 b/bisect-linux-boot-gem5 index 91aa346..8ae557f 100755 --- a/bisect-linux-boot-gem5 +++ b/bisect-linux-boot-gem5 @@ -6,8 +6,8 @@ import shutil import sys import common -build_linux = imp.load_source('build-linux', os.path.join(common.root_dir, 'build_linux')) -run = imp.load_source('run', os.path.join(common.root_dir, 'run')) +build_linux = imp.load_source('build-linux', os.path.join(kwargs['root_dir'], 'build_linux')) +run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run')) parser = common.get_argparse( argparse_args={ @@ -24,7 +24,7 @@ args = common.setup(parser) # We need a clean rebuild because rebuilds at different revisions: # - may fail # - may not actually rebuild all files, e.g. on header changes -common.rmrf(common.linux_build_dir) +common.rmrf(kwargs['linux_build_dir']) build_linux.LinuxComponent().do_build(args) status = run.main(args, { 'eval': 'm5 exit', diff --git a/build b/build index a0c810c..d7ad995 100755 --- a/build +++ b/build @@ -49,7 +49,7 @@ class Component: def run_cmd(cmd, arch): global args cmd_abs = cmd.copy() - cmd_abs[0] = os.path.join(common.root_dir, cmd[0]) + cmd_abs[0] = os.path.join(kwargs['root_dir'], cmd[0]) cmd_abs.extend(['--arch', arch]) if kwargs['extra_args']: cmd_abs.append(kwargs['extra_args']) @@ -86,15 +86,15 @@ name_to_component_map = { # Leaves without dependencies. 'baremetal-qemu': Component( lambda arch: run_cmd(['build-baremetal', '--qemu'], arch), - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'baremetal-gem5': Component( lambda arch: run_cmd(['build-baremetal', '--gem5'], arch), - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'baremetal-gem5-pbx': Component( lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch), - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'buildroot': buildroot_component, 'buildroot-gcc': buildroot_component, @@ -103,7 +103,7 @@ name_to_component_map = { ), 'crosstool-ng': Component( lambda arch: run_cmd(['build-crosstool-ng'], arch), - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], # http://crosstool-ng.github.io/docs/os-setup/ apt_get_pkgs={ 'bison', @@ -197,7 +197,7 @@ name_to_component_map = { 'gem5-baremetal', 'baremetal-gem5-pbx', ], - supported_archs=common.crosstool_ng_supported_archs, + supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'all-linux': Component(dependencies=[ 'qemu-gem5-buildroot', @@ -296,10 +296,10 @@ group = parser.add_mutually_exclusive_group(required=False) group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\ Build the selected components for all archs. ''') -group.add_argument('-a', '--arch', choices=common.arch_choices, default=[], action='append', help='''\ +group.add_argument('-a', '--arch', choices=kwargs['arch_choices'], default=[], action='append', help='''\ Build the selected components for this arch. Select multiple archs by passing this option multiple times. Default: [{}] -'''.format(common.default_arch)) +'''.format(kwargs['default_arch'])) parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\ Also download all dependencies required for a given build: Ubuntu packages, Python packages and git submodules. @@ -314,7 +314,7 @@ Extra args to pass to all scripts. ) parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\ Which components to build. Default: qemu-buildroot -'''.format(common.default_arch)) +'''.format(kwargs['default_arch'])) common.add_dry_run_argument(parser) args = parser.parse_args() common.setup_dry_run_arguments(args) @@ -322,14 +322,14 @@ common.setup_dry_run_arguments(args) # Decide archs. if kwargs['arch'] == []: if kwargs['all'] or kwargs['all_archs']: - archs = common.all_archs.copy() + archs = kwargs['all_archs'].copy() else: - archs = set([common.default_arch]) + archs = set([kwargs['default_arch']]) else: archs = set() for arch in kwargs['arch']: - if arch in common.arch_short_to_long_dict: - arch = common.arch_short_to_long_dict[arch] + if arch in kwargs['arch_short_to_long_dict']: + arch = kwargs['arch_short_to_long_dict'][arch] archs.add(arch) # Decide components. @@ -393,7 +393,7 @@ if kwargs['download_dependencies']: 'libsdl2-dev', } apt_get_pkgs.difference_update(interacive_pkgs) - if common.in_docker: + if kwargs['in_docker']: sudo = [] # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai os.environ['DEBIAN_FRONTEND'] = 'noninteractive' @@ -406,7 +406,7 @@ if kwargs['download_dependencies']: f.write(sources_txt) else: sudo = ['sudo'] - if common.in_docker or kwargs['travis']: + if kwargs['in_docker'] or kwargs['travis']: y = ['-y'] else: y = [] @@ -416,25 +416,25 @@ if kwargs['download_dependencies']: if apt_get_pkgs: self.sh.run_cmd( sudo + ['apt-get', 'install'] + y + [LF] + - common.add_newlines(sorted(apt_get_pkgs)) + self.sh.add_newlines(sorted(apt_get_pkgs)) ) if apt_build_deps: self.sh.run_cmd( sudo + ['apt-get', 'build-dep'] + y + [LF] + - common.add_newlines(sorted(apt_build_deps)) + self.sh.add_newlines(sorted(apt_build_deps)) ) if python2_pkgs: self.sh.run_cmd( ['python', '-m', 'pip', 'install', '--user', LF] + - common.add_newlines(sorted(python2_pkgs)) + self.sh.add_newlines(sorted(python2_pkgs)) ) if python3_pkgs: # Not with pip executable directly: # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 self.sh.run_cmd( ['python3', '-m', 'pip', 'install', '--user', LF] + - common.add_newlines(sorted(python3_pkgs)) + self.sh.add_newlines(sorted(python3_pkgs)) ) git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive'] if submodules: @@ -450,7 +450,7 @@ if kwargs['download_dependencies']: # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 self.sh.run_cmd( git_cmd_common + ['--', LF] + - common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules)]) + self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules)]) ) if submodules_shallow: # == Shallow cloning. @@ -474,7 +474,7 @@ if kwargs['download_dependencies']: # self.sh.run_cmd( git_cmd_common + ['--depth', '1', '--', LF] + - common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules_shallow)]) + self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules_shallow)]) ) # Do the build. diff --git a/build-baremetal b/build-baremetal index 7383118..372aad4 100755 --- a/build-baremetal +++ b/build-baremetal @@ -8,43 +8,43 @@ class BaremetalComponent(common.Component): def do_build(self, args): common.assert_crosstool_ng_supports_arch(kwargs['arch']) build_dir = self.get_build_dir(args) - bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext)) + bootloader_obj = os.path.join(kwargs['baremetal_build_lib_dir'], 'bootloader{}'.format(kwargs['obj_ext'])) common_basename_noext = 'common' - common_src = os.path.join(common.root_dir, common_basename_noext + common.c_ext) - common_obj = os.path.join(common.baremetal_build_lib_dir, common_basename_noext + common.obj_ext) + common_src = os.path.join(kwargs['root_dir'], common_basename_noext + kwargs['c_ext']) + common_obj = os.path.join(kwargs['baremetal_build_lib_dir'], common_basename_noext + kwargs['obj_ext']) syscalls_basename_noext = 'syscalls' - syscalls_src = os.path.join(common.baremetal_src_lib_dir, syscalls_basename_noext + common.c_ext) - syscalls_obj = os.path.join(common.baremetal_build_lib_dir, syscalls_basename_noext + common.obj_ext) + syscalls_src = os.path.join(kwargs['baremetal_src_lib_dir'], syscalls_basename_noext + kwargs['c_ext']) + syscalls_obj = os.path.join(kwargs['baremetal_build_lib_dir'], syscalls_basename_noext + kwargs['obj_ext']) common_objs = [common_obj, syscalls_obj] cflags = [ - '-I', common.baremetal_src_lib_dir, LF, - '-I', common.root_dir, LF, + '-I', kwargs['baremetal_src_lib_dir'], LF, + '-I', kwargs['root_dir'], LF, '-O0', LF, '-ggdb3', LF, - '-mcpu={}'.format(common.mcpu), LF, + '-mcpu={}'.format(kwargs['mcpu']), LF, '-nostartfiles', LF, ] if kwargs['prebuilt']: gcc = 'arm-none-eabi-gcc' else: - os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH'] + os.environ['PATH'] = kwargs['crosstool_ng_bin_dir'] + os.environ['PATH'] gcc = common.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng']) - if common.emulator == 'gem5': - if common.machine == 'VExpress_GEM5_V1': + if kwargs['emulator'] == 'gem5': + if kwargs['machine'] == 'VExpress_GEM5_V1': entry_address = 0x80000000 uart_address = 0x1c090000 - elif common.machine == 'RealViewPBX': + elif kwargs['machine'] == 'RealViewPBX': entry_address = 0x10000 uart_address = 0x10009000 else: - raise Exception('unknown machine: ' + common.machine) + raise Exception('unknown machine: ' + kwargs['machine']) cflags.extend(['-D', 'GEM5'.format(uart_address), LF]) else: entry_address = 0x40000000 uart_address = 0x09000000 os.makedirs(build_dir, exist_ok=True) - os.makedirs(common.baremetal_build_lib_dir, exist_ok=True) - src = os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(kwargs['arch'], common.asm_ext)) + os.makedirs(kwargs['baremetal_build_lib_dir'], exist_ok=True) + src = os.path.join(kwargs['baremetal_src_lib_dir'], '{}{}'.format(kwargs['arch'], kwargs['asm_ext'])) if common.need_rebuild([src], bootloader_obj): self.sh.run_cmd( [gcc, LF] + @@ -87,7 +87,7 @@ class BaremetalComponent(common.Component): common_objs=common_objs, ) arch_dir = os.path.join('arch', kwargs['arch']) - if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)): + if os.path.isdir(os.path.join(kwargs['baremetal_src_dir'], arch_dir)): self._build_dir( arch_dir, gcc=gcc, @@ -97,7 +97,7 @@ class BaremetalComponent(common.Component): common_objs=common_objs, ) arch_dir = os.path.join('arch', kwargs['arch'], 'no_bootloader') - if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)): + if os.path.isdir(os.path.join(kwargs['baremetal_src_dir'], arch_dir)): self._build_dir( arch_dir, gcc=gcc, @@ -116,7 +116,7 @@ Build the baremetal examples with crosstool-NG. } def get_build_dir(self, args): - return common.baremetal_build_dir + return kwargs['baremetal_build_dir'] def get_default_args(self): return {'baremetal': 'all'} @@ -137,18 +137,18 @@ Build the baremetal examples with crosstool-NG. Place outputs on the same subpath or the output directory. """ - in_dir = os.path.join(common.baremetal_src_dir, subpath) - out_dir = os.path.join(common.baremetal_build_dir, subpath) + in_dir = os.path.join(kwargs['baremetal_src_dir'], subpath) + out_dir = os.path.join(kwargs['baremetal_build_dir'], subpath) os.makedirs(out_dir, exist_ok=True) common_objs = common_objs.copy() if bootloader: common_objs.append(bootloader_obj) for in_basename in os.listdir(in_dir): in_path = os.path.join(in_dir, in_basename) - if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext): + if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (kwargs['c_ext'], kwargs['asm_ext']): in_name = os.path.splitext(in_basename)[0] - main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext)) - src = os.path.join(common.baremetal_src_dir, in_path) + main_obj = os.path.join(kwargs['baremetal_build_dir'], subpath, '{}{}'.format(in_name, kwargs['obj_ext'])) + src = os.path.join(kwargs['baremetal_src_dir'], in_path) if common.need_rebuild([src], main_obj): self.sh.run_cmd( [gcc, LF] + @@ -160,8 +160,8 @@ Build the baremetal examples with crosstool-NG. ] ) objs = common_objs + [main_obj] - out = os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext) - link_script = os.path.join(common.baremetal_src_dir, 'link.ld') + out = os.path.join(kwargs['baremetal_build_dir'], subpath, in_name + kwargs['baremetal_build_ext']) + link_script = os.path.join(kwargs['baremetal_src_dir'], 'link.ld') if common.need_rebuild(objs + [link_script], out): self.sh.run_cmd( [gcc, LF] + @@ -171,7 +171,7 @@ Build the baremetal examples with crosstool-NG. '-o', out, LF, '-T', link_script, LF, ] + - common.add_newlines(objs) + self.sh.add_newlines(objs) ) if __name__ == '__main__': diff --git a/build-buildroot b/build-buildroot index 9b955fb..88a6f54 100755 --- a/build-buildroot +++ b/build-buildroot @@ -67,11 +67,11 @@ usually extra Buildroot targets. def do_build(self, args): build_dir = self.get_build_dir(args) - os.makedirs(common.out_dir, exist_ok=True) - extra_make_args = common.add_newlines(kwargs['extra_make_args']) + os.makedirs(kwargs['out_dir'], exist_ok=True) + extra_make_args = self.sh.add_newlines(kwargs['extra_make_args']) if kwargs['build_linux']: extra_make_args.extend(['linux-reconfigure', LF]) - if common.emulator == 'gem5': + if kwargs['emulator'] == 'gem5': extra_make_args.extend(['gem5-reconfigure', LF]) if kwargs['arch'] == 'x86_64': defconfig = 'qemu_x86_64_defconfig' @@ -80,24 +80,24 @@ usually extra Buildroot targets. elif kwargs['arch'] == 'aarch64': defconfig = 'qemu_aarch64_virt_defconfig' br2_external_dirs = [] - for package_dir in os.listdir(common.packages_dir): - package_dir_abs = os.path.join(common.packages_dir, package_dir) + for package_dir in os.listdir(kwargs['packages_dir']): + package_dir_abs = os.path.join(kwargs['packages_dir'], package_dir) if os.path.isdir(package_dir_abs): br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs)) br2_external_str = ':'.join(br2_external_dirs) self.sh.run_cmd( [ 'make', LF, - 'O={}'.format(common.buildroot_build_dir), LF, + 'O={}'.format(kwargs['buildroot_build_dir']), LF, 'BR2_EXTERNAL={}'.format(br2_external_str), LF, defconfig, LF, ], - cwd=common.buildroot_src_dir, + cwd=kwargs['buildroot_src_dir'], ) configs = kwargs['config'] configs.extend([ 'BR2_JLEVEL={}'.format(kwargs['nproc']), - 'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir), + 'BR2_DL_DIR="{}"'.format(kwargs['buildroot_download_dir']), ]) if not kwargs['build_linux']: configs.extend([ @@ -107,38 +107,38 @@ usually extra Buildroot targets. if not kwargs['baseline']: configs.extend([ 'BR2_GLOBAL_PATCH_DIR="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global')) + self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'patches', 'global')) ), 'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment')) + self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'busybox_config_fragment')) ), 'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override')) + self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'buildroot_override')) ), 'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script')) + self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'rootfs-post-build-script')) ), 'BR2_ROOTFS_USERS_TABLES="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table')) + self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'user_table')) ), ]) if not kwargs['no_overlay']: configs.append('BR2_ROOTFS_OVERLAY="{}"'.format( - self._path_relative_to_buildroot(common.out_rootfs_overlay_dir) + self._path_relative_to_buildroot(kwargs['out_rootfs_overlay_dir']) )) config_fragments = [ - os.path.join(common.root_dir, 'buildroot_config', 'default') + os.path.join(kwargs['root_dir'], 'buildroot_config', 'default') ] + kwargs['config_fragment'] # TODO Can't get rid of these for now with nice fragments on Buildroot: # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config - self.sh.write_configs(common.buildroot_config_file, configs, config_fragments) + self.sh.write_configs(kwargs['buildroot_config_file'], configs, config_fragments) self.sh.run_cmd( [ 'make', LF, - 'O={}'.format(common.buildroot_build_dir), LF, + 'O={}'.format(kwargs['buildroot_build_dir']), LF, 'olddefconfig', LF, ], - cwd=common.buildroot_src_dir, + cwd=kwargs['buildroot_src_dir'], ) common.make_build_dirs() if not kwargs['no_all']: @@ -146,21 +146,21 @@ usually extra Buildroot targets. self.sh.run_cmd( [ 'make', LF, - 'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_source_dir), LF, - 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), LF, - 'O={}'.format(common.buildroot_build_dir), LF, + 'LKMC_GEM5_SRCDIR="{}"'.format(kwargs['gem5_source_dir']), LF, + 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(kwargs['parsec_benchmark_src_dir']), LF, + 'O={}'.format(kwargs['buildroot_build_dir']), LF, 'V={}'.format(int(kwargs['verbose'])), LF, ] + extra_make_args , - out_file=os.path.join(common.buildroot_build_dir, 'lkmc.log'), + out_file=os.path.join(kwargs['buildroot_build_dir'], 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], - cwd=common.buildroot_src_dir, + cwd=kwargs['buildroot_src_dir'], ) # Create the qcow2 from ext2. # Skip if qemu is not present, because gem5 does not need the qcow2. # so we don't force a QEMU build for gem5. - if not kwargs['no_all'] and os.path.exists(common.qemu_img_executable): + if not kwargs['no_all'] and os.path.exists(kwargs['qemu_img_executable']): common.raw_to_qcow2() def get_argparse_args(self): @@ -171,7 +171,7 @@ Run Linux on an emulator } def get_build_dir(self, args): - return common.buildroot_build_dir + return kwargs['buildroot_build_dir'] _defaults = { 'baseline': False, @@ -184,7 +184,7 @@ Run Linux on an emulator } def _path_relative_to_buildroot(self, abspath): - return os.path.relpath(abspath, common.buildroot_src_dir) + return os.path.relpath(abspath, kwargs['buildroot_src_dir']) if __name__ == '__main__': BuildrootComponent().build() diff --git a/build-crosstool-ng b/build-crosstool-ng index b497d55..98c8c4d 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -8,20 +8,20 @@ class CrosstoolNgComponent(common.Component): def do_build(self, args): common.assert_crosstool_ng_supports_arch(kwargs['arch']) build_dir = self.get_build_dir(args) - defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig') - os.makedirs(common.crosstool_ng_util_dir, exist_ok=True) - os.makedirs(common.crosstool_ng_download_dir, exist_ok=True) + defconfig_dest = os.path.join(kwargs['crosstool_ng_util_dir'], 'defconfig') + os.makedirs(kwargs['crosstool_ng_util_dir'], exist_ok=True) + os.makedirs(kwargs['crosstool_ng_download_dir'], exist_ok=True) # Bootstrap out-ot-tree WONTFIX. I've tried. # https://github.com/crosstool-ng/crosstool-ng/issues/1021 - os.chdir(common.crosstool_ng_src_dir) + os.chdir(kwargs['crosstool_ng_src_dir']) self.sh.run_cmd( - [os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), LF], + [os.path.join(kwargs['crosstool_ng_src_dir'], 'bootstrap'), LF], ) - os.chdir(common.crosstool_ng_util_dir) + os.chdir(kwargs['crosstool_ng_util_dir']) self.sh.run_cmd( [ - os.path.join(common.crosstool_ng_src_dir, 'configure'), LF, + os.path.join(kwargs['crosstool_ng_src_dir'], 'configure'), LF, '--enable-local', LF, ], ) @@ -34,33 +34,33 @@ class CrosstoolNgComponent(common.Component): # Build the toolchain. self.sh.cp( - os.path.join(common.root_dir, 'crosstool_ng_config', kwargs['arch']), + os.path.join(kwargs['root_dir'], 'crosstool_ng_config', kwargs['arch']), defconfig_dest ) common.write_configs( - common.crosstool_ng_defconfig, + kwargs['crosstool_ng_defconfig'], [ - 'CT_PREFIX_DIR="{}"'.format(common.crosstool_ng_install_dir), + 'CT_PREFIX_DIR="{}"'.format(kwargs['crosstool_ng_install_dir']), 'CT_WORK_DIR="{}"'.format(build_dir), - 'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir), + 'CT_LOCAL_TARBALLS_DIR="{}"'.format(kwargs['crosstool_ng_download_dir']), ] ) self.sh.run_cmd( [ - common.crosstool_ng_executable, LF, + kwargs['crosstool_ng_executable'], LF, 'defconfig', LF, ], ) os.unlink(defconfig_dest) self.sh.run_cmd( [ - common.crosstool_ng_executable, LF, + kwargs['crosstool_ng_executable'], LF, 'build', LF, 'CT_JOBS={}'.format(str(kwargs['nproc'])), LF, ], out_file=os.path.join(build_dir, 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], - extra_paths=[common.ccache_dir], + extra_paths=[kwargs['ccache_dir']], ) def get_argparse_args(self): @@ -71,7 +71,7 @@ Build crosstool-NG with Newlib for bare metal compilation' } def get_build_dir(self, args): - return common.crosstool_ng_build_dir + return kwargs['crosstool_ng_build_dir'] if __name__ == '__main__': CrosstoolNgComponent().build() diff --git a/build-docker b/build-docker index 7fab055..06ac41e 100755 --- a/build-docker +++ b/build-docker @@ -44,7 +44,7 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- '--privileged', '-t', '-w', target_dir, - '-v', '{}:{}'.format(common.root_dir, target_dir), + '-v', '{}:{}'.format(kwargs['root_dir'], target_dir), 'ubuntu:18.04', 'bash', ]) @@ -52,11 +52,11 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- 'docker', 'export', '-o', - common.docker_tar_file, + kwargs['docker_tar_file'], container_name, ]) - tar = tarfile.open(common.docker_tar_file) - tar.extractall(common.docker_tar_dir) + tar = tarfile.open(kwargs['docker_tar_file']) + tar.extractall(kwargs['docker_tar_dir']) tar.close() # sudo not required in theory # https://askubuntu.com/questions/1046828/how-to-run-libguestfs-tools-tools-such-as-virt-make-fs-without-sudo @@ -65,13 +65,13 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest- '--format', 'raw', '--size', '+1G', '--type', 'ext2', - common.docker_tar_dir, - common.docker_rootfs_raw_file, + kwargs['docker_tar_dir'], + kwargs['docker_rootfs_raw_file'], ]) common.raw_to_qcow2(prebuilt=True) def get_build_dir(self, args): - return common.docker_build_dir + return kwargs['docker_build_dir'] def get_default_args(self): return {'docker': True} diff --git a/build-gem5 b/build-gem5 index 195a203..7147ac1 100755 --- a/build-gem5 +++ b/build-gem5 @@ -16,49 +16,46 @@ class Main(common.BuildCliFunction): nargs='*', ) - def build(self, **kwargs): - build_dir = self.get_build_dir(**kwargs) - binaries_dir = os.path.join(kwargs['gem5_system_dir'], 'binaries') - disks_dir = os.path.join(kwargs['gem5_system_dir'], 'disks') + def build(self): + build_dir = self.get_build_dir() + binaries_dir = os.path.join(self.env['gem5_system_dir'], 'binaries') + disks_dir = os.path.join(self.env['gem5_system_dir'], 'disks') os.makedirs(binaries_dir, exist_ok=True) os.makedirs(disks_dir, exist_ok=True) - if kwargs['gem5_source_dir'] is None: - if not os.path.exists(os.path.join(kwargs['gem5_source_dir'], '.git')): - if kwargs['gem5_source_dir'] == kwargs['gem5_default_src_dir']: + if self.env['gem5_source_dir'] is None: + if not os.path.exists(os.path.join(self.env['gem5_source_dir'], '.git')): + if self.env['gem5_source_dir'] == self.env['gem5_default_src_dir']: raise Exception('gem5 submodule not checked out') self.sh.run_cmd([ 'git', LF, - '-C', kwargs['gem5_default_src_dir'], LF, + '-C', self.env['gem5_default_src_dir'], LF, 'worktree', 'add', LF, - '-b', os.path.join('wt', kwargs['gem5_build_id']), LF, - kwargs['gem5_source_dir'], LF, + '-b', os.path.join('wt', self.env['gem5_build_id']), LF, + self.env['gem5_source_dir'], LF, ]) - if kwargs['verbose']: + if self.env['verbose']: verbose = ['--verbose', LF] else: verbose = [] - if kwargs['arch'] == 'x86_64': + if self.env['arch'] == 'x86_64': dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img') with open(dummy_img_path, 'wb') as dummy_img_file: zeroes = b'\x00' * (2 ** 16) for i in range(2 ** 10): dummy_img_file.write(zeroes) - self.sh.run_cmd(['mkswap', dummy_img_path]) + self.sh.run_cmd(['mkswap', dummy_img_path, LF]) with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'): # This file must always be present, despite --kernel overriding that default and selecting the kernel. # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present. pass - elif kwargs['arch'] == 'arm' or kwargs['arch'] == 'aarch64': - gem5_system_src_dir = os.path.join(kwargs['gem5_source_dir'], 'system') + elif self.env['arch'] == 'arm' or self.env['arch'] == 'aarch64': + gem5_system_src_dir = os.path.join(self.env['gem5_source_dir'], 'system') # dtb dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') - dt_build_dir = os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt') - self.sh.run_cmd([ - 'make', LF, - '-C', dt_src_dir, LF, - ]) - common.copy_dir_if_update_non_recursive( + dt_build_dir = os.path.join(self.env['gem5_system_dir'], 'arm', 'dt') + self.sh.run_cmd(['make', '-C', dt_src_dir, LF]) + self.sh.copy_dir_if_update_non_recursive( srcdir=dt_src_dir, destdir=dt_build_dir, filter_ext='.dtb', @@ -78,38 +75,35 @@ class Main(common.BuildCliFunction): # Bootloader 64. bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') # TODO cross_compile is ignored because the make does not use CC... - self.sh.run_cmd([ - 'make', LF, - '-C', bootloader64_dir, LF - ]) + self.sh.run_cmd(['make', '-C', bootloader64_dir, LF]) self.sh.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) self.sh.run_cmd( ( [ 'scons', LF, - '-j', str(kwargs['nproc']), LF, + '-j', str(self.env['nproc']), LF, '--gold-linker', LF, '--ignore-style', LF, - kwargs['gem5_executable'], LF, + self.env['gem5_executable'], LF, ] + verbose + - self.sh.add_newlines(kwargs['extra_scons_args']) + self.sh.add_newlines(self.env['extra_scons_args']) ), - cwd=kwargs['gem5_source_dir'], - extra_paths=[kwargs['ccache_dir']], + cwd=self.env['gem5_source_dir'], + extra_paths=[self.env['ccache_dir']], ) - term_src_dir = os.path.join(kwargs['gem5_source_dir'], 'util/term') + term_src_dir = os.path.join(self.env['gem5_source_dir'], 'util/term') m5term_build = os.path.join(term_src_dir, 'm5term') - self.sh.run_cmd(['make', '-C', term_src_dir]) - if os.path.exists(kwargs['gem5_m5term']): + self.sh.run_cmd(['make', '-C', term_src_dir, LF]) + if os.path.exists(self.env['gem5_m5term']): # Otherwise self.sh.cp would fail with "Text file busy" if you # tried to rebuild while running m5term: # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512 - os.unlink(kwargs['gem5_m5term']) - self.sh.cp(m5term_build, kwargs['gem5_m5term']) + os.unlink(self.env['gem5_m5term']) + self.sh.cp(m5term_build, self.env['gem5_m5term']) - def get_build_dir(self, **kwargs): - return kwargs['gem5_build_dir'] + def get_build_dir(self): + return self.env['gem5_build_dir'] if __name__ == '__main__': Main().cli() diff --git a/build-linux b/build-linux index a3902ff..43d951f 100755 --- a/build-linux +++ b/build-linux @@ -6,8 +6,13 @@ import shutil import common from shell_helpers import LF -class LinuxComponent(common.Component): - def __init__(self, parser): +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ +Build the Linux kernel. +''' + ) self.add_argument( '--config', default=[], action='append', help='''\ @@ -44,48 +49,48 @@ Configure the kernel, but don't build it. nargs='*' ) - def build(self, **kwargs): - build_dir = self.get_build_dir(**kwargs) - if kwargs['initrd'] or kwargs['initramfs']: + def build(self): + build_dir = self.get_build_dir() + if self.env['initrd'] or self.env['initramfs']: raise Exception('just trolling, --initrd and --initramfs are broken for now') os.makedirs(build_dir, exist_ok=True) tool = 'gcc' - gcc = common.get_toolchain_tool(tool) + gcc = self.get_toolchain_tool(tool) prefix = gcc[:-len(tool)] common_args = { - 'cwd': common.linux_src_dir, + 'cwd': self.env['linux_src_dir'], } ccache = shutil.which('ccache') if ccache is not None: cc = '{} {}'.format(ccache, gcc) else: cc = gcc - if kwargs['verbose']: + if self.env['verbose']: verbose = ['V=1'] else: verbose = [] common_make_args = [ 'make', LF, - '-j', str(kwargs['nproc']), LF, - 'ARCH={}'.format(common.linux_arch), LF, + '-j', str(self.env['nproc']), LF, + 'ARCH={}'.format(self.env['linux_arch']), LF, 'CROSS_COMPILE={}'.format(prefix), LF, 'CC={}'.format(cc), LF, 'O={}'.format(build_dir), LF, ] + verbose - if kwargs['custom_config_file'] is not None: - if not os.path.exists(kwargs['custom_config_file']): - raise Exception('config fragment file does not exist: {}'.format(kwargs['custom_config_file'])) - base_config_file = kwargs['custom_config_file'] + if self.env['custom_config_file'] is not None: + if not os.path.exists(self.env['custom_config_file']): + raise Exception('config fragment file does not exist: {}'.format(self.env['custom_config_file'])) + base_config_file = self.env['custom_config_file'] config_fragments = [] else: - base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(kwargs['arch'])) + base_config_file = os.path.join(self.env['linux_config_dir'], 'buildroot-{}'.format(self.env['arch'])) config_fragments = ['min', 'default'] for i, config_fragment in enumerate(config_fragments): - config_fragments[i] = os.path.join(common.linux_config_dir, config_fragment) - config_fragments.extend(kwargs['config_fragment']) - if kwargs['config'] != []: + config_fragments[i] = os.path.join(self.env['linux_config_dir'], config_fragment) + config_fragments.extend(self.env['config_fragment']) + if self.env['config'] != []: cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment') - cli_config_str = '\n'.join(kwargs['config']) + cli_config_str = '\n'.join(self.env['config']) common.write_string_to_file(cli_config_fragment_path, cli_config_str) config_fragments.append(cli_config_fragment_path) self.sh.cp( @@ -94,12 +99,12 @@ Configure the kernel, but don't build it. ) self.sh.run_cmd( [ - os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), LF, + os.path.join(self.env['linux_src_dir'], 'scripts', 'kconfig', 'merge_config.sh'), LF, '-m', LF, '-O', build_dir, LF, os.path.join(build_dir, '.config'), LF, ] + - common.add_newlines(config_fragments) + self.sh.add_newlines(config_fragments) ) self.sh.run_cmd( ( @@ -108,11 +113,11 @@ Configure the kernel, but don't build it. ), **common_args ) - if not kwargs['config_only']: + if not self.env['config_only']: self.sh.run_cmd( ( common_make_args + - common.add_newlines(kwargs['extra_make_args']) + self.sh.add_newlines(self.env['extra_make_args']) ), **common_args ) @@ -120,7 +125,7 @@ Configure the kernel, but don't build it. ( common_make_args + [ - 'INSTALL_MOD_PATH={}'.format(common.out_rootfs_overlay_dir), LF, + 'INSTALL_MOD_PATH={}'.format(self.env['out_rootfs_overlay_dir']), LF, 'modules_install', LF, ] ), @@ -130,15 +135,8 @@ Configure the kernel, but don't build it. # TODO Basically all kernel modules also basically leak full host paths. Just terrible. Buildroot deals with that stuff nicely for us. # common.rmrf() - def get_argparse_args(self): - return { - 'description': '''\ -Build the Linux kernel. -''' - } - - def get_build_dir(self, args): - return common.linux_build_dir + def get_build_dir(self): + return self.env['linux_build_dir'] if __name__ == '__main__': - LinuxComponent().build() + Main().cli() diff --git a/build-m5 b/build-m5 index 86983eb..0ae8bbe 100755 --- a/build-m5 +++ b/build-m5 @@ -19,25 +19,25 @@ class M5Component(common.Component): '-f', 'Makefile.{}'.format(arch), LF, 'CC={}'.format(cc), LF, 'LD={}'.format(ld), LF, - 'PWD={}'.format(common.gem5_m5_source_dir), LF, + 'PWD={}'.format(kwargs['gem5_m5_source_dir']), LF, ] def do_build(self, args): - os.makedirs(common.gem5_m5_build_dir, exist_ok=True) + os.makedirs(kwargs['gem5_m5_build_dir'], exist_ok=True) # We must clean first or else the build outputs of one arch can conflict with the other. # I should stop being lazy and go actually patch gem5 to support out of tree m5 build... self.clean(args) self.sh.run_cmd( self.get_make_cmd(args), - cwd=common.gem5_m5_source_dir, + cwd=kwargs['gem5_m5_source_dir'], ) - os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True) - self.sh.cp(os.path.join(common.gem5_m5_source_dir, 'm5'), common.out_rootfs_overlay_bin_dir) + os.makedirs(kwargs['out_rootfs_overlay_bin_dir'], exist_ok=True) + self.sh.cp(os.path.join(kwargs['gem5_m5_source_dir'], 'm5'), kwargs['out_rootfs_overlay_bin_dir']) def clean(self, args): self.sh.run_cmd( self.get_make_cmd(args) + ['clean', LF], - cwd=common.gem5_m5_source_dir, + cwd=kwargs['gem5_m5_source_dir'], ) if __name__ == '__main__': diff --git a/build-modules b/build-modules index 0bb2f5d..3c96d9b 100755 --- a/build-modules +++ b/build-modules @@ -42,34 +42,34 @@ Use the host packaged cross toolchain. # This copies only modified files as per: # https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory distutils.dir_util.copy_tree( - common.kernel_modules_src_dir, - os.path.join(build_dir, common.kernel_modules_subdir), + kwargs['kernel_modules_src_dir'], + os.path.join(build_dir, kwargs['kernel_modules_subdir']), update=1, ) distutils.dir_util.copy_tree( - common.include_src_dir, - os.path.join(build_dir, common.include_subdir), + kwargs['include_src_dir'], + os.path.join(build_dir, kwargs['include_subdir']), update=1, ) all_kernel_modules = [] - for basename in os.listdir(common.kernel_modules_src_dir): - src = os.path.join(common.kernel_modules_src_dir, basename) + for basename in os.listdir(kwargs['kernel_modules_src_dir']): + src = os.path.join(kwargs['kernel_modules_src_dir'], basename) if os.path.isfile(src): noext, ext = os.path.splitext(basename) - if ext == common.c_ext: + if ext == kwargs['c_ext']: all_kernel_modules.append(noext) if kwargs['kernel_modules'] == []: kernel_modules = all_kernel_modules else: kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], kwargs['kernel_modules']) - object_files = map(lambda x: x + common.obj_ext, kernel_modules) + object_files = map(lambda x: x + kwargs['obj_ext'], kernel_modules) tool = 'gcc' if kwargs['host']: allowed_toolchains = ['host'] - build_subdir = common.kernel_modules_build_host_subdir + build_subdir = kwargs['kernel_modules_build_host_subdir'] else: allowed_toolchains = None - build_subdir = common.kernel_modules_build_subdir + build_subdir = kwargs['kernel_modules_build_subdir'] gcc = common.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains) prefix = gcc[:-len(tool)] ccache = shutil.which('ccache') @@ -84,13 +84,13 @@ Use the host packaged cross toolchain. if kwargs['host']: linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build') else: - linux_dir = common.linux_build_dir + linux_dir = kwargs['linux_build_dir'] self.sh.run_cmd( ( [ 'make', LF, '-j', str(kwargs['nproc']), LF, - 'ARCH={}'.format(common.linux_arch), LF, + 'ARCH={}'.format(kwargs['linux_arch']), LF, 'CC={}'.format(cc), LF, 'CROSS_COMPILE={}'.format(prefix), LF, 'LINUX_DIR={}'.format(linux_dir), LF, @@ -100,13 +100,13 @@ Use the host packaged cross toolchain. common.shlex_split(kwargs['make_args']) + verbose ), - cwd=os.path.join(common.kernel_modules_build_subdir), + cwd=os.path.join(kwargs['kernel_modules_build_subdir']), ) if not kwargs['host']: common.copy_dir_if_update_non_recursive( - srcdir=common.kernel_modules_build_subdir, - destdir=common.out_rootfs_overlay_dir, - filter_ext=common.kernel_module_ext, + srcdir=kwargs['kernel_modules_build_subdir'], + destdir=kwargs['out_rootfs_overlay_dir'], + filter_ext=kwargs['kernel_module_ext'], ) def get_argparse_args(self): @@ -120,9 +120,9 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host def get_build_dir(self, args): if kwargs['host']: - return os.path.join(common.kernel_modules_build_host_dir) + return os.path.join(kwargs['kernel_modules_build_host_dir']) else: - return os.path.join(common.kernel_modules_build_dir) + return os.path.join(kwargs['kernel_modules_build_dir']) if __name__ == '__main__': ModulesComponent().build() diff --git a/build-qemu b/build-qemu index b633d9a..55d06d3 100755 --- a/build-qemu +++ b/build-qemu @@ -32,15 +32,15 @@ class QemuComponent(common.Component): target_list = '{}-softmmu'.format(kwargs['arch']) self.sh.run_cmd( [ - os.path.join(common.qemu_src_dir, 'configure'), LF, + os.path.join(kwargs['qemu_src_dir'], 'configure'), LF, '--enable-debug', LF, '--enable-trace-backends=simple', LF, '--target-list={}'.format(target_list), LF, '--enable-sdl', LF, '--with-sdlabi=2.0', LF, ] + - common.add_newlines(kwargs['extra_config_args']), - extra_paths=[common.ccache_dir], + self.sh.add_newlines(kwargs['extra_config_args']), + extra_paths=[kwargs['ccache_dir']], cwd=build_dir ) self.sh.run_cmd( @@ -53,11 +53,11 @@ class QemuComponent(common.Component): verbose ), cwd=build_dir, - extra_paths=[common.ccache_dir], + extra_paths=[kwargs['ccache_dir']], ) def get_build_dir(self, args): - return common.qemu_build_dir + return kwargs['qemu_build_dir'] if __name__ == '__main__': QemuComponent().build() diff --git a/build-userland b/build-userland index 3e7274c..d225b09 100755 --- a/build-userland +++ b/build-userland @@ -59,25 +59,25 @@ has the OpenBLAS libraries and headers installed. 'make', LF, '-j', str(kwargs['nproc']), LF, 'ARCH={}'.format(kwargs['arch']), LF, - 'CCFLAGS_SCRIPT={} {}'.format('-I', common.userland_src_dir), LF, - 'COMMON_DIR={}'.format(common.root_dir), LF, + 'CCFLAGS_SCRIPT={} {}'.format('-I', kwargs['userland_src_dir']), LF, + 'COMMON_DIR={}'.format(kwargs['root_dir']), LF, 'CC={}'.format(cc), LF, 'CXX={}'.format(cxx), LF, - 'PKG_CONFIG={}'.format(common.buildroot_pkg_config), LF, - 'STAGING_DIR={}'.format(common.buildroot_staging_dir), LF, + 'PKG_CONFIG={}'.format(kwargs['buildroot_pkg_config']), LF, + 'STAGING_DIR={}'.format(kwargs['buildroot_staging_dir']), LF, 'OUT_DIR={}'.format(build_dir), LF, ] + - common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in kwargs['has_package']]) + + self.sh.add_newlines(['HAS_{}=y'.format(package.upper()) for package in kwargs['has_package']]) + shlex.split(kwargs['make_args']) + - common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in kwargs['targets']]) + self.sh.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + kwargs['userland_build_ext'] for target in kwargs['targets']]) ), - cwd=common.userland_src_dir, - extra_paths=[common.ccache_dir], + cwd=kwargs['userland_src_dir'], + extra_paths=[kwargs['ccache_dir']], ) common.copy_dir_if_update_non_recursive( srcdir=build_dir, - destdir=common.out_rootfs_overlay_dir, - filter_ext=common.userland_build_ext, + destdir=kwargs['out_rootfs_overlay_dir'], + filter_ext=kwargs['userland_build_ext'], ) def get_argparse_args(self): @@ -86,7 +86,7 @@ has the OpenBLAS libraries and headers installed. } def get_build_dir(self, args): - return common.userland_build_dir + return kwargs['userland_build_dir'] if __name__ == '__main__': UserlandComponent().build() diff --git a/cli_function.py b/cli_function.py index 216bdce..98df08e 100644 --- a/cli_function.py +++ b/cli_function.py @@ -119,11 +119,11 @@ class CliFunction: raise Exception('Value not given for mandatory argument: ' + key) return self.main(**args_with_defaults) - def __init__(self, config_file=None): + def __init__(self, config_file=None, description=None): self._all_keys = set() self._arguments = [] - self.description = None self._config_file = config_file + self._description = description if self._config_file is not None: self.add_argument( '--config-file', @@ -146,7 +146,7 @@ class CliFunction: to get all arguments. ''' parser = argparse.ArgumentParser( - description=self.description, + description=self._description, formatter_class=argparse.RawTextHelpFormatter, ) for argument in self._arguments: @@ -177,7 +177,13 @@ class CliFunction: if __name__ == '__main__': class OneCliFunction(CliFunction): def __init__(self): - super().__init__(config_file='cli_function_test_config.py') + super().__init__( + config_file='cli_function_test_config.py', + description = '''\ +Description of this +amazing function! +''', + ) self.add_argument('-a', '--asdf', default='A', help='Help for asdf'), self.add_argument('-q', '--qwer', default='Q', help='Help for qwer'), self.add_argument('-b', '--bool', default=True, help='Help for bool'), @@ -186,10 +192,6 @@ if __name__ == '__main__': self.add_argument('pos-mandatory', help='Help for pos-mandatory', type=int), self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int), self.add_argument('args-star', help='Help for args-star', nargs='*'), - self.description = '''\ -Description of this -amazing function! -''' def main(self, **kwargs): del kwargs['config_file'] return kwargs diff --git a/common.py b/common.py index b703398..25106fa 100644 --- a/common.py +++ b/common.py @@ -5,7 +5,6 @@ import base64 import collections import copy import datetime -import distutils.file_util import glob import imp import json @@ -99,8 +98,9 @@ class LkmcCliFunction(cli_function.CliFunction): * command timing * some common flags, e.g.: --arch, --dry-run, --verbose ''' - def __init__(self): - super().__init__(config_file=common.consts['config_file']) + def __init__(self, *args, **kwargs): + kwargs['config_file'] = consts['config_file'] + super().__init__(*args, **kwargs) # Args for all scripts. self.add_argument( @@ -262,24 +262,327 @@ to allow overriding configs from the CLI. ''' ) + def _init_env(self, env): + ''' + Update the kwargs from the command line with derived arguments. + ''' + def join(*paths): + return os.path.join(*paths) + if env['qemu'] or not env['gem5']: + env['emulator'] = 'qemu' + else: + env['emulator'] = 'gem5' + if env['arch'] in env['arch_short_to_long_dict']: + env['arch'] = env['arch_short_to_long_dict'][env['arch']] + if env['userland_build_id'] is None: + env['userland_build_id'] = env['default_build_id'] + env['userland_build_id_given'] = False + else: + env['userland_build_id_given'] = True + if env['gem5_worktree'] is not None and env['gem5_build_id'] is None: + env['gem5_build_id'] = env['gem5_worktree'] + env['is_arm'] = False + if env['arch'] == 'arm': + env['armv'] = 7 + env['gem5_arch'] = 'ARM' + env['mcpu'] = 'cortex-a15' + env['buildroot_toolchain_prefix'] = 'arm-buildroot-linux-uclibcgnueabihf' + env['crosstool_ng_toolchain_prefix'] = 'arm-unknown-eabi' + env['ubuntu_toolchain_prefix'] = 'arm-linux-gnueabihf' + if env['emulator'] == 'gem5': + if env['machine'] is None: + env['machine'] = 'VExpress_GEM5_V1' + else: + if env['machine'] is None: + env['machine'] = 'virt' + env['is_arm'] = True + elif env['arch'] == 'aarch64': + env['armv'] = 8 + env['gem5_arch'] = 'ARM' + env['mcpu'] = 'cortex-a57' + env['buildroot_toolchain_prefix'] = 'aarch64-buildroot-linux-uclibc' + env['crosstool_ng_toolchain_prefix'] = 'aarch64-unknown-elf' + env['ubuntu_toolchain_prefix'] = 'aarch64-linux-gnu' + if env['emulator'] == 'gem5': + if env['machine'] is None: + env['machine'] = 'VExpress_GEM5_V1' + else: + if env['machine'] is None: + env['machine'] = 'virt' + env['is_arm'] = True + elif env['arch'] == 'x86_64': + env['crosstool_ng_toolchain_prefix'] = 'x86_64-unknown-elf' + env['gem5_arch'] = 'X86' + env['buildroot_toolchain_prefix'] = 'x86_64-buildroot-linux-uclibc' + env['ubuntu_toolchain_prefix'] = 'x86_64-linux-gnu' + if env['emulator'] == 'gem5': + if env['machine'] is None: + env['machine'] = 'TODO' + else: + if env['machine'] is None: + env['machine'] = 'pc' + + # Buildroot + env['buildroot_build_dir'] = join(env['buildroot_out_dir'], 'build', env['buildroot_build_id'], env['arch']) + env['buildroot_download_dir'] = join(env['buildroot_out_dir'], 'download') + env['buildroot_config_file'] = join(env['buildroot_build_dir'], '.config') + env['buildroot_build_build_dir'] = join(env['buildroot_build_dir'], 'build') + env['buildroot_linux_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom') + env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], "vmlinux") + env['host_dir'] = join(env['buildroot_build_dir'], 'host') + env['host_bin_dir'] = join(env['host_dir'], 'usr', 'bin') + env['buildroot_pkg_config'] = join(env['host_bin_dir'], 'pkg-config') + env['buildroot_images_dir'] = join(env['buildroot_build_dir'], 'images') + env['buildroot_rootfs_raw_file'] = join(env['buildroot_images_dir'], 'rootfs.ext2') + env['buildroot_qcow2_file'] = env['buildroot_rootfs_raw_file'] + '.qcow2' + env['staging_dir'] = join(env['out_dir'], 'staging', env['arch']) + env['buildroot_staging_dir'] = join(env['buildroot_build_dir'], 'staging') + env['target_dir'] = join(env['buildroot_build_dir'], 'target') + env['linux_buildroot_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom') + + # QEMU + env['qemu_build_dir'] = join(env['out_dir'], 'qemu', env['qemu_build_id']) + env['qemu_executable_basename'] = 'qemu-system-{}'.format(env['arch']) + env['qemu_executable'] = join(env['qemu_build_dir'], '{}-softmmu'.format(env['arch']), env['qemu_executable_basename']) + env['qemu_img_basename'] = 'qemu-img' + env['qemu_img_executable'] = join(env['qemu_build_dir'], env['qemu_img_basename']) + + # gem5 + if env['gem5_build_dir'] is None: + env['gem5_build_dir'] = join(env['gem5_out_dir'], env['gem5_build_id'], env['gem5_build_type']) + env['gem5_fake_iso'] = join(env['gem5_out_dir'], 'fake.iso') + env['gem5_m5term'] = join(env['gem5_build_dir'], 'm5term') + env['gem5_build_build_dir'] = join(env['gem5_build_dir'], 'build') + env['gem5_executable'] = join(env['gem5_build_build_dir'], env['gem5_arch'], 'gem5.{}'.format(env['gem5_build_type'])) + env['gem5_system_dir'] = join(env['gem5_build_dir'], 'system') + + # gem5 source + if env['gem5_source_dir'] is not None: + assert os.path.exists(env['gem5_source_dir']) + else: + if env['gem5_worktree'] is not None: + env['gem5_source_dir'] = join(env['gem5_non_default_src_root_dir'], env['gem5_worktree']) + else: + env['gem5_source_dir'] = env['gem5_default_src_dir'] + env['gem5_m5_source_dir'] = join(env['gem5_source_dir'], 'util', 'm5') + env['gem5_config_dir'] = join(env['gem5_source_dir'], 'configs') + env['gem5_se_file'] = join(env['gem5_config_dir'], 'example', 'se.py') + env['gem5_fs_file'] = join(env['gem5_config_dir'], 'example', 'fs.py') + + # crosstool-ng + env['crosstool_ng_buildid_dir'] = join(env['crosstool_ng_out_dir'], 'build', env['crosstool_ng_build_id']) + env['crosstool_ng_install_dir'] = join(env['crosstool_ng_buildid_dir'], 'install', env['arch']) + env['crosstool_ng_bin_dir'] = join(env['crosstool_ng_install_dir'], 'bin') + env['crosstool_ng_util_dir'] = join(env['crosstool_ng_buildid_dir'], 'util') + env['crosstool_ng_config'] = join(env['crosstool_ng_util_dir'], '.config') + env['crosstool_ng_defconfig'] = join(env['crosstool_ng_util_dir'], 'defconfig') + env['crosstool_ng_executable'] = join(env['crosstool_ng_util_dir'], 'ct-ng') + env['crosstool_ng_build_dir'] = join(env['crosstool_ng_buildid_dir'], 'build') + env['crosstool_ng_download_dir'] = join(env['crosstool_ng_out_dir'], 'download') + + # run + env['gem5_run_dir'] = join(env['run_dir_base'], 'gem5', env['arch'], str(env['run_id'])) + env['m5out_dir'] = join(env['gem5_run_dir'], 'm5out') + env['stats_file'] = join(env['m5out_dir'], 'stats.txt') + env['gem5_trace_txt_file'] = join(env['m5out_dir'], 'trace.txt') + env['gem5_guest_terminal_file'] = join(env['m5out_dir'], 'system.terminal') + env['gem5_readfile'] = join(env['gem5_run_dir'], 'readfile') + env['gem5_termout_file'] = join(env['gem5_run_dir'], 'termout.txt') + env['qemu_run_dir'] = join(env['run_dir_base'], 'qemu', env['arch'], str(env['run_id'])) + env['qemu_termout_file'] = join(env['qemu_run_dir'], 'termout.txt') + env['qemu_trace_basename'] = 'trace.bin' + env['qemu_trace_file'] = join(env['qemu_run_dir'], 'trace.bin') + env['qemu_trace_txt_file'] = join(env['qemu_run_dir'], 'trace.txt') + env['qemu_rrfile'] = join(env['qemu_run_dir'], 'rrfile') + env['gem5_out_dir'] = join(env['out_dir'], 'gem5') + + # Ports + if env['port_offset'] is None: + try: + env['port_offset'] = int(env['run_id']) + except ValueError: + env['port_offset'] = 0 + if env['emulator'] == 'gem5': + env['gem5_telnet_port'] = 3456 + env['port_offset'] + env['gdb_port'] = 7000 + env['port_offset'] + else: + env['qemu_base_port'] = 45454 + 10 * env['port_offset'] + env['qemu_monitor_port'] = env['qemu_base_port'] + 0 + env['qemu_hostfwd_generic_port'] = env['qemu_base_port'] + 1 + env['qemu_hostfwd_ssh_port'] = env['qemu_base_port'] + 2 + env['qemu_gdb_port'] = env['qemu_base_port'] + 3 + env['extra_serial_port'] = env['qemu_base_port'] + 4 + env['gdb_port'] = env['qemu_gdb_port'] + env['qemu_background_serial_file'] = join(env['qemu_run_dir'], 'background.log') + + # gem5 QEMU polymorphism. + if env['emulator'] == 'gem5': + env['executable'] = env['gem5_executable'] + env['run_dir'] = env['gem5_run_dir'] + env['termout_file'] = env['gem5_termout_file'] + env['guest_terminal_file'] = env['gem5_guest_terminal_file'] + env['trace_txt_file'] = env['gem5_trace_txt_file'] + else: + env['executable'] = env['qemu_executable'] + env['run_dir'] = env['qemu_run_dir'] + env['termout_file'] = env['qemu_termout_file'] + env['trace_txt_file'] = env['qemu_trace_txt_file'] + env['run_cmd_file'] = join(env['run_dir'], 'run.sh') + + # Linux kernl. + if 'linux_build_id' in env: + env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch']) + env['lkmc_vmlinux'] = join(env['linux_build_dir'], "vmlinux") + if env['arch'] == 'arm': + env['linux_arch'] = 'arm' + env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage') + elif env['arch'] == 'aarch64': + env['linux_arch'] = 'arm64' + env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'Image') + elif env['arch'] == 'x86_64': + env['linux_arch'] = 'x86' + env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'bzImage') + env['lkmc_linux_image'] = join(env['linux_build_dir'], env['linux_image_prefix']) + env['buildroot_linux_image'] = join(env['buildroot_linux_build_dir'], env['linux_image_prefix']) + if env['buildroot_linux']: + env['vmlinux'] = env['buildroot_vmlinux'] + env['linux_image'] = env['buildroot_linux_image'] + else: + env['vmlinux'] = env['lkmc_vmlinux'] + env['linux_image'] = env['lkmc_linux_image'] + + # Kernel modules. + env['kernel_modules_build_dir'] = join(env['kernel_modules_build_base_dir'], env['arch']) + env['kernel_modules_build_subdir'] = join(env['kernel_modules_build_dir'], env['kernel_modules_subdir']) + env['kernel_modules_build_host_dir'] = join(env['kernel_modules_build_base_dir'], 'host') + env['kernel_modules_build_host_subdir'] = join(env['kernel_modules_build_host_dir'], env['kernel_modules_subdir']) + env['userland_build_dir'] = join(env['out_dir'], 'userland', env['userland_build_id'], env['arch']) + env['out_rootfs_overlay_dir'] = join(env['out_dir'], 'rootfs_overlay', env['arch']) + env['out_rootfs_overlay_bin_dir'] = join(env['out_rootfs_overlay_dir'], 'bin') + + # Baremetal. + env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal') + env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename']) + if env['emulator'] == 'gem5': + env['simulator_name'] = 'gem5' + else: + env['simulator_name'] = 'qemu' + env['baremetal_build_dir'] = join(env['out_dir'], 'baremetal', env['arch'], env['simulator_name'], env['machine']) + env['baremetal_build_lib_dir'] = join(env['baremetal_build_dir'], env['baremetal_lib_basename']) + env['baremetal_build_ext'] = '.elf' + + # Docker + env['docker_build_dir'] = join(env['out_dir'], 'docker', env['arch']) + env['docker_tar_dir'] = join(env['docker_build_dir'], 'export') + env['docker_tar_file'] = join(env['docker_build_dir'], 'export.tar') + env['docker_rootfs_raw_file'] = join(env['docker_build_dir'], 'export.ext2') + env['docker_qcow2_file'] = join(env['docker_rootfs_raw_file'] + '.qcow2') + if env['docker']: + env['rootfs_raw_file'] = env['docker_rootfs_raw_file'] + env['qcow2_file'] = env['docker_qcow2_file'] + else: + env['rootfs_raw_file'] = env['buildroot_rootfs_raw_file'] + env['qcow2_file'] = env['buildroot_qcow2_file'] + + # Image. + if env['baremetal'] is None: + if env['emulator'] == 'gem5': + env['image'] = env['vmlinux'] + env['disk_image'] = env['rootfs_raw_file'] + else: + env['image'] = env['linux_image'] + env['disk_image'] = env['qcow2_file'] + else: + env['disk_image'] = env['gem5_fake_iso'] + if env['baremetal'] == 'all': + path = env['baremetal'] + else: + path = resolve_executable( + env['baremetal'], + env['baremetal_src_dir'], + env['baremetal_build_dir'], + env['baremetal_build_ext'], + ) + source_path_noext = os.path.splitext(join( + env['baremetal_src_dir'], + os.path.relpath(path, env['baremetal_build_dir']) + ))[0] + for ext in [c_ext, asm_ext]: + source_path = source_path_noext + ext + if os.path.exists(source_path): + env['source_path'] = source_path + break + env['image'] = path + self.env = env + + def get_elf_entry(self, elf_file_path): + readelf_header = subprocess.check_output([ + self.get_toolchain_tool('readelf'), + '-h', + elf_file_path + ]) + for line in readelf_header.decode().split('\n'): + split = line.split() + if line.startswith(' Entry point address:'): + addr = line.split()[-1] + break + return int(addr, 0) + + def get_toolchain_prefix(self, tool, allowed_toolchains=None): + buildroot_full_prefix = os.path.join(self.env['host_bin_dir'], self.env['buildroot_toolchain_prefix']) + buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool)) + crosstool_ng_full_prefix = os.path.join(self.env['crosstool_ng_bin_dir'], self.env['crosstool_ng_toolchain_prefix']) + crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool)) + host_tool = '{}-{}'.format(self.env['ubuntu_toolchain_prefix'], tool) + host_path = shutil.which(host_tool) + if host_path is not None: + host_exists = True + host_full_prefix = host_path[:-(len(tool)+1)] + else: + host_exists = False + host_full_prefix = None + known_toolchains = { + 'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix), + 'buildroot': (buildroot_exists, buildroot_full_prefix), + 'host': (host_exists, host_full_prefix), + } + if allowed_toolchains is None: + if self.env['baremetal'] is None: + allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] + else: + allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] + tried = [] + for toolchain in allowed_toolchains: + exists, prefix = known_toolchains[toolchain] + tried.append('{}-{}'.format(prefix, tool)) + if exists: + return prefix + raise Exception('Tool not found. Tried:\n' + '\n'.join(tried)) + + def get_toolchain_tool(self, tool, allowed_toolchains=None): + return '{}-{}'.format(self.get_toolchain_prefix(tool, allowed_toolchains), tool) + def main(self, **kwargs): ''' Time the main of the derived class. ''' if not kwargs['dry_run']: start_time = time.time() - kwargs.update(common.consts) - kwargs = set_kwargs(kwargs) - self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run']) - self.timed_main(**kwargs) + kwargs.update(consts) + self._init_env(kwargs) + self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run']) + self.timed_main() if not kwargs['dry_run']: end_time = time.time() - common.print_time(end_time - start_time) + print_time(end_time - start_time) def run_cmd(self, *args, **kwargs): self.sh.run_cmd(*args, **kwargs) - def timed_main(self, **kwargs): + def timed_main(self): + ''' + Main action of the derived class. + ''' raise NotImplementedError() class BuildCliFunction(LkmcCliFunction): @@ -289,8 +592,8 @@ class BuildCliFunction(LkmcCliFunction): * `--clean` to clean the build directory * `--nproc` to set he number of build threads ''' - def __init__(self): - super().__init__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.add_argument( '--clean', default=False, @@ -304,33 +607,33 @@ class BuildCliFunction(LkmcCliFunction): help='Number of processors to use for the build.', ) - def clean(self, **kwargs): - build_dir = self.get_build_dir(kwargs) + def clean(self): + build_dir = self.get_build_dir() if build_dir is not None: - common.rmrf(build_dir) + self.sh.rmrf(build_dir) - def build(self, **kwargs): + def build(self): ''' Do the actual main build work. ''' raise NotImplementedError() - def get_build_dir(self, **kwargs): + def get_build_dir(self): return None - def timed_main(self, **kwargs): + def timed_main(self): ''' Parse CLI, and to the build based on it. The actual build work is done by do_build in implementing classes. ''' - if kwargs['clean']: - self.clean(kwargs) + if self.env['clean']: + self.clean() else: - self.build(**kwargs) + self.build() def assert_crosstool_ng_supports_arch(arch): - if arch not in common.crosstool_ng_supported_archs: + if arch not in kwargs['crosstool_ng_supported_archs']: raise Exception('arch not yet supported: ' + arch) def base64_encode(string): @@ -340,29 +643,16 @@ def gem_list_checkpoint_dirs(): ''' List checkpoint directory, oldest first. ''' - prefix_re = re.compile(common.gem5_cpt_prefix) - files = list(filter(lambda x: os.path.isdir(os.path.join(common.m5out_dir, x)) and prefix_re.search(x), os.listdir(common.m5out_dir))) - files.sort(key=lambda x: os.path.getmtime(os.path.join(common.m5out_dir, x))) + prefix_re = re.compile(kwargs['gem5_cpt_prefix']) + files = list(filter(lambda x: os.path.isdir(os.path.join(kwargs['m5out_dir'], x)) and prefix_re.search(x), os.listdir(kwargs['m5out_dir']))) + files.sort(key=lambda x: os.path.getmtime(os.path.join(kwargs['m5out_dir'], x))) return files -def get_elf_entry(elf_file_path): - readelf_header = subprocess.check_output([ - common.get_toolchain_tool('readelf'), - '-h', - elf_file_path - ]) - for line in readelf_header.decode().split('\n'): - split = line.split() - if line.startswith(' Entry point address:'): - addr = line.split()[-1] - break - return int(addr, 0) - def get_stats(stat_re=None, stats_file=None): if stat_re is None: stat_re = '^system.cpu[0-9]*.numCycles$' if stats_file is None: - stats_file = common.stats_file + stats_file = kwargs['stats_file'] stat_re = re.compile(stat_re) ret = [] with open(stats_file, 'r') as statfile: @@ -373,40 +663,6 @@ def get_stats(stat_re=None, stats_file=None): ret.append(cols[1]) return ret -def get_toolchain_prefix(tool, allowed_toolchains=None): - buildroot_full_prefix = os.path.join(common.host_bin_dir, common.buildroot_toolchain_prefix) - buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool)) - crosstool_ng_full_prefix = os.path.join(common.crosstool_ng_bin_dir, common.crosstool_ng_toolchain_prefix) - crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool)) - host_tool = '{}-{}'.format(common.ubuntu_toolchain_prefix, tool) - host_path = shutil.which(host_tool) - if host_path is not None: - host_exists = True - host_full_prefix = host_path[:-(len(tool)+1)] - else: - host_exists = False - host_full_prefix = None - known_toolchains = { - 'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix), - 'buildroot': (buildroot_exists, buildroot_full_prefix), - 'host': (host_exists, host_full_prefix), - } - if allowed_toolchains is None: - if common['baremetal'] is None: - allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] - else: - allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] - tried = [] - for toolchain in allowed_toolchains: - exists, prefix = known_toolchains[toolchain] - tried.append('{}-{}'.format(prefix, tool)) - if exists: - return prefix - raise Exception('Tool not found. Tried:\n' + '\n'.join(tried)) - -def get_toolchain_tool(tool, allowed_toolchains=None): - return '{}-{}'.format(common.get_toolchain_prefix(tool, allowed_toolchains), tool) - def github_make_request( authenticate=False, data=None, @@ -441,18 +697,18 @@ def log_error(msg): print('error: {}'.format(msg), file=sys.stderr) def make_build_dirs(): - os.makedirs(common.buildroot_build_build_dir, exist_ok=True) - os.makedirs(common.gem5_build_dir, exist_ok=True) - os.makedirs(common.out_rootfs_overlay_dir, exist_ok=True) + os.makedirs(kwargs['buildroot_build_build_dir'], exist_ok=True) + os.makedirs(kwargs['gem5_build_dir'], exist_ok=True) + os.makedirs(kwargs['out_rootfs_overlay_dir'], exist_ok=True) def make_run_dirs(): ''' Make directories required for the run. The user could nuke those anytime between runs to try and clean things up. ''' - os.makedirs(common.gem5_run_dir, exist_ok=True) - os.makedirs(common.p9_dir, exist_ok=True) - os.makedirs(common.qemu_run_dir, exist_ok=True) + os.makedirs(kwargs['gem5_run_dir'], exist_ok=True) + os.makedirs(kwargs['p9_dir'], exist_ok=True) + os.makedirs(kwargs['qemu_run_dir'], exist_ok=True) def need_rebuild(srcs, dst): if not os.path.exists(dst): @@ -468,17 +724,17 @@ def print_time(ellapsed_seconds): print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) def raw_to_qcow2(prebuilt=False, reverse=False): - if prebuilt or not os.path.exists(common.qemu_img_executable): + if prebuilt or not os.path.exists(kwargs['qemu_img_executable']): disable_trace = [] - qemu_img_executable = common.qemu_img_basename + qemu_img_executable = kwargs['qemu_img_basename'] else: # Prevent qemu-img from generating trace files like QEMU. Disgusting. disable_trace = ['-T', 'pr_manager_run,file=/dev/null', LF,] - qemu_img_executable = common.qemu_img_executable + qemu_img_executable = kwargs['qemu_img_executable'] infmt = 'raw' outfmt = 'qcow2' - infile = common.rootfs_raw_file - outfile = common.qcow2_file + infile = kwargs['rootfs_raw_file'] + outfile = kwargs['qcow2_file'] if reverse: tmp = infmt infmt = outfmt @@ -525,263 +781,9 @@ def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext): raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths)) def resolve_userland(path): - return common.resolve_executable( + return resolve_executable( path, - common.userland_src_dir, - common.userland_build_dir, - common.userland_build_ext, + kwargs['userland_src_dir'], + kwargs['userland_build_dir'], + kwargs['userland_build_ext'], ) - -def set_kwargs(kwargs): - ''' - Update the kwargs from the command line with derived arguments. - ''' - def join(*paths): - return os.path.join(*paths) - kwargs = collections.defaultdict(lambda: None, **kwargs) - if kwargs['qemu'] or not kwargs['gem5']: - kwargs['emulator'] = 'qemu' - else: - kwargs['emulator'] = 'gem5' - if kwargs['arch'] in kwargs['arch_short_to_long_dict']: - kwargs['arch'] = kwargs['arch_short_to_long_dict'][kwargs['arch']] - if kwargs['userland_build_id'] is None: - kwargs['userland_build_id'] = kwargs['default_build_id'] - kwargs['userland_build_id_given'] = False - else: - kwargs['userland_build_id_given'] = True - if kwargs['gem5_worktree'] is not None and kwargs['gem5_build_id'] is None: - kwargs['gem5_build_id'] = kwargs['gem5_worktree'] - kwargs['is_arm'] = False - if kwargs['arch'] == 'arm': - kwargs['armv'] = 7 - kwargs['gem5_arch'] = 'ARM' - kwargs['mcpu'] = 'cortex-a15' - kwargs['buildroot_toolchain_prefix'] = 'arm-buildroot-linux-uclibcgnueabihf' - kwargs['crosstool_ng_toolchain_prefix'] = 'arm-unknown-eabi' - kwargs['ubuntu_toolchain_prefix'] = 'arm-linux-gnueabihf' - if kwargs['emulator'] == 'gem5': - if kwargs['machine'] is None: - kwargs['machine'] = 'VExpress_GEM5_V1' - else: - if kwargs['machine'] is None: - kwargs['machine'] = 'virt' - kwargs['is_arm'] = True - elif kwargs['arch'] == 'aarch64': - kwargs['armv'] = 8 - kwargs['gem5_arch'] = 'ARM' - kwargs['mcpu'] = 'cortex-a57' - kwargs['buildroot_toolchain_prefix'] = 'aarch64-buildroot-linux-uclibc' - kwargs['crosstool_ng_toolchain_prefix'] = 'aarch64-unknown-elf' - kwargs['ubuntu_toolchain_prefix'] = 'aarch64-linux-gnu' - if kwargs['emulator'] == 'gem5': - if kwargs['machine'] is None: - kwargs['machine'] = 'VExpress_GEM5_V1' - else: - if kwargs['machine'] is None: - kwargs['machine'] = 'virt' - kwargs['is_arm'] = True - elif kwargs['arch'] == 'x86_64': - kwargs['crosstool_ng_toolchain_prefix'] = 'x86_64-unknown-elf' - kwargs['gem5_arch'] = 'X86' - kwargs['buildroot_toolchain_prefix'] = 'x86_64-buildroot-linux-uclibc' - kwargs['ubuntu_toolchain_prefix'] = 'x86_64-linux-gnu' - if kwargs['emulator'] == 'gem5': - if kwargs['machine'] is None: - kwargs['machine'] = 'TODO' - else: - if kwargs['machine'] is None: - kwargs['machine'] = 'pc' - - # Buildroot - kwargs['buildroot_build_dir'] = join(kwargs['buildroot_out_dir'], 'build', kwargs['buildroot_build_id'], kwargs['arch']) - kwargs['buildroot_download_dir'] = join(kwargs['buildroot_out_dir'], 'download') - kwargs['buildroot_config_file'] = join(kwargs['buildroot_build_dir'], '.config') - kwargs['buildroot_build_build_dir'] = join(kwargs['buildroot_build_dir'], 'build') - kwargs['buildroot_linux_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom') - kwargs['buildroot_vmlinux'] = join(kwargs['buildroot_linux_build_dir'], "vmlinux") - kwargs['host_dir'] = join(kwargs['buildroot_build_dir'], 'host') - kwargs['host_bin_dir'] = join(kwargs['host_dir'], 'usr', 'bin') - kwargs['buildroot_pkg_config'] = join(kwargs['host_bin_dir'], 'pkg-config') - kwargs['buildroot_images_dir'] = join(kwargs['buildroot_build_dir'], 'images') - kwargs['buildroot_rootfs_raw_file'] = join(kwargs['buildroot_images_dir'], 'rootfs.ext2') - kwargs['buildroot_qcow2_file'] = kwargs['buildroot_rootfs_raw_file'] + '.qcow2' - kwargs['staging_dir'] = join(kwargs['out_dir'], 'staging', kwargs['arch']) - kwargs['buildroot_staging_dir'] = join(kwargs['buildroot_build_dir'], 'staging') - kwargs['target_dir'] = join(kwargs['buildroot_build_dir'], 'target') - kwargs['linux_buildroot_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom') - - # QEMU - kwargs['qemu_build_dir'] = join(kwargs['out_dir'], 'qemu', kwargs['qemu_build_id']) - kwargs['qemu_executable_basename'] = 'qemu-system-{}'.format(kwargs['arch']) - kwargs['qemu_executable'] = join(kwargs['qemu_build_dir'], '{}-softmmu'.format(kwargs['arch']), kwargs['qemu_executable_basename']) - kwargs['qemu_img_basename'] = 'qemu-img' - kwargs['qemu_img_executable'] = join(kwargs['qemu_build_dir'], kwargs['qemu_img_basename']) - - # gem5 - if kwargs['gem5_build_dir'] is None: - kwargs['gem5_build_dir'] = join(kwargs['gem5_out_dir'], kwargs['gem5_build_id'], kwargs['gem5_build_type']) - kwargs['gem5_fake_iso'] = join(kwargs['gem5_out_dir'], 'fake.iso') - kwargs['gem5_m5term'] = join(kwargs['gem5_build_dir'], 'm5term') - kwargs['gem5_build_build_dir'] = join(kwargs['gem5_build_dir'], 'build') - kwargs['gem5_executable'] = join(kwargs['gem5_build_build_dir'], kwargs['gem5_arch'], 'gem5.{}'.format(kwargs['gem5_build_type'])) - kwargs['gem5_system_dir'] = join(kwargs['gem5_build_dir'], 'system') - - # gem5 source - if kwargs['gem5_source_dir'] is not None: - assert os.path.exists(kwargs['gem5_source_dir']) - else: - if kwargs['gem5_worktree'] is not None: - kwargs['gem5_source_dir'] = join(kwargs['gem5_non_default_src_root_dir'], kwargs['gem5_worktree']) - else: - kwargs['gem5_source_dir'] = kwargs['gem5_default_src_dir'] - kwargs['gem5_m5_source_dir'] = join(kwargs['gem5_source_dir'], 'util', 'm5') - kwargs['gem5_config_dir'] = join(kwargs['gem5_source_dir'], 'configs') - kwargs['gem5_se_file'] = join(kwargs['gem5_config_dir'], 'example', 'se.py') - kwargs['gem5_fs_file'] = join(kwargs['gem5_config_dir'], 'example', 'fs.py') - - # crosstool-ng - kwargs['crosstool_ng_buildid_dir'] = join(kwargs['crosstool_ng_out_dir'], 'build', kwargs['crosstool_ng_build_id']) - kwargs['crosstool_ng_install_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'install', kwargs['arch']) - kwargs['crosstool_ng_bin_dir'] = join(kwargs['crosstool_ng_install_dir'], 'bin') - kwargs['crosstool_ng_util_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'util') - kwargs['crosstool_ng_config'] = join(kwargs['crosstool_ng_util_dir'], '.config') - kwargs['crosstool_ng_defconfig'] = join(kwargs['crosstool_ng_util_dir'], 'defconfig') - kwargs['crosstool_ng_executable'] = join(kwargs['crosstool_ng_util_dir'], 'ct-ng') - kwargs['crosstool_ng_build_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'build') - kwargs['crosstool_ng_download_dir'] = join(kwargs['crosstool_ng_out_dir'], 'download') - - # run - kwargs['gem5_run_dir'] = join(kwargs['run_dir_base'], 'gem5', kwargs['arch'], str(kwargs['run_id'])) - kwargs['m5out_dir'] = join(kwargs['gem5_run_dir'], 'm5out') - kwargs['stats_file'] = join(kwargs['m5out_dir'], 'stats.txt') - kwargs['gem5_trace_txt_file'] = join(kwargs['m5out_dir'], 'trace.txt') - kwargs['gem5_guest_terminal_file'] = join(kwargs['m5out_dir'], 'system.terminal') - kwargs['gem5_readfile'] = join(kwargs['gem5_run_dir'], 'readfile') - kwargs['gem5_termout_file'] = join(kwargs['gem5_run_dir'], 'termout.txt') - kwargs['qemu_run_dir'] = join(kwargs['run_dir_base'], 'qemu', kwargs['arch'], str(kwargs['run_id'])) - kwargs['qemu_termout_file'] = join(kwargs['qemu_run_dir'], 'termout.txt') - kwargs['qemu_trace_basename'] = 'trace.bin' - kwargs['qemu_trace_file'] = join(kwargs['qemu_run_dir'], 'trace.bin') - kwargs['qemu_trace_txt_file'] = join(kwargs['qemu_run_dir'], 'trace.txt') - kwargs['qemu_rrfile'] = join(kwargs['qemu_run_dir'], 'rrfile') - kwargs['gem5_out_dir'] = join(kwargs['out_dir'], 'gem5') - - # Ports - if kwargs['port_offset'] is None: - try: - kwargs['port_offset'] = int(kwargs['run_id']) - except ValueError: - kwargs['port_offset'] = 0 - if kwargs['emulator'] == 'gem5': - kwargs['gem5_telnet_port'] = 3456 + kwargs['port_offset'] - kwargs['gdb_port'] = 7000 + kwargs['port_offset'] - else: - kwargs['qemu_base_port'] = 45454 + 10 * kwargs['port_offset'] - kwargs['qemu_monitor_port'] = kwargs['qemu_base_port'] + 0 - kwargs['qemu_hostfwd_generic_port'] = kwargs['qemu_base_port'] + 1 - kwargs['qemu_hostfwd_ssh_port'] = kwargs['qemu_base_port'] + 2 - kwargs['qemu_gdb_port'] = kwargs['qemu_base_port'] + 3 - kwargs['extra_serial_port'] = kwargs['qemu_base_port'] + 4 - kwargs['gdb_port'] = kwargs['qemu_gdb_port'] - kwargs['qemu_background_serial_file'] = join(kwargs['qemu_run_dir'], 'background.log') - - # gem5 QEMU polymorphism. - if kwargs['emulator'] == 'gem5': - kwargs['executable'] = kwargs['gem5_executable'] - kwargs['run_dir'] = kwargs['gem5_run_dir'] - kwargs['termout_file'] = kwargs['gem5_termout_file'] - kwargs['guest_terminal_file'] = kwargs['gem5_guest_terminal_file'] - kwargs['trace_txt_file'] = kwargs['gem5_trace_txt_file'] - else: - kwargs['executable'] = kwargs['qemu_executable'] - kwargs['run_dir'] = kwargs['qemu_run_dir'] - kwargs['termout_file'] = kwargs['qemu_termout_file'] - kwargs['trace_txt_file'] = kwargs['qemu_trace_txt_file'] - kwargs['run_cmd_file'] = join(kwargs['run_dir'], 'run.sh') - - # Linux kernl. - if 'linux_build_id' in kwargs: - kwargs['linux_build_dir'] = join(kwargs['out_dir'], 'linux', kwargs['linux_build_id'], kwargs['arch']) - kwargs['lkmc_vmlinux'] = join(kwargs['linux_build_dir'], "vmlinux") - if kwargs['arch'] == 'arm': - kwargs['linux_arch'] = 'arm' - kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'zImage') - elif kwargs['arch'] == 'aarch64': - kwargs['linux_arch'] = 'arm64' - kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'Image') - elif kwargs['arch'] == 'x86_64': - kwargs['linux_arch'] = 'x86' - kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'bzImage') - kwargs['lkmc_linux_image'] = join(kwargs['linux_build_dir'], kwargs['linux_image_prefix']) - kwargs['buildroot_linux_image'] = join(kwargs['buildroot_linux_build_dir'], kwargs['linux_image_prefix']) - if kwargs['buildroot_linux']: - kwargs['vmlinux'] = kwargs['buildroot_vmlinux'] - kwargs['linux_image'] = kwargs['buildroot_linux_image'] - else: - kwargs['vmlinux'] = kwargs['lkmc_vmlinux'] - kwargs['linux_image'] = kwargs['lkmc_linux_image'] - - # Kernel modules. - kwargs['kernel_modules_build_dir'] = join(kwargs['kernel_modules_build_base_dir'], kwargs['arch']) - kwargs['kernel_modules_build_subdir'] = join(kwargs['kernel_modules_build_dir'], kwargs['kernel_modules_subdir']) - kwargs['kernel_modules_build_host_dir'] = join(kwargs['kernel_modules_build_base_dir'], 'host') - kwargs['kernel_modules_build_host_subdir'] = join(kwargs['kernel_modules_build_host_dir'], kwargs['kernel_modules_subdir']) - kwargs['userland_build_dir'] = join(kwargs['out_dir'], 'userland', kwargs['userland_build_id'], kwargs['arch']) - kwargs['out_rootfs_overlay_dir'] = join(kwargs['out_dir'], 'rootfs_overlay', kwargs['arch']) - kwargs['out_rootfs_overlay_bin_dir'] = join(kwargs['out_rootfs_overlay_dir'], 'bin') - - # Baremetal. - kwargs['baremetal_src_dir'] = join(kwargs['root_dir'], 'baremetal') - kwargs['baremetal_src_lib_dir'] = join(kwargs['baremetal_src_dir'], kwargs['baremetal_lib_basename']) - if kwargs['emulator'] == 'gem5': - kwargs['simulator_name'] = 'gem5' - else: - kwargs['simulator_name'] = 'qemu' - kwargs['baremetal_build_dir'] = join(kwargs['out_dir'], 'baremetal', kwargs['arch'], kwargs['simulator_name'], kwargs['machine']) - kwargs['baremetal_build_lib_dir'] = join(kwargs['baremetal_build_dir'], kwargs['baremetal_lib_basename']) - kwargs['baremetal_build_ext'] = '.elf' - - # Docker - kwargs['docker_build_dir'] = join(kwargs['out_dir'], 'docker', kwargs['arch']) - kwargs['docker_tar_dir'] = join(kwargs['docker_build_dir'], 'export') - kwargs['docker_tar_file'] = join(kwargs['docker_build_dir'], 'export.tar') - kwargs['docker_rootfs_raw_file'] = join(kwargs['docker_build_dir'], 'export.ext2') - kwargs['docker_qcow2_file'] = join(kwargs['docker_rootfs_raw_file'] + '.qcow2') - if kwargs['docker']: - kwargs['rootfs_raw_file'] = kwargs['docker_rootfs_raw_file'] - kwargs['qcow2_file'] = kwargs['docker_qcow2_file'] - else: - kwargs['rootfs_raw_file'] = kwargs['buildroot_rootfs_raw_file'] - kwargs['qcow2_file'] = kwargs['buildroot_qcow2_file'] - - # Image. - if kwargs['baremetal'] is None: - if kwargs['emulator'] == 'gem5': - kwargs['image'] = kwargs['vmlinux'] - kwargs['disk_image'] = kwargs['rootfs_raw_file'] - else: - kwargs['image'] = kwargs['linux_image'] - kwargs['disk_image'] = kwargs['qcow2_file'] - else: - kwargs['disk_image'] = kwargs['gem5_fake_iso'] - if kwargs['baremetal'] == 'all': - path = kwargs['baremetal'] - else: - path = kwargs.resolve_executable( - kwargs['baremetal'], - kwargs['baremetal_src_dir'], - kwargs['baremetal_build_dir'], - kwargs['baremetal_build_ext'], - ) - source_path_noext = os.path.splitext(join( - kwargs['baremetal_src_dir'], - os.path.relpath(path, kwargs['baremetal_build_dir']) - ))[0] - for ext in [c_ext, asm_ext]: - source_path = source_path_noext + ext - if os.path.exists(source_path): - kwargs['source_path'] = source_path - break - kwargs['image'] = path - return dict(kwargs) diff --git a/copy-overlay b/copy-overlay index bbf3470..7a5af04 100755 --- a/copy-overlay +++ b/copy-overlay @@ -9,8 +9,8 @@ import common class CopyOverlayComponent(common.Component): def do_build(self, args): distutils.dir_util.copy_tree( - common.rootfs_overlay_dir, - common.out_rootfs_overlay_dir, + kwargs['rootfs_overlay_dir'], + kwargs['out_rootfs_overlay_dir'], update=1, ) diff --git a/gem5-shell b/gem5-shell index 7ba405a..debb4b4 100755 --- a/gem5-shell +++ b/gem5-shell @@ -10,7 +10,7 @@ parser = common.get_argparse( ) args = common.setup(parser) sys.exit(self.sh.run_cmd([ - common.gem5_m5term, LF, + kwargs['gem5_m5term'], LF, 'localhost', LF, - str(common.gem5_telnet_port), LF, + str(kwargs['gem5_telnet_port']), LF, ])) diff --git a/getvar b/getvar index e706967..c4b2bf6 100755 --- a/getvar +++ b/getvar @@ -5,12 +5,12 @@ import types import common parser = common.get_argparse(argparse_args={ - 'description': '''Print the value of a common.py variable. + 'description': '''Print the value of a kwargs['py'] variable. This is useful to: * give dry commands on the README that don't change when we refactor directory structure -* create simple bash scripts that call use common.py variables +* create simple bash scripts that call use kwargs['py'] variables For example, to get the Buildroot output directory for an ARM build, use: diff --git a/qemu-monitor b/qemu-monitor index 9c0ad1d..5c6ba29 100755 --- a/qemu-monitor +++ b/qemu-monitor @@ -27,7 +27,7 @@ def write_and_read(tn, cmd, prompt): tn.write(cmd.encode('utf-8')) return '\n'.join(tn.read_until(prompt).decode('utf-8').splitlines()[1:])[:-len(prompt)] -with telnetlib.Telnet('localhost', common.qemu_monitor_port) as tn: +with telnetlib.Telnet('localhost', kwargs['qemu_monitor_port']) as tn: # Couldn't disable server echo, so just removing the write for now. # https://stackoverflow.com/questions/12421799/how-to-disable-telnet-echo-in-python-telnetlib # sock = tn.get_socket() diff --git a/qemu-trace2txt b/qemu-trace2txt index 8de5815..2d40534 100755 --- a/qemu-trace2txt +++ b/qemu-trace2txt @@ -9,12 +9,12 @@ import common def main(): return self.sh.run_cmd( [ - os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), LF, - os.path.join(common.qemu_build_dir, 'trace-events-all'), LF, - os.path.join(common.qemu_trace_file), LF, + os.path.join(kwargs['qemu_src_dir'], 'scripts/simpletrace.py'), LF, + os.path.join(kwargs['qemu_build_dir'], 'trace-events-all'), LF, + os.path.join(kwargs['qemu_trace_file']), LF, ], - cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'), - out_file=common.qemu_trace_txt_file, + cmd_file=os.path.join(kwargs['run_dir'], 'qemu-trace2txt'), + out_file=kwargs['qemu_trace_txt_file'], show_stdout=False, ) diff --git a/release b/release index dc8cc54..34ac894 100755 --- a/release +++ b/release @@ -10,21 +10,21 @@ import subprocess import time import common -release_zip = imp.load_source('release_zip', os.path.join(common.root_dir, 'release-zip')) -release_upload = imp.load_source('release_upload', os.path.join(common.root_dir, 'release-upload')) +release_zip = imp.load_source('release_zip', os.path.join(kwargs['root_dir'], 'release-zip')) +release_upload = imp.load_source('release_upload', os.path.join(kwargs['root_dir'], 'release-upload')) start_time = time.time() # TODO factor those out so we don't redo the same thing multiple times. -# subprocess.check_call([os.path.join(common.root_dir, 'test')]) -# subprocess.check_call([os.path.join(common.root_dir, ''bench-all', '-A', '-u']) +# subprocess.check_call([os.path.join(kwargs['root_dir'], 'test')]) +# subprocess.check_call([os.path.join(kwargs['root_dir'], ''bench-all', '-A', '-u']) # A clean release requires a full rebuild unless we hack it :-( # We can't just use our current build as it contains packages we've # installed in random experiments. And with EXT2: we can't easily # know what the smallest root filesystem size is and use it either... # https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot -subprocess.check_call([os.path.join(common.root_dir, 'configure'), '--all']) -subprocess.check_call([os.path.join(common.root_dir, 'build'), '--all-archs', 'release']) +subprocess.check_call([os.path.join(kwargs['root_dir'], 'configure'), '--all']) +subprocess.check_call([os.path.join(kwargs['root_dir'], 'build'), '--all-archs', 'release']) release_zip.main() subprocess.check_call(['git', 'push']) release_upload.main() diff --git a/release-upload b/release-upload index 5dc6581..5416a05 100755 --- a/release-upload +++ b/release-upload @@ -18,9 +18,9 @@ import urllib.error import common def main(): - repo = common.github_repo_id - tag = 'sha-{}'.format(common.sha) - upload_path = common.release_zip_file + repo = kwargs['github_repo_id'] + tag = 'sha-{}'.format(kwargs['sha']) + upload_path = kwargs['release_zip_file'] # Check the release already exists. try: diff --git a/release-zip b/release-zip index 05b2390..c7aa747 100755 --- a/release-zip +++ b/release-zip @@ -11,14 +11,14 @@ import zipfile import common def main(): - os.makedirs(common.release_dir, exist_ok=True) - if os.path.exists(common.release_zip_file): - os.unlink(common.release_zip_file) - zipf = zipfile.ZipFile(common.release_zip_file, 'w', zipfile.ZIP_DEFLATED) - for arch in common.all_archs: + os.makedirs(kwargs['release_dir'], exist_ok=True) + if os.path.exists(kwargs['release_zip_file']): + os.unlink(kwargs['release_zip_file']) + zipf = zipfile.ZipFile(kwargs['release_zip_file'], 'w', zipfile.ZIP_DEFLATED) + for arch in kwargs['all_archs']: common.setup(common.get_argparse(default_args={'arch': arch})) - zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir)) - zipf.write(common.linux_image, arcname=os.path.relpath(common.linux_image, common.root_dir)) + zipf.write(kwargs['qcow2_file'], arcname=os.path.relpath(kwargs['qcow2_file'], kwargs['root_dir'])) + zipf.write(kwargs['linux_image'], arcname=os.path.relpath(kwargs['linux_image'], kwargs['root_dir'])) zipf.close() if __name__ == '__main__': diff --git a/run b/run index bedfe60..206ec5f 100755 --- a/run +++ b/run @@ -85,7 +85,7 @@ def main(args, extra_args=None): console_count = 0 if kwargs['arch'] == 'x86_64': console_type = 'ttyS' - elif common.is_arm: + elif kwargs['is_arm']: console_type = 'ttyAMA' console = '{}{}'.format(console_type, console_count) console_count += 1 @@ -118,36 +118,36 @@ def main(args, extra_args=None): def raise_rootfs_not_found(): if not kwargs['dry_run']: raise Exception('Root filesystem not found. Did you build it?\n' \ - 'Tried to use: ' + common.disk_image) + 'Tried to use: ' + kwargs['disk_image']) def raise_image_not_found(): if not kwargs['dry_run']: raise Exception('Executable image not found. Did you build it?\n' \ - 'Tried to use: ' + common.image) - if common.image is None: + 'Tried to use: ' + kwargs['image']) + if kwargs['image'] is None: raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) cmd = debug_vm.copy() - if common.emulator == 'gem5': - if common.baremetal is None: - if not os.path.exists(common.rootfs_raw_file): - if not os.path.exists(common.qcow2_file): + if kwargs['emulator'] == 'gem5': + if kwargs['baremetal'] is None: + if not os.path.exists(kwargs['rootfs_raw_file']): + if not os.path.exists(kwargs['qcow2_file']): raise_rootfs_not_found() common.raw_to_qcow2(prebuilt=kwargs['prebuilt'], reverse=True) else: - if not os.path.exists(common.gem5_fake_iso): - os.makedirs(os.path.dirname(common.gem5_fake_iso), exist_ok=True) - common.write_string_to_file(common.gem5_fake_iso, 'a' * 512) - if not os.path.exists(common.image): + if not os.path.exists(kwargs['gem5_fake_iso']): + os.makedirs(os.path.dirname(kwargs['gem5_fake_iso']), exist_ok=True) + common.write_string_to_file(kwargs['gem5_fake_iso'], 'a' * 512) + if not os.path.exists(kwargs['image']): # This is to run gem5 from a prebuilt download. - if (not common.baremetal is None) or (not os.path.exists(common.linux_image)): + if (not kwargs['baremetal'] is None) or (not os.path.exists(kwargs['linux_image'])): raise_image_not_found() - self.sh.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) - os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True) - common.write_string_to_file(common.gem5_readfile, kwargs['gem5_readfile']) + self.sh.run_cmd([os.path.join(kwargs['extract_vmlinux'], kwargs['linux_image'])]) + os.makedirs(os.path.dirname(kwargs['gem5_readfile']), exist_ok=True) + common.write_string_to_file(kwargs['gem5_readfile'], kwargs['gem5_readfile']) memory = '{}B'.format(kwargs['memory']) gem5_exe_args = common.shlex_split(kwargs['gem5_exe_args']) if do_trace: gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), LF]) - extra_env['M5_PATH'] = common.gem5_system_dir + extra_env['M5_PATH'] = kwargs['gem5_system_dir'] # https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071 extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true' if kwargs['trace_stdout']: @@ -156,16 +156,16 @@ def main(args, extra_args=None): debug_file = 'trace.txt' cmd.extend( [ - common.executable, LF, + kwargs['executable'], LF, '--debug-file', debug_file, LF, '--listener-mode', 'on', LF, - '--outdir', common.m5out_dir, LF, + '--outdir', kwargs['m5out_dir'], LF, ] + gem5_exe_args ) if kwargs['userland'] is not None: cmd.extend([ - common.gem5_se_file, LF, + kwargs['gem5_se_file'], LF, '-c', common.resolve_userland(kwargs['userland']), LF, ]) else: @@ -176,28 +176,28 @@ def main(args, extra_args=None): cpt_dir = cpt_dirs[-kwargs['gem5_restore']] extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) cmd.extend([ - common.gem5_fs_file, LF, - '--disk-image', common.disk_image, LF, - '--kernel', common.image, LF, + kwargs['gem5_fs_file'], LF, + '--disk-image', kwargs['disk_image'], LF, + '--kernel', kwargs['image'], LF, '--mem-size', memory, LF, '--num-cpus', str(kwargs['cpus']), LF, - '--script', common.gem5_readfile, LF, + '--script', kwargs['gem5_readfile'], LF, ]) if kwargs['arch'] == 'x86_64': if kwargs['kvm']: cmd.extend(['--cpu-type', 'X86KvmCPU', LF]) cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), LF]) - elif common.is_arm: + elif kwargs['is_arm']: if kwargs['kvm']: cmd.extend(['--cpu-type', 'ArmV8KvmCPU', LF]) cmd.extend([ # TODO why is it mandatory to pass mem= here? Not true for QEMU. # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), LF, - '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, kwargs['cpus'])), LF, - '--machine-type', common.machine, LF, + '--dtb-filename', os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(kwargs['armv'], kwargs['cpus'])), LF, + '--machine-type', kwargs['machine'], LF, ]) - if common.baremetal is None: + if kwargs['baremetal'] is None: cmd.extend([ '--param', 'system.panic_on_panic = True', LF]) else: @@ -215,14 +215,14 @@ def main(args, extra_args=None): cpu_type = 'atomic' if kwargs['gem5_restore'] is not None: cpt_dir = common.gem_list_checkpoint_dirs()[-kwargs['gem5_restore']] - extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)]) + extra_emulator_args.extend(['--restore-from', os.path.join(kwargs['m5out_dir'], cpt_dir)]) cmd.extend([ - os.path.join(common.gem5_source_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF, + os.path.join(kwargs['gem5_source_dir'], 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF, '--big-cpus', '2', LF, '--cpu-type', cpu_type, LF, - '--disk', common.disk_image, LF, - '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), LF, - '--kernel', common.image, LF, + '--disk', kwargs['disk_image'], LF, + '--dtb', os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), LF, + '--kernel', kwargs['image'], LF, '--little-cpus', '2', LF, ]) if kwargs['wait_gdb']: @@ -230,17 +230,17 @@ def main(args, extra_args=None): cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', LF]) else: qemu_user_and_system_options = [ - '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), LF, + '-trace', 'enable={},file={}'.format(trace_type, kwargs['qemu_trace_file']), LF, ] if kwargs['userland'] is not None: if kwargs['wait_gdb']: - debug_args = ['-g', str(common.gdb_port), LF] + debug_args = ['-g', str(kwargs['gdb_port']), LF] else: debug_args = [] cmd.extend( [ - os.path.join(common.qemu_build_dir, '{}-linux-user'.format(kwargs['arch']), 'qemu-{}'.format(kwargs['arch'])), LF, - '-L', common.target_dir, LF + os.path.join(kwargs['qemu_build_dir'], '{}-linux-user'.format(kwargs['arch']), 'qemu-{}'.format(kwargs['arch'])), LF, + '-L', kwargs['target_dir'], LF ] + qemu_user_and_system_options + common.shlex_split(kwargs['userland_before']) + @@ -250,15 +250,15 @@ def main(args, extra_args=None): ] ) else: - if not os.path.exists(common.image): + if not os.path.exists(kwargs['image']): raise_image_not_found() extra_emulator_args.extend(extra_qemu_args) common.make_run_dirs() - if kwargs['prebuilt'] or not os.path.exists(common.qemu_executable): - qemu_executable = common.qemu_executable_basename + if kwargs['prebuilt'] or not os.path.exists(kwargs['qemu_executable']): + qemu_executable = kwargs['qemu_executable_basename'] qemu_executable_prebuilt = True else: - qemu_executable = common.qemu_executable + qemu_executable = kwargs['qemu_executable'] qemu_executable_prebuilt = False qemu_executable = shutil.which(qemu_executable) if qemu_executable is None: @@ -268,17 +268,17 @@ def main(args, extra_args=None): serial_monitor = [] else: if kwargs['background']: - serial_monitor = ['-serial', 'file:{}'.format(common.qemu_background_serial_file), LF] + serial_monitor = ['-serial', 'file:{}'.format(kwargs['qemu_background_serial_file']), LF] else: serial_monitor = ['-serial', 'mon:stdio', LF] if kwargs['kvm']: extra_emulator_args.extend(['-enable-kvm', LF]) - extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port), LF]) + extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(kwargs['extra_serial_port']), LF]) virtfs_data = [ - (common.p9_dir, 'host_data'), - (common.out_dir, 'host_out'), - (common.out_rootfs_overlay_dir, 'host_out_rootfs_overlay'), - (common.rootfs_overlay_dir, 'host_rootfs_overlay'), + (kwargs['p9_dir'], 'host_data'), + (kwargs['out_dir'], 'host_out'), + (kwargs['out_rootfs_overlay_dir'], 'host_out_rootfs_overlay'), + (kwargs['rootfs_overlay_dir'], 'host_rootfs_overlay'), ] virtfs_cmd = [] for virtfs_dir, virtfs_tag in virtfs_data: @@ -293,11 +293,11 @@ def main(args, extra_args=None): [ qemu_executable, LF, '-device', 'rtl8139,netdev=net0', LF, - '-gdb', 'tcp::{}'.format(common.gdb_port), LF, - '-kernel', common.image, LF, + '-gdb', 'tcp::{}'.format(kwargs['gdb_port']), LF, + '-kernel', kwargs['image'], LF, '-m', kwargs['memory'], LF, - '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), LF, - '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), LF, + '-monitor', 'telnet::{},server,nowait'.format(kwargs['qemu_monitor_port']), LF, + '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(kwargs['qemu_hostfwd_generic_port'], kwargs['qemu_hostfwd_generic_port'], kwargs['qemu_hostfwd_ssh_port']), LF, '-no-reboot', LF, '-smp', str(kwargs['cpus']), LF, ] + @@ -308,7 +308,7 @@ def main(args, extra_args=None): if not qemu_executable_prebuilt: cmd.extend(qemu_user_and_system_options) if kwargs['initrd']: - extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')]) + extra_emulator_args.extend(['-initrd', os.path.join(kwargs['buildroot_images_dir'], 'rootfs.cpio')]) rr = kwargs['record'] or kwargs['replay'] if ramfs: # TODO why is this needed, and why any string works. @@ -324,14 +324,14 @@ def main(args, extra_args=None): root = 'root=/dev/vda' rrid = '' snapshot = ',snapshot' - if common.baremetal is None: - if not os.path.exists(common.qcow2_file): - if not os.path.exists(common.rootfs_raw_file): + if kwargs['baremetal'] is None: + if not os.path.exists(kwargs['qcow2_file']): + if not os.path.exists(kwargs['rootfs_raw_file']): raise_rootfs_not_found() common.raw_to_qcow2(prebuilt=kwargs['prebuilt']) extra_emulator_args.extend([ '-drive', - 'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid), + 'file={},format=qcow2,if={}{}{}'.format(kwargs['disk_image'], driveif, snapshot, rrid), LF, ]) if rr: @@ -342,7 +342,7 @@ def main(args, extra_args=None): if rr: extra_emulator_args.extend([ '-object', 'filter-replay,id=replay,netdev=net0', - '-icount', 'shift=7,rr={},rrfile={}'.format('record' if kwargs['record'] else 'replay', common.qemu_rrfile), + '-icount', 'shift=7,rr={},rrfile={}'.format('record' if kwargs['record'] else 'replay', kwargs['qemu_rrfile']), ]) virtio_gpu_pci = [] else: @@ -350,10 +350,10 @@ def main(args, extra_args=None): if kwargs['arch'] == 'x86_64': append = ['-append', '{} nopat {}'.format(root, kernel_cli), LF] cmd.extend([ - '-M', common.machine, LF, + '-M', kwargs['machine'], LF, '-device', 'edu', LF, ]) - elif common.is_arm: + elif kwargs['is_arm']: extra_emulator_args.extend(['-semihosting', LF]) if kwargs['arch'] == 'arm': cpu = 'cortex-a15' @@ -364,16 +364,16 @@ def main(args, extra_args=None): [ # highmem=off needed since v3.0.0 due to: # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html - '-M', '{},highmem=off'.format(common.machine), LF, + '-M', '{},highmem=off'.format(kwargs['machine']), LF, '-cpu', cpu, LF, ] + virtio_gpu_pci ) - if common.baremetal is None: + if kwargs['baremetal'] is None: cmd.extend(append) if kwargs['tmux'] is not None: tmux_args = '--run-id {}'.format(kwargs['run_id']) - if common.emulator == 'gem5': + if kwargs['emulator'] == 'gem5': tmux_cmd = './gem5-shell' elif kwargs['wait_gdb']: tmux_cmd = './run-gdb' @@ -386,13 +386,13 @@ def main(args, extra_args=None): kwargs['linux_build_id'], kwargs['run_id'], ) - if common.baremetal: - tmux_args += " --baremetal '{}'".format(common.baremetal) + if kwargs['baremetal']: + tmux_args += " --baremetal '{}'".format(kwargs['baremetal']) if kwargs['userland']: tmux_args += " --userland '{}'".format(kwargs['userland']) tmux_args += ' {}'.format(kwargs['tmux']) subprocess.Popen([ - os.path.join(common.root_dir, 'tmu'), + os.path.join(kwargs['root_dir'], 'tmu'), "sleep 2;{} {}".format(tmux_cmd, tmux_args) ]) cmd.extend(extra_emulator_args) @@ -400,10 +400,10 @@ def main(args, extra_args=None): if debug_vm or kwargs['terminal']: out_file = None else: - out_file = common.termout_file - self.sh.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) + out_file = kwargs['termout_file'] + self.sh.run_cmd(cmd, cmd_file=kwargs['run_cmd_file'], out_file=out_file, extra_env=extra_env) # Check if guest panicked. - if common.emulator == 'gem5': + if kwargs['emulator'] == 'gem5': # We have to do some parsing here because gem5 exits with status 0 even when panic happens. # Grepping for '^panic: ' does not work because some errors don't show that message. panic_msg = b'--- BEGIN LIBC BACKTRACE ---$' @@ -412,16 +412,16 @@ def main(args, extra_args=None): panic_re = re.compile(panic_msg) error_string_found = False if out_file is not None and not kwargs['dry_run']: - with open(common.termout_file, 'br') as logfile: + with open(kwargs['termout_file'], 'br') as logfile: for line in logfile: if panic_re.search(line): error_string_found = True - if os.path.exists(common.guest_terminal_file): - with open(common.guest_terminal_file, 'br') as logfile: + if os.path.exists(kwargs['guest_terminal_file']): + with open(kwargs['guest_terminal_file'], 'br') as logfile: lines = logfile.readlines() if lines: last_line = lines[-1] - if last_line.rstrip() == common.magic_fail_string: + if last_line.rstrip() == kwargs['magic_fail_string']: error_string_found = True if error_string_found: common.log_error('simulation error detected by parsing logs') diff --git a/run-docker b/run-docker index d27302e..a258a3a 100755 --- a/run-docker +++ b/run-docker @@ -5,10 +5,10 @@ import os import common -container_name = common.repo_short_id -container_hostname = common.repo_short_id -image_name = common.repo_short_id -target_dir = '/root/{}'.format(common.repo_short_id) +container_name = kwargs['repo_short_id'] +container_hostname = kwargs['repo_short_id'] +image_name = kwargs['repo_short_id'] +target_dir = '/root/{}'.format(kwargs['repo_short_id']) docker = ['sudo', 'docker'] def create(args): self.sh.run_cmd(docker + ['build', '-t', image_name, '.', LF]) diff --git a/run-gdb b/run-gdb index 3f6a037..5a0e712 100755 --- a/run-gdb +++ b/run-gdb @@ -120,15 +120,15 @@ def main(args, extra_args=None): break_at = ['-ex', 'break {}'.format(kwargs['break_at']), LF] else: break_at = [] - linux_full_system = (common.baremetal is None and kwargs['userland'] is None) + linux_full_system = (kwargs['baremetal'] is None and kwargs['userland'] is None) if kwargs['userland']: image = common.resolve_userland(kwargs['userland']) - elif common.baremetal: - image = common.image - test_script_path = os.path.splitext(common.source_path)[0] + '.py' + elif kwargs['baremetal']: + image = kwargs['image'] + test_script_path = os.path.splitext(kwargs['source_path'])[0] + '.py' else: - image = common.vmlinux - if common.baremetal: + image = kwargs['vmlinux'] + if kwargs['baremetal']: allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] else: allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] @@ -138,14 +138,14 @@ def main(args, extra_args=None): ['-q', LF] ) if linux_full_system: - cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), LF]) + cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(kwargs['linux_build_dir']), LF]) if kwargs['sim']: target = 'sim' else: if kwargs['kgdb']: - port = common.extra_serial_port + port = kwargs['extra_serial_port'] else: - port = common.gdb_port + port = kwargs['gdb_port'] target = 'remote localhost:{}'.format(port) cmd.extend([ '-ex', 'file {}'.format(image), LF, @@ -175,11 +175,11 @@ def main(args, extra_args=None): # which gets put on the kernel build root when python debugging scripts are enabled. cmd.extend(['-ex', 'continue', LF]) if not kwargs['no_lxsymbols'] and linux_full_system: - cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), LF]) + cmd.extend(['-ex', 'lx-symbols {}'.format(kwargs['kernel_modules_build_subdir']), LF]) cmd.extend(after) if kwargs['test']: GdbTestcase( - common.source_path, + kwargs['source_path'], test_script_path, cmd, verbose=kwargs['verbose'], @@ -190,8 +190,8 @@ def main(args, extra_args=None): # vmlinux: No such file or directory. return self.sh.run_cmd( cmd, - cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'), - cwd=common.linux_build_dir + cmd_file=os.path.join(kwargs['run_dir'], 'run-gdb.sh'), + cwd=kwargs['linux_build_dir'] ) if __name__ == '__main__': diff --git a/run-gdb-user b/run-gdb-user index 1ad643a..5774bba 100755 --- a/run-gdb-user +++ b/run-gdb-user @@ -5,7 +5,7 @@ import os import sys import common -rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'run-gdb')) +rungdb = imp.load_source('rungdb', os.path.join(kwargs['root_dir'], 'run-gdb')) parser = common.get_argparse(argparse_args={ 'description': '''GDB step debug guest userland processes without gdbserver. @@ -25,7 +25,7 @@ parser.add_argument( ) args = common.setup(parser) executable = common.resolve_userland(kwargs['executable']) -addr = common.get_elf_entry(os.path.join(common.buildroot_build_build_dir, executable)) +addr = common.get_elf_entry(os.path.join(kwargs['buildroot_build_build_dir'], executable)) extra_args = {} extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(addr)) # Or else lx-symbols throws for arm: diff --git a/run-gdbserver b/run-gdbserver index 623e7fe..41a1546 100755 --- a/run-gdbserver +++ b/run-gdbserver @@ -20,9 +20,9 @@ args = common.setup(parser) sys.exit(subprocess.Popen([ common.get_toolchain_tool('gdb'), '-q', - '-ex', 'set sysroot {}'.format(common.buildroot_staging_dir), - '-ex', 'target remote localhost:{}'.format(common.qemu_hostfwd_generic_port), + '-ex', 'set sysroot {}'.format(kwargs['buildroot_staging_dir']), + '-ex', 'target remote localhost:{}'.format(kwargs['qemu_hostfwd_generic_port']), '-ex', 'tbreak {}'.format(kwargs['break_at']), '-ex', 'continue', - os.path.join(common.buildroot_build_build_dir, common.resolve_userland(kwargs['executable'])), + os.path.join(kwargs['buildroot_build_build_dir'], common.resolve_userland(kwargs['executable'])), ]).wait()) diff --git a/run-toolchain b/run-toolchain index 07096ac..1d46142 100755 --- a/run-toolchain +++ b/run-toolchain @@ -35,16 +35,16 @@ parser.add_argument( nargs='*' ) args = common.setup(parser) -if common.baremetal is None: - image = common.vmlinux +if kwargs['baremetal'] is None: + image = kwargs['vmlinux'] else: - image = common.image + image = kwargs['image'] tool= common.get_toolchain_tool(kwargs['tool']) if kwargs['dry']: print(tool) else: sys.exit(self.sh.run_cmd( [tool, LF] - + common.add_newlines(kwargs['extra_args']), - cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'), + + self.sh.add_newlines(kwargs['extra_args']), + cmd_file=os.path.join(kwargs['run_dir'], 'run-toolchain.sh'), )) diff --git a/shell_helpers.py b/shell_helpers.py index 2496d58..51d7fb8 100644 --- a/shell_helpers.py +++ b/shell_helpers.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 +import distutils.file_util import itertools import os import shlex import shutil import signal +import subprocess class LF: ''' @@ -227,8 +229,8 @@ class ShellHelpers: def strip_newlines(self, cmd): return [x for x in cmd if x != LF] - def rmrf(path): - self.print_cmd(['rm', '-r', '-f', path]) + def rmrf(self, path): + self.print_cmd(['rm', '-r', '-f', path, LF]) if not self.dry_run and os.path.exists(path): shutil.rmtree(path) diff --git a/trace-boot b/trace-boot index 66e44c5..56780fa 100755 --- a/trace-boot +++ b/trace-boot @@ -6,8 +6,8 @@ import subprocess import re import common -run = imp.load_source('run', os.path.join(common.root_dir, 'run')) -qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(common.root_dir, 'qemu-trace2txt')) +run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run')) +qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(kwargs['root_dir'], 'qemu-trace2txt')) parser = common.get_argparse(argparse_args={ 'description': '''Trace the PIC addresses executed on a Linux kernel boot. @@ -23,7 +23,7 @@ args = common.setup(parser) extra_args = { 'extra_emulator_args': kwargs['extra_emulator_args'], } -if common.emulator == 'gem5': +if kwargs['emulator'] == 'gem5': extra_args.update({ 'eval': 'm5 exit', 'trace': 'Exec,-ExecSymbol,-ExecMicro', @@ -39,10 +39,10 @@ else: # Instruction count. # We could put this on a separate script, but it just adds more arch boilerplate to a new script. # So let's just leave it here for now since it did not add a significant processing time. - kernel_entry_addr = hex(common.get_elf_entry(common.vmlinux)) + kernel_entry_addr = hex(common.get_elf_entry(kwargs['vmlinux'])) nlines = 0 nlines_firmware = 0 - with open(common.qemu_trace_txt_file, 'r') as trace_file: + with open(kwargs['qemu_trace_txt_file'], 'r') as trace_file: in_firmware = True for line in trace_file: line = line.rstrip() diff --git a/trace2line b/trace2line index 451a50d..4363848 100755 --- a/trace2line +++ b/trace2line @@ -19,17 +19,17 @@ parser = common.get_argparse(argparse_args={ }) args = common.setup(parser) sys.exit(subprocess.Popen([ - os.path.join(common.root_dir, 'trace2line.sh'), - 'true' if common.emulator == 'gem5' else 'false', - common.trace_txt_file, + os.path.join(kwargs['root_dir'], 'trace2line.sh'), + 'true' if kwargs['emulator'] == 'gem5' else 'false', + kwargs['trace_txt_file'], common.get_toolchain_tool('addr2line'), - common.vmlinux, - common.run_dir, + kwargs['vmlinux'], + kwargs['run_dir'], ]).wait()) # This was the full conversion attempt. -# if common.emulator == 'gem5': +# if kwargs['emulator'] == 'gem5': # def get_pc(line): # # TODO # # stdin = sed -r 's/^.* (0x[^. ]*)[. ].*/\1/' "$common_trace_txt_file") @@ -42,15 +42,15 @@ sys.exit(subprocess.Popen([ # [ # common.get_toolchain_tool('addr2line'), # '-e', -# common.vmlinux, +# kwargs['vmlinux'], # '-f', # '-p', # ], # stdout=subprocess.PIPE, # stdin=subprocess.PIPE, # ) as proc, \ -# open(common.trace_txt_file, 'r') as infile, \ -# open(os.path.join(common.run_dir, 'trace-lines.txt'), 'w') as outfile \ +# open(kwargs['trace_txt_file'], 'r') as infile, \ +# open(os.path.join(kwargs['run_dir'], 'trace-lines.txt'), 'w') as outfile \ # : # for in_line in infile: # proc.stdin.write(get_pc(in_line).encode()) @@ -58,5 +58,5 @@ sys.exit(subprocess.Popen([ # stdout = proc.stdout.read() # outfile.write(stdout.decode()) # # TODO -# # sed -E "s|at ${common.linux_build_dir}/(\./\|)||" +# # sed -E "s|at ${kwargs['linux_build_dir']}/(\./\|)||" # # uniq -c