From 8fb9db39316d43a6dbd571e04dd46ae73915027f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sun, 4 Nov 2018 00:00:00 +0000 Subject: [PATCH] manually encode newlines on all printed commands This way we group key value arguments: e.g.: make \ -j 8 \ all instead of: make \ -j \ 8 \ all and reach CLI nirvana, while also subtly breaking several commands due to lack of testing. --- build | 4 ++ build-baremetal | 50 ++++++++--------- build-buildroot | 32 +++++------ build-crosstool-ng | 20 +++---- build-gem5 | 37 ++++++++----- build-linux | 24 ++++----- build-m5 | 14 ++--- build-modules | 19 ++++--- build-qemu | 18 +++---- build-userland | 18 +++---- common.py | 64 +++++++++++++++++----- gem5-shell | 7 ++- qemu-trace2txt | 11 ++-- run | 131 +++++++++++++++++++++++---------------------- run-gdb | 21 ++++---- run-toolchain | 4 +- 16 files changed, 263 insertions(+), 211 deletions(-) diff --git a/build b/build index b4da027..82e3145 100755 --- a/build +++ b/build @@ -76,6 +76,9 @@ name_to_component_map = { 'qemu': Component( lambda arch: run_cmd(['build-qemu'], arch), ), + 'qemu-user': Component( + lambda arch: run_cmd(['build-qemu', '--userland'], arch), + ), 'userland': Component( lambda arch: run_cmd(['build-userland'], arch), ), @@ -85,6 +88,7 @@ name_to_component_map = { 'qemu-gem5-buildroot', 'gem5-debug', 'gem5-fast', + 'qemu-user', ]), 'gem5-buildroot': Component(dependencies=[ 'buildroot-gcc', diff --git a/build-baremetal b/build-baremetal index 52725a3..37d0cf1 100755 --- a/build-baremetal +++ b/build-baremetal @@ -11,10 +11,10 @@ class BaremetalComponent(common.Component): bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext)) common_obj = os.path.join(common.baremetal_build_lib_dir, 'common{}'.format(common.obj_ext)) cflags = [ - '-ggdb3', - '-mcpu={}'.format(common.mcpu), - '-nostartfiles', - '-O0', + '-ggdb3', common.Newline, + '-mcpu={}'.format(common.mcpu), common.Newline, + '-nostartfiles', common.Newline, + '-O0', common.Newline, ] if args.prebuilt: gcc = 'arm-none-eabi-gcc' @@ -36,23 +36,23 @@ class BaremetalComponent(common.Component): os.makedirs(build_dir, exist_ok=True) os.makedirs(common.baremetal_build_lib_dir, exist_ok=True) common.run_cmd( - [gcc] + + [gcc, common.Newline] + cflags + [ - '-c', - '-o', bootloader_obj, - os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)), + '-c', common.Newline, + '-o', bootloader_obj, common.Newline, + os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)), common.Newline, ] ) common.run_cmd( - [gcc] + + [gcc, common.Newline] + cflags + [ - '-c', - '-D', - 'UART0_ADDR={:#x}'.format(uart_address), - '-o', common_obj, - os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext), + '-c', common.Newline, + '-D', common.Newline, + 'UART0_ADDR={:#x}'.format(uart_address), common.Newline, + '-o', common_obj, common.Newline, + os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext), common.Newline, ] ) self._build_dir( @@ -117,7 +117,7 @@ Build the baremetal examples with crosstool-NG. out_dir = os.path.join(common.baremetal_build_dir, subpath) os.makedirs(out_dir, exist_ok=True) if bootloader: - bootloader_cmd = [bootloader_obj] + bootloader_cmd = [bootloader_obj, common.Newline] else: bootloader_cmd = [] for in_basename in os.listdir(in_dir): @@ -126,26 +126,26 @@ Build the baremetal examples with crosstool-NG. in_name = os.path.splitext(in_basename)[0] main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext)) common.run_cmd( - [gcc] + + [gcc, common.Newline] + cflags + [ - '-c', - '-o', main_obj, - os.path.join(common.baremetal_src_dir, in_path), + '-c', common.Newline, + '-o', main_obj, common.Newline, + os.path.join(common.baremetal_src_dir, in_path), common.Newline, ] ) common.run_cmd( - [gcc] + + [gcc, common.Newline] + cflags + [ - '-Wl,--section-start=.text={:#x}'.format(entry_address), - '-o', os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext), - '-T', os.path.join(common.baremetal_src_dir, 'link.ld'), + '-Wl,--section-start=.text={:#x}'.format(entry_address), common.Newline, + '-o', os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext), common.Newline, + '-T', os.path.join(common.baremetal_src_dir, 'link.ld'), common.Newline, ] + bootloader_cmd + [ - common_obj, - main_obj, + common_obj, common.Newline, + main_obj, common.Newline, ] ) diff --git a/build-buildroot b/build-buildroot index e49f211..a21645e 100755 --- a/build-buildroot +++ b/build-buildroot @@ -59,11 +59,11 @@ usually extra Buildroot targets. def do_build(self, args): build_dir = self.get_build_dir(args) os.makedirs(common.out_dir, exist_ok=True) - extra_make_args = args.extra_make_args.copy() + extra_make_args = common.add_newlines(args.extra_make_args) if args.build_linux: - extra_make_args.append('linux-reconfigure') + extra_make_args.extend(['linux-reconfigure', common.Newline]) if args.gem5: - extra_make_args.append('gem5-reconfigure') + extra_make_args.extend(['gem5-reconfigure', common.Newline]) if args.arch == 'x86_64': defconfig = 'qemu_x86_64_defconfig' elif args.arch == 'arm': @@ -78,10 +78,10 @@ usually extra Buildroot targets. br2_external_str = ':'.join(br2_external_dirs) common.run_cmd( [ - 'make', - 'O={}'.format(common.buildroot_build_dir), - 'BR2_EXTERNAL={}'.format(br2_external_str), - defconfig, + 'make', common.Newline, + 'O={}'.format(common.buildroot_build_dir), common.Newline, + 'BR2_EXTERNAL={}'.format(br2_external_str), common.Newline, + defconfig, common.Newline, ], cwd=common.buildroot_src_dir, ) @@ -122,22 +122,22 @@ usually extra Buildroot targets. common.write_configs(common.buildroot_config_file, configs, config_fragments) common.run_cmd( [ - 'make', - 'O={}'.format(common.buildroot_build_dir), - 'olddefconfig', + 'make', common.Newline, + 'O={}'.format(common.buildroot_build_dir), common.Newline, + 'olddefconfig', common.Newline, ], cwd=common.buildroot_src_dir, ) common.make_build_dirs() if not args.no_all: - extra_make_args.append('all') + extra_make_args.extend(['all', common.Newline]) common.run_cmd( [ - 'make', - 'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_src_dir), - 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), - 'O={}'.format(common.buildroot_build_dir), - 'V={}'.format(int(args.verbose)), + 'make', common.Newline, + 'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_src_dir), common.Newline, + 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), common.Newline, + 'O={}'.format(common.buildroot_build_dir), common.Newline, + 'V={}'.format(int(args.verbose)), common.Newline, ] + extra_make_args , diff --git a/build-crosstool-ng b/build-crosstool-ng index 50440e0..485ed0b 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -16,19 +16,19 @@ class CrosstoolNgComponent(common.Component): # https://github.com/crosstool-ng/crosstool-ng/issues/1021 os.chdir(common.crosstool_ng_src_dir) common.run_cmd( - [os.path.join(common.crosstool_ng_src_dir, 'bootstrap')], + [os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), common.Newline], ) os.chdir(common.crosstool_ng_util_dir) common.run_cmd( [ - os.path.join(common.crosstool_ng_src_dir, 'configure'), - '--enable-local', + os.path.join(common.crosstool_ng_src_dir, 'configure'), common.Newline, + '--enable-local', common.Newline, ], ) common.run_cmd( [ - 'make', - '-j', str(args.nproc), + 'make', common.Newline, + '-j', str(args.nproc), common.Newline, ], ) @@ -47,16 +47,16 @@ class CrosstoolNgComponent(common.Component): ) common.run_cmd( [ - common.crosstool_ng_executable, - 'defconfig', + common.crosstool_ng_executable, common.Newline, + 'defconfig', common.Newline, ], ) os.unlink(defconfig_dest) common.run_cmd( [ - common.crosstool_ng_executable, - 'build', - 'CT_JOBS={}'.format(str(args.nproc)), + common.crosstool_ng_executable, common.Newline, + 'build', common.Newline, + 'CT_JOBS={}'.format(str(args.nproc)), common.Newline, ], out_file=os.path.join(build_dir, 'lkmc.log'), delete_env=['LD_LIBRARY_PATH'], diff --git a/build-gem5 b/build-gem5 index c1b9d46..0ca4a97 100755 --- a/build-gem5 +++ b/build-gem5 @@ -26,14 +26,14 @@ class Gem5Component(common.Component): if common.gem5_src_dir == common.gem5_default_src_dir: raise Exception('gem5 submodule not checked out') common.run_cmd([ - 'git', - '-C', common.gem5_default_src_dir, - 'worktree', 'add', - '-b', os.path.join('wt', args.gem5_build_id), - common.gem5_src_dir + 'git', common.Newline, + '-C', common.gem5_default_src_dir, common.Newline, + 'worktree', 'add', common.Newline, + '-b', os.path.join('wt', args.gem5_build_id), common.Newline, + common.gem5_src_dir, common.Newline, ]) if args.verbose: - verbose = ['--verbose'] + verbose = ['--verbose', common.Newline] else: verbose = [] if args.arch == 'x86_64': @@ -53,7 +53,10 @@ class Gem5Component(common.Component): # dtb dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt') - common.run_cmd(['make', '-C', dt_src_dir]) + common.run_cmd([ + 'make', common.Newline, + '-C', dt_src_dir, common.Newline, + ]) common.copy_dir_if_update_non_recursive( srcdir=dt_src_dir, destdir=dt_build_dir, @@ -63,25 +66,31 @@ class Gem5Component(common.Component): # Bootloader 32. bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader') # TODO use the buildroot cross compiler here, and remove the dependencies from configure. - common.run_cmd(['make', '-C', bootloader32_dir]) + common.run_cmd([ + 'make', common.Newline, + '-C', bootloader32_dir, common.Newline, + ]) # bootloader common.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) # Bootloader 64. bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') # TODO cross_compile is ignored because the make does not use CC... - common.run_cmd(['make', '-C', bootloader64_dir]) + common.run_cmd([ + 'make', common.Newline, + '-C', bootloader64_dir, common.Newline + ]) common.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) common.run_cmd( ( [ - 'scons', - '-j', str(args.nproc), - '--ignore-style', - common.gem5_executable + 'scons', common.Newline, + '-j', str(args.nproc), common.Newline, + '--ignore-style', common.Newline, + common.gem5_executable, common.Newline, ] + verbose + - args.extra_scons_args + common.add_newlines(args.extra_scons_args) ), cwd=common.gem5_src_dir, extra_paths=[common.ccache_dir], diff --git a/build-linux b/build-linux index 6f0c451..9b2ca53 100755 --- a/build-linux +++ b/build-linux @@ -66,12 +66,12 @@ Configure the kernel, but don't build it. else: cc = gcc common_make_args = [ - 'make', - '-j', str(args.nproc), - 'ARCH={}'.format(common.linux_arch), - 'CROSS_COMPILE={}'.format(prefix), - 'CC={}'.format(cc), - 'O={}'.format(build_dir), + 'make', common.Newline, + '-j', str(args.nproc), common.Newline, + 'ARCH={}'.format(common.linux_arch), common.Newline, + 'CROSS_COMPILE={}'.format(prefix), common.Newline, + 'CC={}'.format(cc), common.Newline, + 'O={}'.format(build_dir), common.Newline, ] if args.verbose: verbose = ['V=1'] @@ -99,17 +99,17 @@ Configure the kernel, but don't build it. ) common.run_cmd( [ - os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), - '-m', - '-O', build_dir, - os.path.join(build_dir, '.config'), + os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), common.Newline, + '-m', common.Newline, + '-O', build_dir, common.Newline, + os.path.join(build_dir, '.config'), common.Newline, ] + config_fragments ) common.run_cmd( ( common_make_args + - ['olddefconfig'] + ['olddefconfig', common.Newline] ), **common_args ) @@ -117,7 +117,7 @@ Configure the kernel, but don't build it. common.run_cmd( ( common_make_args + - args.extra_make_args + common.add_newlines(args.extra_make_args) ), **common_args ) diff --git a/build-m5 b/build-m5 index b1dd68d..18c7ee2 100755 --- a/build-m5 +++ b/build-m5 @@ -14,12 +14,12 @@ class M5Component(common.Component): else: arch = args.arch return [ - 'make', - '-j', str(args.nproc), - '-f', 'Makefile.{}'.format(arch), - 'CC={}'.format(cc), - 'LD={}'.format(ld), - 'PWD={}'.format(common.gem5_m5_src_dir), + 'make', common.Newline, + '-j', str(args.nproc), common.Newline, + '-f', 'Makefile.{}'.format(arch), common.Newline, + 'CC={}'.format(cc), common.Newline, + 'LD={}'.format(ld), common.Newline, + 'PWD={}'.format(common.gem5_m5_src_dir), common.Newline, ] def do_build(self, args): @@ -36,7 +36,7 @@ class M5Component(common.Component): def clean(self, args): common.run_cmd( - self.get_make_cmd(args) + ['clean'], + self.get_make_cmd(args) + ['clean', common.Newline], cwd=common.gem5_m5_src_dir, ) diff --git a/build-modules b/build-modules index 9e33ff2..dcfc62d 100755 --- a/build-modules +++ b/build-modules @@ -3,7 +3,6 @@ import distutils.dir_util import os import platform -import shlex import shutil import common @@ -90,16 +89,16 @@ Use the host packaged cross toolchain. common.run_cmd( ( [ - 'make', - '-j', str(args.nproc), - 'ARCH={}'.format(common.linux_arch), - 'CC={}'.format(cc), - 'CROSS_COMPILE={}'.format(prefix), - 'LINUX_DIR={}'.format(linux_dir), - 'M={}'.format(build_subdir), - 'OBJECT_FILES={}'.format(' '.join(object_files)), + 'make', common.Newline, + '-j', str(args.nproc), common.Newline, + 'ARCH={}'.format(common.linux_arch), common.Newline, + 'CC={}'.format(cc), common.Newline, + 'CROSS_COMPILE={}'.format(prefix), common.Newline, + 'LINUX_DIR={}'.format(linux_dir), common.Newline, + 'M={}'.format(build_subdir), common.Newline, + 'OBJECT_FILES={}'.format(' '.join(object_files)), common.Newline, ] + - shlex.split(args.make_args) + + common.shlex_split(args.make_args) + verbose ), cwd=os.path.join(common.kernel_modules_build_subdir), diff --git a/build-qemu b/build-qemu index 3135204..5299d07 100755 --- a/build-qemu +++ b/build-qemu @@ -32,22 +32,22 @@ class QemuComponent(common.Component): target_list = '{}-softmmu'.format(args.arch) common.run_cmd( [ - os.path.join(common.qemu_src_dir, 'configure'), - '--enable-debug', - '--enable-trace-backends=simple', - '--target-list={}'.format(target_list), - '--enable-sdl', - '--with-sdlabi=2.0', + os.path.join(common.qemu_src_dir, 'configure'), common.Newline, + '--enable-debug', common.Newline, + '--enable-trace-backends=simple', common.Newline, + '--target-list={}'.format(target_list), common.Newline, + '--enable-sdl', common.Newline, + '--with-sdlabi=2.0', common.Newline, ] + - args.extra_config_args, + common.add_newlines(args.extra_config_args), extra_paths=[common.ccache_dir], cwd=build_dir ) common.run_cmd( ( [ - 'make', - '-j', str(args.nproc), + 'make', common.Newline, + '-j', str(args.nproc), common.Newline, ] + verbose diff --git a/build-userland b/build-userland index 6d2516c..95001ad 100755 --- a/build-userland +++ b/build-userland @@ -56,17 +56,17 @@ has the OpenBLAS libraries and headers installed. common.run_cmd( ( [ - 'make', - '-j', str(args.nproc), - 'CC={}'.format(cc), - 'CXX={}'.format(cxx), - 'PKG_CONFIG={}'.format(common.buildroot_pkg_config), - 'STAGING_DIR={}'.format(common.buildroot_staging_dir), - 'OUT_DIR={}'.format(build_dir), + 'make', common.Newline, + '-j', str(args.nproc), common.Newline, + 'CC={}'.format(cc), common.Newline, + 'CXX={}'.format(cxx), common.Newline, + 'PKG_CONFIG={}'.format(common.buildroot_pkg_config), common.Newline, + 'STAGING_DIR={}'.format(common.buildroot_staging_dir), common.Newline, + 'OUT_DIR={}'.format(build_dir), common.Newline, ] + - ['HAS_{}=y'.format(package.upper()) for package in args.has_package] + + common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in args.has_package]) + shlex.split(args.make_args) + - [os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets] + common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets]) ), cwd=common.userland_src_dir, extra_paths=[common.ccache_dir], diff --git a/common.py b/common.py index 0dd90e5..66456a5 100644 --- a/common.py +++ b/common.py @@ -8,6 +8,7 @@ import datetime import distutils.file_util import glob import imp +import itertools import json import multiprocessing import os @@ -144,6 +145,13 @@ class Component: ''' return {} +class Newline: + ''' + Singleton class. Can be used in print_cmd to print out nicer command lines + with -key on the same line as "-key value". + ''' + pass + def add_dry_run_argument(parser): parser.add_argument('--dry-run', default=False, action='store_true', help='''\ Print the commands that would be run, but don't run them. @@ -154,6 +162,12 @@ Bash equivalents even for actions taken directly in Python without shelling out. mkdir are generally omitted since those are obvious. ''') +def add_newlines(cmd): + out = [] + for arg in cmd: + out.extend([arg, this_module.Newline]) + return out + def base64_encode(string): return base64.b64encode(string.encode()).decode() @@ -456,19 +470,29 @@ def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None): Format a command given as a list of strings so that it can be viewed nicely and executed by bash directly and print it to stdout. ''' - newline_separator = ' \\\n' + last_newline = ' \\\n' + newline_separator = last_newline + ' ' out = [] if extra_env is None: extra_env = {} if cwd is not None: - out.append('cd {} &&{}'.format(shlex.quote(cwd), newline_separator)) + out.append('cd {} &&'.format(shlex.quote(cwd))) if extra_paths is not None: - out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)) + newline_separator) + out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths))) for key in extra_env: - out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])) + newline_separator) + out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key]))) + cmd_quote = [] + has_newline = False for arg in cmd: - out.append(shlex.quote(arg) + newline_separator) - return ' '.join(out) + ';' + if arg == this_module.Newline: + cmd_quote.append(arg) + has_newline = True + else: + cmd_quote.append(shlex.quote(arg)) + if has_newline: + cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == this_module.Newline) if not x] + out.extend(cmd_quote) + return newline_separator.join(out) + last_newline + ';' def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): ''' @@ -476,6 +500,9 @@ def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): Optionally save the command to cmd_file file, and add extra_env environment variables to the command generated. + + If cmd contains at least one common.Newline, newlines are only added on common.Newline. + Otherwise, newlines are added automatically after every word. ''' global dry_run if type(cmd) is str: @@ -513,14 +540,14 @@ def raw_to_qcow2(prebuilt=False, reverse=False): infile = outfile outfile = tmp this_module.run_cmd([ - qemu_img_executable, + qemu_img_executable, this_module.Newline, # Prevent qemu-img from generating trace files like QEMU. Disgusting. - '-T', 'pr_manager_run,file=/dev/null', - 'convert', - '-f', infmt, - '-O', outfmt, - infile, - outfile, + '-T', 'pr_manager_run,file=/dev/null', this_module.Newline, + 'convert', this_module.Newline, + '-f', infmt, this_module.Newline, + '-O', outfmt, this_module.Newline, + infile, this_module.Newline, + outfile, this_module.Newline, ]) def raise_no_x86(arch): @@ -563,7 +590,7 @@ def run_cmd( Wait until the command finishes execution. - :param cmd: command to run + :param cmd: command to run. common.Newline entries are magic get skipped. :type cmd: List[str] :param cmd_file: if not None, write the command to be run to that file @@ -625,6 +652,7 @@ def run_cmd( #sigpipe_old = signal.getsignal(signal.SIGPIPE) #signal.signal(signal.SIGPIPE, signal.SIG_DFL) + cmd = [x for x in cmd if x != this_module.Newline] if not dry_run and not this_module.dry_run: # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: @@ -886,6 +914,14 @@ def setup(parser): def setup_dry_run_arguments(args): this_module.dry_run = args.dry_run +def shlex_split(string): + ''' + shlex_split, but also add Newline after every word. + + Not perfect since it does not group arguments, but I don't see a solution. + ''' + return this_module.add_newlines(shlex.split(string)) + def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext): if os.path.isabs(in_path): return in_path diff --git a/gem5-shell b/gem5-shell index 1abe48a..f669b57 100755 --- a/gem5-shell +++ b/gem5-shell @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import subprocess import sys import common @@ -10,4 +9,8 @@ parser = common.get_argparse( argparse_args={'description':'Connect a terminal to a running gem5 instance'} ) args = common.setup(parser) -sys.exit(common.run_cmd([str(common.gem5_m5term), 'localhost', str(common.gem5_telnet_port)])) +sys.exit(common.run_cmd([ + common.gem5_m5term, common.Newline, + 'localhost', common.Newline, + str(common.gem5_telnet_port), common.Newline, +])) diff --git a/qemu-trace2txt b/qemu-trace2txt index ff58d8a..fa4ab1f 100755 --- a/qemu-trace2txt +++ b/qemu-trace2txt @@ -7,13 +7,12 @@ import sys import common def main(): - cmd = [ - os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), - os.path.join(common.qemu_build_dir, 'trace-events-all'), - os.path.join(common.qemu_trace_file), - ] return common.run_cmd( - cmd, + [ + os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), common.Newline, + os.path.join(common.qemu_build_dir, 'trace-events-all'), common.Newline, + os.path.join(common.qemu_trace_file), common.Newline, + ], cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'), out_file=common.qemu_trace_txt_file, show_stdout=False, diff --git a/run b/run index 933a37a..ab0fb45 100755 --- a/run +++ b/run @@ -2,7 +2,6 @@ import os import re -import shlex import shutil import subprocess import sys @@ -55,18 +54,17 @@ def main(args, extra_args=None): extra_emulator_args = [] extra_qemu_args = [] if args.debug_vm is not None: - print(args.debug_vm) - debug_vm = ['gdb', '-q'] + shlex.split(args.debug_vm) + ['--args'] + debug_vm = ['gdb', common.Newline, '-q', common.Newline] + common.shlex_split(args.debug_vm) + ['--args', common.Newline] else: debug_vm = [] if args.wait_gdb: - extra_qemu_args.append('-S') + extra_qemu_args.extend(['-S', common.Newline]) if args.eval_busybox is not None: kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(common.base64_encode(args.eval_busybox)) if args.kernel_cli_after_dash is not None: kernel_cli_after_dash += ' {}'.format(args.kernel_cli_after_dash) if args.vnc: - vnc = ['-vnc', ':0'] + vnc = ['-vnc', ':0', common.Newline] else: vnc = [] if args.initrd or args.initramfs: @@ -81,7 +79,7 @@ def main(args, extra_args=None): kernel_cli += ' {}=/eval_base64.sh'.format(initarg) kernel_cli_after_dash += ' lkmc_eval="{}"'.format(common.base64_encode(args.eval)) if not args.graphic: - extra_qemu_args.append('-nographic') + extra_qemu_args.extend(['-nographic', common.Newline]) console = None console_type = None console_count = 0 @@ -144,24 +142,27 @@ def main(args, extra_args=None): os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True) common.write_string_to_file(common.gem5_readfile, args.gem5_readfile) memory = '{}B'.format(args.memory) - gem5_exe_args = shlex.split(args.gem5_exe_args) + gem5_exe_args = common.shlex_split(args.gem5_exe_args) if do_trace: - gem5_exe_args.append('--debug-flags={}'.format(trace_type)) + gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), common.Newline]) extra_env['M5_PATH'] = common.gem5_system_dir # https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071 extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true' cmd.extend( [ - common.executable, - '--debug-file=trace.txt', + common.executable, common.Newline, + '--debug-file=trace.txt', common.Newline ] + gem5_exe_args + [ - '-d', common.m5out_dir + '-d', common.m5out_dir, common.Newline ] ) if args.userland is not None: - cmd.extend([common.gem5_se_file, '-c', common.resolve_userland(args.userland)]) + cmd.extend([ + common.gem5_se_file, common.Newline, + '-c', common.resolve_userland(args.userland), common.Newline, + ]) else: if args.gem5_script == 'fs': # TODO port @@ -170,67 +171,67 @@ def main(args, extra_args=None): cpt_dir = cpt_dirs[-args.gem5_restore] extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) cmd.extend([ - common.gem5_fs_file, - '--disk-image', common.disk_image, - '--kernel', common.image, - '--mem-size', memory, - '--num-cpus', str(args.cpus), - '--script', common.gem5_readfile, + common.gem5_fs_file, common.Newline, + '--disk-image', common.disk_image, common.Newline, + '--kernel', common.image, common.Newline, + '--mem-size', memory, common.Newline, + '--num-cpus', str(args.cpus), common.Newline, + '--script', common.gem5_readfile, common.Newline, ]) if args.arch == 'x86_64': if args.kvm: - cmd.extend(['--cpu-type', 'X86KvmCPU']) - cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli)]) + cmd.extend(['--cpu-type', 'X86KvmCPU', common.Newline]) + cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), common.Newline]) elif common.is_arm: # TODO why is it mandatory to pass mem= here? Not true for QEMU. # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? cmd.extend([ - '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), - '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)), - '--machine-type', common.machine, - '--param', 'system.panic_on_panic = True', + '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), common.Newline, + '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)), common.Newline, + '--machine-type', common.machine, common.Newline, + '--param', 'system.panic_on_panic = True', common.Newline, ]) if not args.baremetal is None: - cmd.append('--bare-metal') + cmd.extend(['--bare-metal', common.Newline]) if args.arch == 'aarch64': # https://stackoverflow.com/questions/43682311/uart-communication-in-gem5-with-arm-bare-metal/50983650#50983650 - cmd.extend(['--param', 'system.highest_el_is_64 = True']) - cmd.extend(['--param', 'system.auto_reset_addr_64 = True']) + cmd.extend(['--param', 'system.highest_el_is_64 = True', common.Newline]) + cmd.extend(['--param', 'system.auto_reset_addr_64 = True', common.Newline]) elif args.gem5_script == 'biglittle': if args.gem5_restore is not None: cpt_dir = common.gem_list_checkpoint_dirs()[-args.gem5_restore] extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)]) cmd.extend([ - os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), - '--big-cpus', '2', - '--cpu-type', 'atomic', - '--disk', common.disk_image, - '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), - '--kernel', common.image, - '--little-cpus', '2' + os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), common.Newline, + '--big-cpus', '2', common.Newline, + '--cpu-type', 'atomic', common.Newline, + '--disk', common.disk_image, common.Newline, + '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), common.Newline, + '--kernel', common.image, common.Newline, + '--little-cpus', '2', common.Newline, ]) if args.wait_gdb: # https://stackoverflow.com/questions/49296092/how-to-make-gem5-wait-for-gdb-to-connect-to-reliably-break-at-start-kernel-of-th - cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True']) + cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', common.Newline]) else: qemu_user_and_system_options = [ - '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), + '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), common.Newline, ] if args.userland is not None: if args.wait_gdb: - debug_args = ['-g', str(common.gdb_port)] + debug_args = ['-g', str(common.gdb_port), common.Newline] else: debug_args = [] cmd.extend( [ - os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)), - '-L', common.target_dir, + os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)), common.Newline, + '-L', common.target_dir, common.Newline ] + qemu_user_and_system_options + - shlex.split(args.userland_before) + + common.shlex_split(args.userland_before) + debug_args + [ - common.resolve_userland(args.userland) + common.resolve_userland(args.userland), common.Newline ] ) else: @@ -250,10 +251,10 @@ def main(args, extra_args=None): if args.debug_vm: serial_monitor = [] else: - serial_monitor = ['-serial', 'mon:stdio'] + serial_monitor = ['-serial', 'mon:stdio', common.Newline] if args.kvm: - extra_emulator_args.append('-enable-kvm') - extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port)]) + extra_emulator_args.extend(['-enable-kvm', common.Newline]) + extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port), common.Newline]) virtfs_data = [ (common.p9_dir, 'host_data'), (common.out_dir, 'host_out'), @@ -266,19 +267,20 @@ def main(args, extra_args=None): virtfs_cmd.extend([ '-virtfs', 'local,path={virtfs_dir},mount_tag={virtfs_tag},security_model=mapped,id={virtfs_tag}' \ - .format(virtfs_dir=virtfs_dir, virtfs_tag=virtfs_tag - )]) + .format(virtfs_dir=virtfs_dir, virtfs_tag=virtfs_tag), + common.Newline, + ]) cmd.extend( [ - qemu_executable, - '-device', 'rtl8139,netdev=net0', - '-gdb', 'tcp::{}'.format(common.gdb_port), - '-kernel', common.image, - '-m', args.memory, - '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), - '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), - '-no-reboot', - '-smp', str(args.cpus), + qemu_executable, common.Newline, + '-device', 'rtl8139,netdev=net0', common.Newline, + '-gdb', 'tcp::{}'.format(common.gdb_port), common.Newline, + '-kernel', common.image, common.Newline, + '-m', args.memory, common.Newline, + '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), common.Newline, + '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), common.Newline, + '-no-reboot', common.Newline, + '-smp', str(args.cpus), common.Newline, ] + virtfs_cmd + qemu_user_and_system_options + @@ -309,12 +311,13 @@ def main(args, extra_args=None): common.raw_to_qcow2(prebuilt=args.prebuilt) extra_emulator_args.extend([ '-drive', - 'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid) + 'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid), + common.Newline, ]) if rr: extra_emulator_args.extend([ - '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', - '-device', 'ide-hd,drive=img-blkreplay' + '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', common.Newline, + '-device', 'ide-hd,drive=img-blkreplay', common.Newline, ]) if rr: extra_emulator_args.extend([ @@ -325,13 +328,13 @@ def main(args, extra_args=None): else: virtio_gpu_pci = ['-device', 'virtio-gpu-pci'] if args.arch == 'x86_64': - append = ['-append', '{} nopat {}'.format(root, kernel_cli)] + append = ['-append', '{} nopat {}'.format(root, kernel_cli), common.Newline] cmd.extend([ - '-M', common.machine, - '-device', 'edu', + '-M', common.machine, common.Newline, + '-device', 'edu', common.Newline, ]) elif common.is_arm: - extra_emulator_args.append('-semihosting') + extra_emulator_args.extend(['-semihosting', common.Newline]) if args.arch == 'arm': cpu = 'cortex-a15' else: @@ -341,8 +344,8 @@ def main(args, extra_args=None): [ # highmem=off needed since v3.0.0 due to: # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html - '-M', '{},highmem=off'.format(common.machine), - '-cpu', cpu, + '-M', '{},highmem=off'.format(common.machine), common.Newline, + '-cpu', cpu, common.Newline, ] + virtio_gpu_pci ) diff --git a/run-gdb b/run-gdb index 9433939..8cd8dae 100755 --- a/run-gdb +++ b/run-gdb @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import shlex import sys import signal import subprocess @@ -33,10 +32,10 @@ def main(args, extra_args=None): ''' global defaults args = common.resolve_args(defaults, args, extra_args) - after = shlex.split(args.after) - before = shlex.split(args.before) + after = common.shlex_split(args.after) + before = common.shlex_split(args.before) if args.break_at is not None: - break_at = ['-ex', 'break {}'.format(args.break_at)] + break_at = ['-ex', 'break {}'.format(args.break_at), common.Newline] else: break_at = [] linux_full_system = (args.baremetal is None and args.userland is None) @@ -51,12 +50,12 @@ def main(args, extra_args=None): else: allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] cmd = ( - [common.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains)] + + [common.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains), common.Newline] + before + - ['-q'] + ['-q', common.Newline] ) if linux_full_system: - cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir)]) + cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), common.Newline]) if args.sim: target = 'sim' else: @@ -66,8 +65,8 @@ def main(args, extra_args=None): port = common.gdb_port target = 'remote localhost:{}'.format(port) cmd.extend([ - '-ex', 'file {}'.format(image), - '-ex', 'target {}'.format(target), + '-ex', 'file {}'.format(image), common.Newline, + '-ex', 'target {}'.format(target), common.Newline, ]) if not args.kgdb: cmd.extend(break_at) @@ -91,9 +90,9 @@ def main(args, extra_args=None): # # The lx-symbols commands gets loaded through the file vmlinux-gdb.py # which gets put on the kernel build root when python debugging scripts are enabled. - cmd.extend(['-ex', 'continue']) + cmd.extend(['-ex', 'continue', common.Newline]) if not args.no_lxsymbols and linux_full_system: - cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir)]) + cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), common.Newline]) cmd.extend(after) # I would rather have cwd be out_rootfs_overlay_dir, # but then lx-symbols cannot fine the vmlinux and fails with: diff --git a/run-toolchain b/run-toolchain index 3ce0c95..f2d6e7c 100755 --- a/run-toolchain +++ b/run-toolchain @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import subprocess import sys import common @@ -45,6 +44,7 @@ if args.dry: print(tool) else: sys.exit(common.run_cmd( - [tool] + args.extra_args, + [tool, common.Newline] + + common.add_newlines(args.extra_args), cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'), ))