diff --git a/build-buildroot b/build-buildroot index 457fe2f..2685460 100755 --- a/build-buildroot +++ b/build-buildroot @@ -129,7 +129,9 @@ usually extra Buildroot targets. config_fragments = [ os.path.join(common.root_dir, 'buildroot_config', 'default') ] + args.config_fragment - common.write_configs(common.buildroot_config_file, configs, config_fragments) + # TODO Can't get rid of these for now with nice fragments on Buildroot: + # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config + self.sh.write_configs(common.buildroot_config_file, configs, config_fragments) common.run_cmd( [ 'make', common.Newline, diff --git a/build-gem5 b/build-gem5 index 4c5e19c..195a203 100755 --- a/build-gem5 +++ b/build-gem5 @@ -5,7 +5,7 @@ import pathlib import subprocess import common -from cli_function import Argument +from shell_helpers import LF class Main(common.BuildCliFunction): def __init__(self): @@ -15,7 +15,6 @@ class Main(common.BuildCliFunction): metavar='extra-scons-args', nargs='*', ) - common.add_args_gem5(self) def build(self, **kwargs): build_dir = self.get_build_dir(**kwargs) @@ -27,15 +26,15 @@ class Main(common.BuildCliFunction): if not os.path.exists(os.path.join(kwargs['gem5_source_dir'], '.git')): if kwargs['gem5_source_dir'] == kwargs['gem5_default_src_dir']: raise Exception('gem5 submodule not checked out') - common.run_cmd([ - 'git', common.Newline, - '-C', kwargs['gem5_default_src_dir'], common.Newline, - 'worktree', 'add', common.Newline, - '-b', os.path.join('wt', kwargs['gem5_build_id']), common.Newline, - kwargs['gem5_source_dir'], common.Newline, + self.sh.run_cmd([ + 'git', LF, + '-C', kwargs['gem5_default_src_dir'], LF, + 'worktree', 'add', LF, + '-b', os.path.join('wt', kwargs['gem5_build_id']), LF, + kwargs['gem5_source_dir'], LF, ]) if kwargs['verbose']: - verbose = ['--verbose', common.Newline] + verbose = ['--verbose', LF] else: verbose = [] if kwargs['arch'] == 'x86_64': @@ -44,7 +43,7 @@ class Main(common.BuildCliFunction): zeroes = b'\x00' * (2 ** 16) for i in range(2 ** 10): dummy_img_file.write(zeroes) - common.run_cmd(['mkswap', dummy_img_path]) + self.sh.run_cmd(['mkswap', dummy_img_path]) with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'): # This file must always be present, despite --kernel overriding that default and selecting the kernel. # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present. @@ -55,9 +54,9 @@ class Main(common.BuildCliFunction): # dtb dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') dt_build_dir = os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt') - common.run_cmd([ - 'make', common.Newline, - '-C', dt_src_dir, common.Newline, + self.sh.run_cmd([ + 'make', LF, + '-C', dt_src_dir, LF, ]) common.copy_dir_if_update_non_recursive( srcdir=dt_src_dir, @@ -68,46 +67,46 @@ class Main(common.BuildCliFunction): # Bootloader 32. bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader') # TODO use the buildroot cross compiler here, and remove the dependencies from configure. - common.run_cmd([ - 'make', common.Newline, - '-C', bootloader32_dir, common.Newline, - 'CROSS_COMPILE=arm-linux-gnueabihf-', common.Newline, + self.sh.run_cmd([ + 'make', LF, + '-C', bootloader32_dir, LF, + 'CROSS_COMPILE=arm-linux-gnueabihf-', LF, ]) # bootloader - common.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) + self.sh.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) # Bootloader 64. bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') # TODO cross_compile is ignored because the make does not use CC... - common.run_cmd([ - 'make', common.Newline, - '-C', bootloader64_dir, common.Newline + self.sh.run_cmd([ + 'make', LF, + '-C', bootloader64_dir, LF ]) - common.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) - common.run_cmd( + self.sh.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) + self.sh.run_cmd( ( [ - 'scons', common.Newline, - '-j', str(kwargs['nproc']), common.Newline, - '--gold-linker', common.Newline, - '--ignore-style', common.Newline, - kwargs['gem5_executable'], common.Newline, + 'scons', LF, + '-j', str(kwargs['nproc']), LF, + '--gold-linker', LF, + '--ignore-style', LF, + kwargs['gem5_executable'], LF, ] + verbose + - common.add_newlines(kwargs['extra_scons_args']) + self.sh.add_newlines(kwargs['extra_scons_args']) ), cwd=kwargs['gem5_source_dir'], extra_paths=[kwargs['ccache_dir']], ) term_src_dir = os.path.join(kwargs['gem5_source_dir'], 'util/term') m5term_build = os.path.join(term_src_dir, 'm5term') - common.run_cmd(['make', '-C', term_src_dir]) + self.sh.run_cmd(['make', '-C', term_src_dir]) if os.path.exists(kwargs['gem5_m5term']): - # Otherwise common.cp would fail with "Text file busy" if you + # Otherwise self.sh.cp would fail with "Text file busy" if you # tried to rebuild while running m5term: # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512 os.unlink(kwargs['gem5_m5term']) - common.cp(m5term_build, kwargs['gem5_m5term']) + self.sh.cp(m5term_build, kwargs['gem5_m5term']) def get_build_dir(self, **kwargs): return kwargs['gem5_build_dir'] diff --git a/cli_function.py b/cli_function.py index f0629d1..216bdce 100644 --- a/cli_function.py +++ b/cli_function.py @@ -195,10 +195,28 @@ amazing function! return kwargs # Code calls. - assert OneCliFunction()(pos_mandatory=1) == {'asdf': 'A', 'qwer': 'Q', 'bool': True, 'bool_cli': True, 'no_default': None, 'pos_mandatory': 1, 'pos_optional': 0, 'args_star': []} - assert OneCliFunction()(pos_mandatory=1, asdf='B') == {'asdf': 'B', 'qwer': 'Q', 'bool': True, 'bool_cli': True, 'no_default': None, 'pos_mandatory': 1, 'pos_optional': 0, 'args_star': []} - assert OneCliFunction()(pos_mandatory=1, bool=False) == {'asdf': 'A', 'qwer': 'Q', 'bool': False, 'bool_cli': True, 'no_default': None, 'pos_mandatory': 1, 'pos_optional': 0, 'args_star': []} - assert OneCliFunction()(pos_mandatory=1, asdf='B', qwer='R', bool=False) == {'asdf': 'B', 'qwer': 'R', 'bool': False, 'bool_cli': True, 'no_default': None, 'pos_mandatory': 1, 'pos_optional': 0, 'args_star': []} + default = OneCliFunction()(pos_mandatory=1) + assert default == {'asdf': 'A', 'qwer': 'Q', 'bool': True, 'bool_cli': True, 'no_default': None, 'pos_mandatory': 1, 'pos_optional': 0, 'args_star': []} + + # asdf + out = OneCliFunction()(pos_mandatory=1, asdf='B') + assert out['asdf'] == 'B' + out['asdf'] = default['asdf'] + assert(out == default) + + # asdf and qwer + out = OneCliFunction()(pos_mandatory=1, asdf='B', qwer='R') + assert out['asdf'] == 'B' + assert out['qwer'] == 'R' + out['asdf'] = default['asdf'] + out['qwer'] = default['qwer'] + assert(out == default) + + # bool + out = OneCliFunction()(pos_mandatory=1, bool=False) + assert out['bool'] == False + out['bool'] = default['bool'] + assert(out == default) # Force a boolean value set on the config to be False on CLI. assert OneCliFunction().cli(['--no-bool-cli', '1'])['bool_cli'] is False diff --git a/common.py b/common.py index 1c2ddc4..47df412 100644 --- a/common.py +++ b/common.py @@ -8,12 +8,10 @@ import datetime import distutils.file_util import glob import imp -import itertools import json import multiprocessing import os import re -import shlex import shutil import signal import stat @@ -24,6 +22,7 @@ import urllib import urllib.request import cli_function +import shell_helpers common = sys.modules[__name__] @@ -90,7 +89,6 @@ consts['header_ext'] = '.h' consts['kernel_module_ext'] = '.ko' consts['obj_ext'] = '.o' consts['config_file'] = os.path.join(consts['data_dir'], 'config.py') -consts['command_prefix'] = '+ ' consts['magic_fail_string'] = b'lkmc_test_fail' consts['baremetal_lib_basename'] = 'lib' @@ -103,6 +101,8 @@ class LkmcCliFunction(cli_function.CliFunction): ''' def __init__(self): super().__init__(config_file=common.consts['config_file']) + + # Args for all scripts. self.add_argument( '-a', '--arch', choices=consts['arch_choices'], default=consts['default_arch'], help='CPU architecture.' @@ -124,6 +124,138 @@ mkdir are generally omitted since those are obvious help='Show full compilation commands when they are not shown by default.' ) + # Gem5 args. + self.add_argument( + '--gem5-build-dir', + help='''\ +Use the given directory as the gem5 build directory. +''' + ) + self.add_argument( + '-M', '--gem5-build-id', default='default', + help='''\ +gem5 build ID. Allows you to keep multiple separate gem5 builds. +''' + ) + self.add_argument( + '--gem5-build-type', default='opt', + help='gem5 build type, most often used for "debug" builds.' + ) + self.add_argument( + '--gem5-source-dir', + help='''\ +Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`. +''' + ) + self.add_argument( + '-N', '--gem5-worktree', + help='''\ +Create and use a git worktree of the gem5 submodule. +See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree +''' + ) + + # Linux kernel. + self.add_argument( + '-L', '--linux-build-id', default=consts['default_build_id'], + help='''\ +Linux build ID. Allows you to keep multiple separate Linux builds. +''' + ) + + # Baremetal. + self.add_argument( + '-b', '--baremetal', + help='''\ +Use the given baremetal executable instead of the Linux kernel. + +If the path is absolute, it is used as is. + +If the path is relative, we assume that it points to a source code +inside baremetal/ and then try to use corresponding executable. +''' + ) + + # Buildroot. + self.add_argument( + '--buildroot-build-id', + default=consts['default_build_id'], + help='Buildroot build ID. Allows you to keep multiple separate gem5 builds.' + ) + self.add_argument( + '--buildroot-linux', default=False, action='store_true', + help='Boot with the Buildroot Linux kernel instead of our custom built one. Mostly for sanity checks.' + ) + + # crosstool-ng + self.add_argument( + '--crosstool-ng-build-id', default=consts['default_build_id'], + help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds.' + ) + self.add_argument( + '--docker', default=False, action='store_true', + help='''\ +Use the docker download Ubuntu root filesystem instead of the default Buildroot one. +''' + ) + + self.add_argument( + '--machine', + help='''Machine type. +QEMU default: virt +gem5 default: VExpress_GEM5_V1 +See the documentation for other values known to work. +''' + ) + + # QEMU. + self.add_argument( + '-Q', '--qemu-build-id', default=consts['default_build_id'], + help='QEMU build ID. Allows you to keep multiple separate QEMU builds.' + ) + + # Userland. + self.add_argument( + '--userland-build-id', default=None + ) + + # Run. + self.add_argument( + '-n', '--run-id', default='0', + help='''\ +ID for run outputs such as gem5's m5out. Allows you to do multiple runs, +and then inspect separate outputs later in different output directories. +''' + ) + self.add_argument( + '-P', '--prebuilt', default=False, action='store_true', + help='''\ +Use prebuilt packaged host utilities as much as possible instead +of the ones we built ourselves. Saves build time, but decreases +the likelihood of incompatibilities. +''' + ) + self.add_argument( + '--port-offset', type=int, + help='''\ +Increase the ports to be used such as for GDB by an offset to run multiple +instances in parallel. Default: the run ID (-n) if that is an integer, otherwise 0. +''' + ) + + # Misc. + self.add_argument( + '-g', '--gem5', default=False, action='store_true', + help='Use gem5 instead of QEMU.' + ) + self.add_argument( + '--qemu', default=False, action='store_true', + help='''\ +Use QEMU as the emulator. This option exists in addition to --gem5 +to allow overriding configs from the CLI. +''' + ) + def main(self, **kwargs): ''' Time the main of the derived class. @@ -132,11 +264,15 @@ mkdir are generally omitted since those are obvious start_time = time.time() kwargs.update(common.consts) kwargs = set_kwargs(kwargs) + self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run']) self.timed_main(**kwargs) if not kwargs['dry_run']: end_time = time.time() common.print_time(end_time - start_time) + def run_cmd(self, *args, **kwargs): + common.run_cmd(*args, **kwargs) + def timed_main(self, **kwargs): raise NotImplementedError() @@ -187,50 +323,6 @@ class BuildCliFunction(LkmcCliFunction): else: self.build(**kwargs) -class Newline: - ''' - Singleton class. Can be used in print_cmd to print out nicer command lines - with --key on the same line as "--key value". - ''' - pass - -def add_args_gem5(parser): - parser.add_argument( - '--gem5-build-dir', - help='''\ -Use the given directory as the gem5 build directory. -''' - ) - parser.add_argument( - '-M', '--gem5-build-id', - help='''\ -gem5 build ID. Allows you to keep multiple separate gem5 builds. -''' - ) - parser.add_argument( - '--gem5-build-type', default='opt', - help='gem5 build type, most often used for "debug" builds.' - ) - parser.add_argument( - '--gem5-source-dir', - help='''\ -Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`. -''' - ) - parser.add_argument( - '-N', '--gem5-worktree', - help='''\ -Create and use a git worktree of the gem5 submodule. -See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree -''' - ) - -def add_newlines(cmd): - out = [] - for arg in cmd: - out.extend([arg, common.Newline]) - return out - def assert_crosstool_ng_supports_arch(arch): if arch not in common.crosstool_ng_supported_archs: raise Exception('arch not yet supported: ' + arch) @@ -238,68 +330,6 @@ def assert_crosstool_ng_supports_arch(arch): def base64_encode(string): return base64.b64encode(string.encode()).decode() -def write_string_to_file(path, string, mode='w'): - if mode == 'a': - redirect = '>>' - else: - redirect = '>' - print_cmd("cat << 'EOF' {} {}\n{}\nEOF".format(redirect, path, string)) - if not common.dry_run: - with open(path, 'a') as f: - f.write(string) - -def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None): - ''' - Format a command given as a list of strings so that it can - be viewed nicely and executed by bash directly and print it to stdout. - ''' - last_newline = ' \\\n' - newline_separator = last_newline + ' ' - out = [] - if extra_env is None: - extra_env = {} - if cwd is not None: - out.append('cd {} &&'.format(shlex.quote(cwd))) - if extra_paths is not None: - out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths))) - for key in extra_env: - out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key]))) - cmd_quote = [] - newline_count = 0 - for arg in cmd: - if arg == common.Newline: - cmd_quote.append(arg) - newline_count += 1 - else: - cmd_quote.append(shlex.quote(arg)) - if newline_count > 0: - cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == common.Newline) if not x] - out.extend(cmd_quote) - if newline_count == 1 and cmd[-1] == common.Newline: - ending = '' - else: - ending = last_newline + ';' - return newline_separator.join(out) + ending - -def copy_dir_if_update_non_recursive(srcdir, destdir, filter_ext=None): - # TODO print rsync equivalent. - os.makedirs(destdir, exist_ok=True) - for basename in os.listdir(srcdir): - src = os.path.join(srcdir, basename) - if os.path.isfile(src): - noext, ext = os.path.splitext(basename) - if filter_ext is not None and ext == filter_ext: - distutils.file_util.copy_file( - src, - os.path.join(destdir, basename), - update=1, - ) - -def cp(src, dest): - print_cmd(['cp', src, dest]) - if not common.dry_run: - shutil.copy2(src, dest) - def gem_list_checkpoint_dirs(): ''' List checkpoint directory, oldest first. @@ -309,101 +339,6 @@ def gem_list_checkpoint_dirs(): files.sort(key=lambda x: os.path.getmtime(os.path.join(common.m5out_dir, x))) return files -def get_argparse(default_args=None): - ''' - Return an argument parser with common arguments set. - - :type default_args: Dict[str,str] - :type argparse_args: Dict - ''' - if default_args is None: - default_args = {} - if argparse_args is None: - argparse_args = {} - emulator_group = parser.add_mutually_exclusive_group(required=False) - parser.add_argument( - '-b', '--baremetal', - help='''\ -Use the given baremetal executable instead of the Linux kernel. - -If the path is absolute, it is used as is. - -If the path is relative, we assume that it points to a source code -inside baremetal/ and then try to use corresponding executable. -''' - ) - parser.add_argument( - '--buildroot-build-id', - default=consts['default_build_id'], - help='Buildroot build ID. Allows you to keep multiple separate gem5 builds.' - ) - parser.add_argument( - '--buildroot-linux', default=False, action='store_true', - help='Boot with the Buildroot Linux kernel instead of our custom built one. Mostly for sanity checks.' - ) - parser.add_argument( - '--crosstool-ng-build-id', default=consts['default_build_id'], - help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds.' - ) - parser.add_argument( - '--docker', default=False, action='store_true', - help='''\ -Use the docker download Ubuntu root filesystem instead of the default Buildroot one. -''' - ) - parser.add_argument( - '-L', '--linux-build-id', default=consts['default_build_id'], - help='Linux build ID. Allows you to keep multiple separate Linux builds.' - ) - parser.add_argument( - '--machine', - help='''Machine type. -QEMU default: virt -gem5 default: VExpress_GEM5_V1 -See the documentation for other values known to work. -''' - ) - emulator_group.add_argument( - '-g', '--gem5', default=False, action='store_true', - help='Use gem5 instead of QEMU.' - ) - parser.add_argument( - '-n', '--run-id', default='0', - help='''\ -ID for run outputs such as gem5's m5out. Allows you to do multiple runs, -and then inspect separate outputs later in different output directories. -''' - ) - parser.add_argument( - '-P', '--prebuilt', default=False, action='store_true', - help='''\ -Use prebuilt packaged host utilities as much as possible instead -of the ones we built ourselves. Saves build time, but decreases -the likelihood of incompatibilities. -''' - ) - parser.add_argument( - '--port-offset', type=int, - help='''\ -Increase the ports to be used such as for GDB by an offset to run multiple -instances in parallel. Default: the run ID (-n) if that is an integer, otherwise 0. -''' - ) - emulator_group.add_argument( - '--qemu', default=False, action='store_true', - help='''\ -Use QEMU as the emulator. This option exists in addition to --gem5 -to allow overriding configs from the CLI. -''' - ) - parser.add_argument( - '-Q', '--qemu-build-id', default=consts['default_build_id'], - help='QEMU build ID. Allows you to keep multiple separate QEMU builds.' - ) - parser.add_argument( - '--userland-build-id', default=None - ) - def get_elf_entry(elf_file_path): readelf_header = subprocess.check_output([ common.get_toolchain_tool('readelf'), @@ -521,29 +456,6 @@ def need_rebuild(srcs, dst): return True return False -def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): - ''' - Print cmd_to_string to stdout. - - Optionally save the command to cmd_file file, and add extra_env - environment variables to the command generated. - - If cmd contains at least one common.Newline, newlines are only added on common.Newline. - Otherwise, newlines are added automatically after every word. - ''' - global dry_run - if type(cmd) is str: - cmd_string = cmd - else: - cmd_string = cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths) - print(common.command_prefix + cmd_string) - if cmd_file is not None: - with open(cmd_file, 'w') as f: - f.write('#!/usr/bin/env bash\n') - f.write(cmd_string) - st = os.stat(cmd_file) - os.chmod(cmd_file, st.st_mode | stat.S_IXUSR) - def print_time(ellapsed_seconds): hours, rem = divmod(ellapsed_seconds, 3600) minutes, seconds = divmod(rem, 60) @@ -589,120 +501,6 @@ def resolve_args(defaults, args, extra_args): argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items())) return argcopy -def rmrf(path): - print_cmd(['rm', '-r', '-f', path]) - if not common.dry_run and os.path.exists(path): - shutil.rmtree(path) - -def run_cmd( - cmd, - cmd_file=None, - out_file=None, - show_stdout=True, - show_cmd=True, - extra_env=None, - extra_paths=None, - delete_env=None, - dry_run=False, - raise_on_failure=True, - **kwargs - ): - ''' - Run a command. Write the command to stdout before running it. - - Wait until the command finishes execution. - - :param cmd: command to run. common.Newline entries are magic get skipped. - :type cmd: List[str] - - :param cmd_file: if not None, write the command to be run to that file - :type cmd_file: str - - :param out_file: if not None, write the stdout and stderr of the command the file - :type out_file: str - - :param show_stdout: wether to show stdout and stderr on the terminal or not - :type show_stdout: bool - - :param extra_env: extra environment variables to add when running the command - :type extra_env: Dict[str,str] - - :param dry_run: don't run the commands, just potentially print them. Debug aid. - :type dry_run: Bool - ''' - if out_file is not None: - stdout = subprocess.PIPE - stderr = subprocess.STDOUT - else: - if show_stdout: - stdout = None - stderr = None - else: - stdout = subprocess.DEVNULL - stderr = subprocess.DEVNULL - if extra_env is None: - extra_env = {} - if delete_env is None: - delete_env = [] - if 'cwd' in kwargs: - cwd = kwargs['cwd'] - else: - cwd = None - env = os.environ.copy() - env.update(extra_env) - if extra_paths is not None: - path = ':'.join(extra_paths) - if 'PATH' in os.environ: - path += ':' + os.environ['PATH'] - env['PATH'] = path - for key in delete_env: - if key in env: - del env[key] - if show_cmd: - print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) - - # Otherwise Ctrl + C gives: - # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). - # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec - sigint_old = signal.getsignal(signal.SIGINT) - signal.signal(signal.SIGINT, signal.SIG_IGN) - - # Otherwise BrokenPipeError when piping through | grep - # But if I do this_module, my terminal gets broken at the end. Why, why, why. - # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python - # Ignoring the exception is not enough as it prints a warning anyways. - #sigpipe_old = signal.getsignal(signal.SIGPIPE) - #signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - cmd = common.strip_newlines(cmd) - if not dry_run and not common.dry_run: - # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 - with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: - if out_file is not None: - os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True) - with open(out_file, 'bw') as logfile: - while True: - byte = proc.stdout.read(1) - if byte: - if show_stdout: - sys.stdout.buffer.write(byte) - try: - sys.stdout.flush() - except BlockingIOError: - # TODO understand. Why, Python, why. - pass - logfile.write(byte) - else: - break - signal.signal(signal.SIGINT, sigint_old) - #signal.signal(signal.SIGPIPE, sigpipe_old) - returncode = proc.returncode - if returncode != 0 and raise_on_failure: - raise Exception('Command exited with status: {}'.format(returncode)) - return returncode - else: - return 0 - def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext): if os.path.isabs(in_path): return in_path @@ -733,13 +531,7 @@ def set_kwargs(kwargs): Update the kwargs from the command line with derived arguments. ''' def join(*paths): - ''' - This is bad, we should just find a nice way to lazily evaluate the kwargs. - ''' - if None in paths: - return None - else: - return os.path.join(*paths) + return os.path.join(*paths) kwargs = collections.defaultdict(lambda: None, **kwargs) if kwargs['qemu'] or not kwargs['gem5']: kwargs['emulator'] = 'qemu' @@ -747,17 +539,12 @@ def set_kwargs(kwargs): kwargs['emulator'] = 'gem5' if kwargs['arch'] in kwargs['arch_short_to_long_dict']: kwargs['arch'] = kwargs['arch_short_to_long_dict'][kwargs['arch']] - if kwargs['gem5_build_id'] is None: - kwargs['gem5_build_id'] = kwargs['default_build_id'] - gem5_build_id_given = False - else: - gem5_build_id_given = True if kwargs['userland_build_id'] is None: kwargs['userland_build_id'] = kwargs['default_build_id'] kwargs['userland_build_id_given'] = False else: kwargs['userland_build_id_given'] = True - if kwargs['gem5_worktree'] is not None and not gem5_build_id_given: + if kwargs['gem5_worktree'] is not None and kwargs['gem5_build_id'] is None: kwargs['gem5_build_id'] = kwargs['gem5_worktree'] kwargs['is_arm'] = False if kwargs['arch'] == 'arm': @@ -799,40 +586,40 @@ def set_kwargs(kwargs): else: if kwargs['machine'] is None: kwargs['machine'] = 'pc' - if 'buildroot_build_id' in kwargs: - kwargs['buildroot_build_dir'] = join(kwargs['buildroot_out_dir'], 'build', kwargs['buildroot_build_id'], kwargs['arch']) - kwargs['buildroot_download_dir'] = join(kwargs['buildroot_out_dir'], 'download') - kwargs['buildroot_config_file'] = join(kwargs['buildroot_build_dir'], '.config') - kwargs['buildroot_build_build_dir'] = join(kwargs['buildroot_build_dir'], 'build') - kwargs['buildroot_linux_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom') - kwargs['buildroot_vmlinux'] = join(kwargs['buildroot_linux_build_dir'], "vmlinux") - kwargs['host_dir'] = join(kwargs['buildroot_build_dir'], 'host') - kwargs['host_bin_dir'] = join(kwargs['host_dir'], 'usr', 'bin') - kwargs['buildroot_pkg_config'] = join(kwargs['host_bin_dir'], 'pkg-config') - kwargs['buildroot_images_dir'] = join(kwargs['buildroot_build_dir'], 'images') - kwargs['buildroot_rootfs_raw_file'] = join(kwargs['buildroot_images_dir'], 'rootfs.ext2') - kwargs['buildroot_qcow2_file'] = kwargs['buildroot_rootfs_raw_file'] + '.qcow2' - kwargs['staging_dir'] = join(kwargs['out_dir'], 'staging', kwargs['arch']) - kwargs['buildroot_staging_dir'] = join(kwargs['buildroot_build_dir'], 'staging') - kwargs['target_dir'] = join(kwargs['buildroot_build_dir'], 'target') - kwargs['linux_buildroot_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom') - if 'qemu_build_id' in kwargs: - kwargs['qemu_build_dir'] = join(kwargs['out_dir'], 'qemu', kwargs['qemu_build_id']) - kwargs['qemu_executable_basename'] = 'qemu-system-{}'.format(kwargs['arch']) - kwargs['qemu_executable'] = join(kwargs['qemu_build_dir'], '{}-softmmu'.format(kwargs['arch']), kwargs['qemu_executable_basename']) - kwargs['qemu_img_basename'] = 'qemu-img' - kwargs['qemu_img_executable'] = join(kwargs['qemu_build_dir'], kwargs['qemu_img_basename']) - kwargs['qemu_termout_file'] = join(kwargs['qemu_run_dir'], 'termout.txt') - # gem5 build - if 'gem5_build_id' in kwargs: - if kwargs['gem5_build_dir'] is None: - kwargs['gem5_build_dir'] = join(kwargs['gem5_out_dir'], kwargs['gem5_build_id'], kwargs['gem5_build_type']) - kwargs['gem5_fake_iso'] = join(kwargs['gem5_out_dir'], 'fake.iso') - kwargs['gem5_m5term'] = join(kwargs['gem5_build_dir'], 'm5term') - kwargs['gem5_build_build_dir'] = join(kwargs['gem5_build_dir'], 'build') - kwargs['gem5_executable'] = join(kwargs['gem5_build_build_dir'], kwargs['gem5_arch'], 'gem5.{}'.format(kwargs['gem5_build_type'])) - kwargs['gem5_system_dir'] = join(kwargs['gem5_build_dir'], 'system') + # Buildroot + kwargs['buildroot_build_dir'] = join(kwargs['buildroot_out_dir'], 'build', kwargs['buildroot_build_id'], kwargs['arch']) + kwargs['buildroot_download_dir'] = join(kwargs['buildroot_out_dir'], 'download') + kwargs['buildroot_config_file'] = join(kwargs['buildroot_build_dir'], '.config') + kwargs['buildroot_build_build_dir'] = join(kwargs['buildroot_build_dir'], 'build') + kwargs['buildroot_linux_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom') + kwargs['buildroot_vmlinux'] = join(kwargs['buildroot_linux_build_dir'], "vmlinux") + kwargs['host_dir'] = join(kwargs['buildroot_build_dir'], 'host') + kwargs['host_bin_dir'] = join(kwargs['host_dir'], 'usr', 'bin') + kwargs['buildroot_pkg_config'] = join(kwargs['host_bin_dir'], 'pkg-config') + kwargs['buildroot_images_dir'] = join(kwargs['buildroot_build_dir'], 'images') + kwargs['buildroot_rootfs_raw_file'] = join(kwargs['buildroot_images_dir'], 'rootfs.ext2') + kwargs['buildroot_qcow2_file'] = kwargs['buildroot_rootfs_raw_file'] + '.qcow2' + kwargs['staging_dir'] = join(kwargs['out_dir'], 'staging', kwargs['arch']) + kwargs['buildroot_staging_dir'] = join(kwargs['buildroot_build_dir'], 'staging') + kwargs['target_dir'] = join(kwargs['buildroot_build_dir'], 'target') + kwargs['linux_buildroot_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom') + + # QEMU + kwargs['qemu_build_dir'] = join(kwargs['out_dir'], 'qemu', kwargs['qemu_build_id']) + kwargs['qemu_executable_basename'] = 'qemu-system-{}'.format(kwargs['arch']) + kwargs['qemu_executable'] = join(kwargs['qemu_build_dir'], '{}-softmmu'.format(kwargs['arch']), kwargs['qemu_executable_basename']) + kwargs['qemu_img_basename'] = 'qemu-img' + kwargs['qemu_img_executable'] = join(kwargs['qemu_build_dir'], kwargs['qemu_img_basename']) + + # gem5 + if kwargs['gem5_build_dir'] is None: + kwargs['gem5_build_dir'] = join(kwargs['gem5_out_dir'], kwargs['gem5_build_id'], kwargs['gem5_build_type']) + kwargs['gem5_fake_iso'] = join(kwargs['gem5_out_dir'], 'fake.iso') + kwargs['gem5_m5term'] = join(kwargs['gem5_build_dir'], 'm5term') + kwargs['gem5_build_build_dir'] = join(kwargs['gem5_build_dir'], 'build') + kwargs['gem5_executable'] = join(kwargs['gem5_build_build_dir'], kwargs['gem5_arch'], 'gem5.{}'.format(kwargs['gem5_build_type'])) + kwargs['gem5_system_dir'] = join(kwargs['gem5_build_dir'], 'system') # gem5 source if kwargs['gem5_source_dir'] is not None: @@ -847,63 +634,63 @@ def set_kwargs(kwargs): kwargs['gem5_se_file'] = join(kwargs['gem5_config_dir'], 'example', 'se.py') kwargs['gem5_fs_file'] = join(kwargs['gem5_config_dir'], 'example', 'fs.py') - if 'crosstool_ng_build_id' in kwargs: - kwargs['crosstool_ng_buildid_dir'] = join(kwargs['crosstool_ng_out_dir'], 'build', kwargs['crosstool_ng_build_id']) - kwargs['crosstool_ng_install_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'install', kwargs['arch']) - kwargs['crosstool_ng_bin_dir'] = join(kwargs['crosstool_ng_install_dir'], 'bin') - kwargs['crosstool_ng_util_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'util') - kwargs['crosstool_ng_config'] = join(kwargs['crosstool_ng_util_dir'], '.config') - kwargs['crosstool_ng_defconfig'] = join(kwargs['crosstool_ng_util_dir'], 'defconfig') - kwargs['crosstool_ng_executable'] = join(kwargs['crosstool_ng_util_dir'], 'ct-ng') - kwargs['crosstool_ng_build_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'build') - kwargs['crosstool_ng_download_dir'] = join(kwargs['crosstool_ng_out_dir'], 'download') - if 'run_id' in kwargs: - kwargs['qemu_guest_terminal_file'] = join(kwargs['m5out_dir'], kwargs['qemu_termout_file']) - kwargs['gem5_run_dir'] = join(kwargs['run_dir_base'], 'gem5', kwargs['arch'], str(kwargs['run_id'])) - kwargs['m5out_dir'] = join(kwargs['gem5_run_dir'], 'm5out') - kwargs['stats_file'] = join(kwargs['m5out_dir'], 'stats.txt') - kwargs['gem5_trace_txt_file'] = join(kwargs['m5out_dir'], 'trace.txt') - kwargs['gem5_guest_terminal_file'] = join(kwargs['m5out_dir'], 'system.terminal') - kwargs['gem5_readfile'] = join(kwargs['gem5_run_dir'], 'readfile') - kwargs['gem5_termout_file'] = join(kwargs['gem5_run_dir'], 'termout.txt') - kwargs['qemu_run_dir'] = join(kwargs['run_dir_base'], 'qemu', kwargs['arch'], str(kwargs['run_id'])) - kwargs['qemu_trace_basename'] = 'trace.bin' - kwargs['qemu_trace_file'] = join(kwargs['qemu_run_dir'], 'trace.bin') - kwargs['qemu_trace_txt_file'] = join(kwargs['qemu_run_dir'], 'trace.txt') - kwargs['qemu_rrfile'] = join(kwargs['qemu_run_dir'], 'rrfile') - kwargs['gem5_out_dir'] = join(kwargs['out_dir'], 'gem5') + # crosstool-ng + kwargs['crosstool_ng_buildid_dir'] = join(kwargs['crosstool_ng_out_dir'], 'build', kwargs['crosstool_ng_build_id']) + kwargs['crosstool_ng_install_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'install', kwargs['arch']) + kwargs['crosstool_ng_bin_dir'] = join(kwargs['crosstool_ng_install_dir'], 'bin') + kwargs['crosstool_ng_util_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'util') + kwargs['crosstool_ng_config'] = join(kwargs['crosstool_ng_util_dir'], '.config') + kwargs['crosstool_ng_defconfig'] = join(kwargs['crosstool_ng_util_dir'], 'defconfig') + kwargs['crosstool_ng_executable'] = join(kwargs['crosstool_ng_util_dir'], 'ct-ng') + kwargs['crosstool_ng_build_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'build') + kwargs['crosstool_ng_download_dir'] = join(kwargs['crosstool_ng_out_dir'], 'download') - # Ports - if kwargs['port_offset'] is None: - try: - kwargs['port_offset'] = int(kwargs['run_id']) - except ValueError: - kwargs['port_offset'] = 0 - if kwargs['emulator'] == 'gem5': - kwargs['gem5_telnet_port'] = 3456 + kwargs['port_offset'] - kwargs['gdb_port'] = 7000 + kwargs['port_offset'] - else: - kwargs['qemu_base_port'] = 45454 + 10 * kwargs['port_offset'] - kwargs['qemu_monitor_port'] = kwargs['qemu_base_port'] + 0 - kwargs['qemu_hostfwd_generic_port'] = kwargs['qemu_base_port'] + 1 - kwargs['qemu_hostfwd_ssh_port'] = kwargs['qemu_base_port'] + 2 - kwargs['qemu_gdb_port'] = kwargs['qemu_base_port'] + 3 - kwargs['extra_serial_port'] = kwargs['qemu_base_port'] + 4 - kwargs['gdb_port'] = kwargs['qemu_gdb_port'] - kwargs['qemu_background_serial_file'] = join(kwargs['qemu_run_dir'], 'background.log') + # run + kwargs['gem5_run_dir'] = join(kwargs['run_dir_base'], 'gem5', kwargs['arch'], str(kwargs['run_id'])) + kwargs['m5out_dir'] = join(kwargs['gem5_run_dir'], 'm5out') + kwargs['stats_file'] = join(kwargs['m5out_dir'], 'stats.txt') + kwargs['gem5_trace_txt_file'] = join(kwargs['m5out_dir'], 'trace.txt') + kwargs['gem5_guest_terminal_file'] = join(kwargs['m5out_dir'], 'system.terminal') + kwargs['gem5_readfile'] = join(kwargs['gem5_run_dir'], 'readfile') + kwargs['gem5_termout_file'] = join(kwargs['gem5_run_dir'], 'termout.txt') + kwargs['qemu_run_dir'] = join(kwargs['run_dir_base'], 'qemu', kwargs['arch'], str(kwargs['run_id'])) + kwargs['qemu_termout_file'] = join(kwargs['qemu_run_dir'], 'termout.txt') + kwargs['qemu_trace_basename'] = 'trace.bin' + kwargs['qemu_trace_file'] = join(kwargs['qemu_run_dir'], 'trace.bin') + kwargs['qemu_trace_txt_file'] = join(kwargs['qemu_run_dir'], 'trace.txt') + kwargs['qemu_rrfile'] = join(kwargs['qemu_run_dir'], 'rrfile') + kwargs['gem5_out_dir'] = join(kwargs['out_dir'], 'gem5') + + # Ports + if kwargs['port_offset'] is None: + try: + kwargs['port_offset'] = int(kwargs['run_id']) + except ValueError: + kwargs['port_offset'] = 0 + if kwargs['emulator'] == 'gem5': + kwargs['gem5_telnet_port'] = 3456 + kwargs['port_offset'] + kwargs['gdb_port'] = 7000 + kwargs['port_offset'] + else: + kwargs['qemu_base_port'] = 45454 + 10 * kwargs['port_offset'] + kwargs['qemu_monitor_port'] = kwargs['qemu_base_port'] + 0 + kwargs['qemu_hostfwd_generic_port'] = kwargs['qemu_base_port'] + 1 + kwargs['qemu_hostfwd_ssh_port'] = kwargs['qemu_base_port'] + 2 + kwargs['qemu_gdb_port'] = kwargs['qemu_base_port'] + 3 + kwargs['extra_serial_port'] = kwargs['qemu_base_port'] + 4 + kwargs['gdb_port'] = kwargs['qemu_gdb_port'] + kwargs['qemu_background_serial_file'] = join(kwargs['qemu_run_dir'], 'background.log') # gem5 QEMU polymorphism. if kwargs['emulator'] == 'gem5': kwargs['executable'] = kwargs['gem5_executable'] kwargs['run_dir'] = kwargs['gem5_run_dir'] kwargs['termout_file'] = kwargs['gem5_termout_file'] - kwargs['guest_terminal_file'] = gem5_guest_terminal_file - kwargs['trace_txt_file'] = gem5_trace_txt_file + kwargs['guest_terminal_file'] = kwargs['gem5_guest_terminal_file'] + kwargs['trace_txt_file'] = kwargs['gem5_trace_txt_file'] else: kwargs['executable'] = kwargs['qemu_executable'] kwargs['run_dir'] = kwargs['qemu_run_dir'] kwargs['termout_file'] = kwargs['qemu_termout_file'] - kwargs['guest_terminal_file'] = kwargs['qemu_guest_terminal_file'] kwargs['trace_txt_file'] = kwargs['qemu_trace_txt_file'] kwargs['run_cmd_file'] = join(kwargs['run_dir'], 'run.sh') @@ -921,7 +708,7 @@ def set_kwargs(kwargs): kwargs['linux_arch'] = 'x86' kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'bzImage') kwargs['lkmc_linux_image'] = join(kwargs['linux_build_dir'], kwargs['linux_image_prefix']) - kwargs['buildroot_linux_image'] = join(kwargs['buildroot_linux_build_dir'], linux_image_prefix) + kwargs['buildroot_linux_image'] = join(kwargs['buildroot_linux_build_dir'], kwargs['linux_image_prefix']) if kwargs['buildroot_linux']: kwargs['vmlinux'] = kwargs['buildroot_vmlinux'] kwargs['linux_image'] = kwargs['buildroot_linux_image'] @@ -992,31 +779,3 @@ def set_kwargs(kwargs): break kwargs['image'] = path return dict(kwargs) - -def shlex_split(string): - ''' - shlex_split, but also add Newline after every word. - - Not perfect since it does not group arguments, but I don't see a solution. - ''' - return common.add_newlines(shlex.split(string)) - -def strip_newlines(cmd): - return [x for x in cmd if x != common.Newline] - -def write_configs(config_path, configs, config_fragments=None): - """ - Write extra configs into the Buildroot config file. - TODO Can't get rid of these for now with nice fragments: - http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config - """ - if config_fragments is None: - config_fragments = [] - with open(config_path, 'a') as config_file: - for config_fragment in config_fragments: - with open(config_fragment, 'r') as config_fragment_file: - print_cmd(['cat', config_fragment, '>>', config_path]) - if not common.dry_run: - for line in config_fragment_file: - config_file.write(line) - write_string_to_file(config_path, '\n'.join(configs), mode='a') diff --git a/shell_helpers.py b/shell_helpers.py new file mode 100644 index 0000000..da08627 --- /dev/null +++ b/shell_helpers.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 + +import itertools +import os +import shlex +import shutil +import signal + +class LF: + ''' + LineFeed (AKA newline). + + Singleton class. Can be used in print_cmd to print out nicer command lines + with --key on the same line as "--key value". + ''' + pass + +class ShellHelpers: + ''' + Helpers to do things which are easy from the shell, + usually filesystem, process or pipe operations. + + Attempt to print shell equivalents of all commands to make things + easy to debug and understand what is going on. + ''' + def __init__(self, dry_run=False): + ''' + :param dry_run: don't run the commands, just potentially print them. Debug aid. + :type dry_run: Bool + ''' + self.dry_run = dry_run + + def add_newlines(self, cmd): + out = [] + for arg in cmd: + out.extend([arg, LF]) + return out + + def cp(self, src, dest, **kwargs): + self.print_cmd(['cp', src, dest]) + if not self.dry_run: + shutil.copy2(src, dest) + + def cmd_to_string(self, cmd, cwd=None, extra_env=None, extra_paths=None): + ''' + Format a command given as a list of strings so that it can + be viewed nicely and executed by bash directly and print it to stdout. + ''' + last_newline = ' \\\n' + newline_separator = last_newline + ' ' + out = [] + if extra_env is None: + extra_env = {} + if cwd is not None: + out.append('cd {} &&'.format(shlex.quote(cwd))) + if extra_paths is not None: + out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths))) + for key in extra_env: + out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key]))) + cmd_quote = [] + newline_count = 0 + for arg in cmd: + if arg == LF: + cmd_quote.append(arg) + newline_count += 1 + else: + cmd_quote.append(shlex.quote(arg)) + if newline_count > 0: + cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == LF) if not x] + out.extend(cmd_quote) + if newline_count == 1 and cmd[-1] == LF: + ending = '' + else: + ending = last_newline + ';' + return newline_separator.join(out) + ending + + def copy_dir_if_update_non_recursive(self, srcdir, destdir, filter_ext=None): + # TODO print rsync equivalent. + os.makedirs(destdir, exist_ok=True) + for basename in os.listdir(srcdir): + src = os.path.join(srcdir, basename) + if os.path.isfile(src): + noext, ext = os.path.splitext(basename) + if filter_ext is not None and ext == filter_ext: + distutils.file_util.copy_file( + src, + os.path.join(destdir, basename), + update=1, + ) + + def print_cmd(self, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): + ''' + Print cmd_to_string to stdout. + + Optionally save the command to cmd_file file, and add extra_env + environment variables to the command generated. + + If cmd contains at least one LF, newlines are only added on common.Newline. + Otherwise, newlines are added automatically after every word. + ''' + if type(cmd) is str: + cmd_string = cmd + else: + cmd_string = self.cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths) + print('+ ' + cmd_string) + if cmd_file is not None: + with open(cmd_file, 'w') as f: + f.write('#!/usr/bin/env bash\n') + f.write(cmd_string) + st = os.stat(cmd_file) + os.chmod(cmd_file, st.st_mode | stat.S_IXUSR) + + def run_cmd( + self, + cmd, + cmd_file=None, + out_file=None, + show_stdout=True, + show_cmd=True, + extra_env=None, + extra_paths=None, + delete_env=None, + raise_on_failure=True, + **kwargs + ): + ''' + Run a command. Write the command to stdout before running it. + + Wait until the command finishes execution. + + :param cmd: command to run. LF entries are magic get skipped. + :type cmd: List[str] + + :param cmd_file: if not None, write the command to be run to that file + :type cmd_file: str + + :param out_file: if not None, write the stdout and stderr of the command the file + :type out_file: str + + :param show_stdout: wether to show stdout and stderr on the terminal or not + :type show_stdout: bool + + :param extra_env: extra environment variables to add when running the command + :type extra_env: Dict[str,str] + ''' + if out_file is not None: + stdout = subprocess.PIPE + stderr = subprocess.STDOUT + else: + if show_stdout: + stdout = None + stderr = None + else: + stdout = subprocess.DEVNULL + stderr = subprocess.DEVNULL + if extra_env is None: + extra_env = {} + if delete_env is None: + delete_env = [] + if 'cwd' in kwargs: + cwd = kwargs['cwd'] + else: + cwd = None + env = os.environ.copy() + env.update(extra_env) + if extra_paths is not None: + path = ':'.join(extra_paths) + if 'PATH' in os.environ: + path += ':' + os.environ['PATH'] + env['PATH'] = path + for key in delete_env: + if key in env: + del env[key] + if show_cmd: + self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) + + # Otherwise Ctrl + C gives: + # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). + # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec + sigint_old = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, signal.SIG_IGN) + + # Otherwise BrokenPipeError when piping through | grep + # But if I do this_module, my terminal gets broken at the end. Why, why, why. + # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python + # Ignoring the exception is not enough as it prints a warning anyways. + #sigpipe_old = signal.getsignal(signal.SIGPIPE) + #signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + cmd = self.strip_newlines(cmd) + if not self.dry_run: + # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 + with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: + if out_file is not None: + os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True) + with open(out_file, 'bw') as logfile: + while True: + byte = proc.stdout.read(1) + if byte: + if show_stdout: + sys.stdout.buffer.write(byte) + try: + sys.stdout.flush() + except BlockingIOError: + # TODO understand. Why, Python, why. + pass + logfile.write(byte) + else: + break + signal.signal(signal.SIGINT, sigint_old) + #signal.signal(signal.SIGPIPE, sigpipe_old) + returncode = proc.returncode + if returncode != 0 and raise_on_failure: + raise Exception('Command exited with status: {}'.format(returncode)) + return returncode + else: + return 0 + + def shlex_split(self, string): + ''' + shlex_split, but also add Newline after every word. + + Not perfect since it does not group arguments, but I don't see a solution. + ''' + return self.add_newlines(shlex.split(string)) + + def strip_newlines(self, cmd): + return [x for x in cmd if x != LF] + + def rmrf(path): + self.print_cmd(['rm', '-r', '-f', path]) + if not self.dry_run and os.path.exists(path): + shutil.rmtree(path) + + def write_configs(self, config_path, configs, config_fragments=None): + ''' + Write extra KEY=val configs into the given config file. + ''' + if config_fragments is None: + config_fragments = [] + with open(config_path, 'a') as config_file: + for config_fragment in config_fragments: + with open(config_fragment, 'r') as config_fragment_file: + self.print_cmd(['cat', config_fragment, '>>', config_path]) + if not self.dry_run: + for line in config_fragment_file: + config_file.write(line) + write_string_to_file(config_path, '\n'.join(configs), mode='a') + + def write_string_to_file(self, path, string, mode='w'): + if mode == 'a': + redirect = '>>' + else: + redirect = '>' + self.print_cmd("cat << 'EOF' {} {}\n{}\nEOF".format(redirect, path, string)) + if not self.dry_run: + with open(path, 'a') as f: + f.write(string)