manually encode newlines on all printed commands

This way we group key value arguments: e.g.:

    make \
    -j 8 \
    all

instead of:

    make \
    -j \
    8 \
    all

and reach CLI nirvana, while also subtly breaking several commands due to
lack of testing.

build

@@ -76,6 +76,9 @@ name_to_component_map = {
     'qemu': Component(
         lambda arch: run_cmd(['build-qemu'], arch),
     ),
+    'qemu-user': Component(
+        lambda arch: run_cmd(['build-qemu', '--userland'], arch),
+    ),
     'userland': Component(
         lambda arch: run_cmd(['build-userland'], arch),
     ),
@@ -85,6 +88,7 @@ name_to_component_map = {
         'qemu-gem5-buildroot',
         'gem5-debug',
         'gem5-fast',
+        'qemu-user',
     ]),
     'gem5-buildroot': Component(dependencies=[
         'buildroot-gcc',

build-baremetal

@@ -11,10 +11,10 @@ class BaremetalComponent(common.Component):
         bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext))
         common_obj = os.path.join(common.baremetal_build_lib_dir, 'common{}'.format(common.obj_ext))
         cflags = [
-            '-ggdb3',
-            '-mcpu={}'.format(common.mcpu),
-            '-nostartfiles',
-            '-O0',
+            '-ggdb3', common.Newline,
+            '-mcpu={}'.format(common.mcpu), common.Newline,
+            '-nostartfiles', common.Newline,
+            '-O0', common.Newline,
         ]
         if args.prebuilt:
             gcc = 'arm-none-eabi-gcc'
@@ -36,23 +36,23 @@ class BaremetalComponent(common.Component):
         os.makedirs(build_dir, exist_ok=True)
         os.makedirs(common.baremetal_build_lib_dir, exist_ok=True)
         common.run_cmd(
-            [gcc] +
+            [gcc,  common.Newline] +
             cflags +
             [
-                '-c',
-                '-o', bootloader_obj,
-                os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)),
+                '-c', common.Newline,
+                '-o', bootloader_obj, common.Newline,
+                os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)), common.Newline,
             ]
         )
         common.run_cmd(
-            [gcc] +
+            [gcc,  common.Newline] +
             cflags +
             [
-                '-c',
-                '-D',
-                'UART0_ADDR={:#x}'.format(uart_address),
-                '-o', common_obj,
-                os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext),
+                '-c', common.Newline,
+                '-D', common.Newline,
+                'UART0_ADDR={:#x}'.format(uart_address), common.Newline,
+                '-o', common_obj, common.Newline,
+                os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext), common.Newline,
             ]
         )
         self._build_dir(
@@ -117,7 +117,7 @@ Build the baremetal examples with crosstool-NG.
         out_dir = os.path.join(common.baremetal_build_dir, subpath)
         os.makedirs(out_dir, exist_ok=True)
         if bootloader:
-            bootloader_cmd = [bootloader_obj]
+            bootloader_cmd = [bootloader_obj, common.Newline]
         else:
             bootloader_cmd = []
         for in_basename in os.listdir(in_dir):
@@ -126,26 +126,26 @@ Build the baremetal examples with crosstool-NG.
                 in_name = os.path.splitext(in_basename)[0]
                 main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext))
                 common.run_cmd(
-                    [gcc] +
+                    [gcc,  common.Newline] +
                     cflags +
                     [
-                        '-c',
-                        '-o', main_obj,
-                        os.path.join(common.baremetal_src_dir, in_path),
+                        '-c', common.Newline,
+                        '-o', main_obj, common.Newline,
+                        os.path.join(common.baremetal_src_dir, in_path), common.Newline,
                     ]
                 )
                 common.run_cmd(
-                    [gcc] +
+                    [gcc,  common.Newline] +
                     cflags +
                     [
-                        '-Wl,--section-start=.text={:#x}'.format(entry_address),
-                        '-o', os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext),
-                        '-T', os.path.join(common.baremetal_src_dir, 'link.ld'),
+                        '-Wl,--section-start=.text={:#x}'.format(entry_address), common.Newline,
+                        '-o', os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext), common.Newline,
+                        '-T', os.path.join(common.baremetal_src_dir, 'link.ld'), common.Newline,
                     ] +
                     bootloader_cmd +
                     [
-                        common_obj,
-                        main_obj,
+                        common_obj, common.Newline,
+                        main_obj, common.Newline,
                     ]
                 )
 

build-buildroot

@@ -59,11 +59,11 @@ usually extra Buildroot targets.
     def do_build(self, args):
         build_dir = self.get_build_dir(args)
         os.makedirs(common.out_dir, exist_ok=True)
-        extra_make_args = args.extra_make_args.copy()
+        extra_make_args = common.add_newlines(args.extra_make_args)
         if args.build_linux:
-            extra_make_args.append('linux-reconfigure')
+            extra_make_args.extend(['linux-reconfigure', common.Newline])
         if args.gem5:
-            extra_make_args.append('gem5-reconfigure')
+            extra_make_args.extend(['gem5-reconfigure', common.Newline])
         if args.arch == 'x86_64':
             defconfig = 'qemu_x86_64_defconfig'
         elif args.arch == 'arm':
@@ -78,10 +78,10 @@ usually extra Buildroot targets.
         br2_external_str = ':'.join(br2_external_dirs)
         common.run_cmd(
             [
-                'make',
-                'O={}'.format(common.buildroot_build_dir),
-                'BR2_EXTERNAL={}'.format(br2_external_str),
-                defconfig,
+                'make', common.Newline,
+                'O={}'.format(common.buildroot_build_dir), common.Newline,
+                'BR2_EXTERNAL={}'.format(br2_external_str), common.Newline,
+                defconfig, common.Newline,
             ],
             cwd=common.buildroot_src_dir,
         )
@@ -122,22 +122,22 @@ usually extra Buildroot targets.
         common.write_configs(common.buildroot_config_file, configs, config_fragments)
         common.run_cmd(
             [
-                'make',
-                'O={}'.format(common.buildroot_build_dir),
-                'olddefconfig',
+                'make', common.Newline,
+                'O={}'.format(common.buildroot_build_dir), common.Newline,
+                'olddefconfig', common.Newline,
             ],
             cwd=common.buildroot_src_dir,
         )
         common.make_build_dirs()
         if not args.no_all:
-            extra_make_args.append('all')
+            extra_make_args.extend(['all', common.Newline])
         common.run_cmd(
             [
-                'make',
-                'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_src_dir),
-                'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir),
-                'O={}'.format(common.buildroot_build_dir),
-                'V={}'.format(int(args.verbose)),
+                'make', common.Newline,
+                'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_src_dir), common.Newline,
+                'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), common.Newline,
+                'O={}'.format(common.buildroot_build_dir), common.Newline,
+                'V={}'.format(int(args.verbose)), common.Newline,
             ] +
             extra_make_args
             ,

build-crosstool-ng

@@ -16,19 +16,19 @@ class CrosstoolNgComponent(common.Component):
         # https://github.com/crosstool-ng/crosstool-ng/issues/1021
         os.chdir(common.crosstool_ng_src_dir)
         common.run_cmd(
-            [os.path.join(common.crosstool_ng_src_dir, 'bootstrap')],
+            [os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), common.Newline],
         )
         os.chdir(common.crosstool_ng_util_dir)
         common.run_cmd(
             [
-                os.path.join(common.crosstool_ng_src_dir, 'configure'),
-                '--enable-local',
+                os.path.join(common.crosstool_ng_src_dir, 'configure'), common.Newline,
+                '--enable-local', common.Newline,
             ],
         )
         common.run_cmd(
             [
-                'make',
-                '-j', str(args.nproc),
+                'make', common.Newline,
+                '-j', str(args.nproc), common.Newline,
             ],
         )
 
@@ -47,16 +47,16 @@ class CrosstoolNgComponent(common.Component):
         )
         common.run_cmd(
             [
-                common.crosstool_ng_executable,
-                'defconfig',
+                common.crosstool_ng_executable, common.Newline,
+                'defconfig', common.Newline,
             ],
         )
         os.unlink(defconfig_dest)
         common.run_cmd(
             [
-                common.crosstool_ng_executable,
-                'build',
-                'CT_JOBS={}'.format(str(args.nproc)),
+                common.crosstool_ng_executable, common.Newline,
+                'build', common.Newline,
+                'CT_JOBS={}'.format(str(args.nproc)), common.Newline,
             ],
             out_file=os.path.join(build_dir, 'lkmc.log'),
             delete_env=['LD_LIBRARY_PATH'],

build-gem5

@@ -26,14 +26,14 @@ class Gem5Component(common.Component):
                 if common.gem5_src_dir == common.gem5_default_src_dir:
                     raise Exception('gem5 submodule not checked out')
                 common.run_cmd([
-                    'git',
-                    '-C', common.gem5_default_src_dir,
-                    'worktree', 'add',
-                    '-b', os.path.join('wt', args.gem5_build_id),
-                    common.gem5_src_dir
+                    'git', common.Newline,
+                    '-C', common.gem5_default_src_dir, common.Newline,
+                    'worktree', 'add', common.Newline,
+                    '-b', os.path.join('wt', args.gem5_build_id), common.Newline,
+                    common.gem5_src_dir, common.Newline,
                 ])
         if args.verbose:
-            verbose = ['--verbose']
+            verbose = ['--verbose', common.Newline]
         else:
             verbose = []
         if args.arch == 'x86_64':
@@ -53,7 +53,10 @@ class Gem5Component(common.Component):
             # dtb
             dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt')
             dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt')
-            common.run_cmd(['make', '-C', dt_src_dir])
+            common.run_cmd([
+                'make', common.Newline,
+                '-C', dt_src_dir, common.Newline,
+            ])
             common.copy_dir_if_update_non_recursive(
                 srcdir=dt_src_dir,
                 destdir=dt_build_dir,
@@ -63,25 +66,31 @@ class Gem5Component(common.Component):
             # Bootloader 32.
             bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader')
             # TODO use the buildroot cross compiler here, and remove the dependencies from configure.
-            common.run_cmd(['make', '-C', bootloader32_dir])
+            common.run_cmd([
+                'make', common.Newline,
+                '-C', bootloader32_dir, common.Newline,
+            ])
             # bootloader
             common.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir)
 
             # Bootloader 64.
             bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader')
             # TODO cross_compile is ignored because the make does not use CC...
-            common.run_cmd(['make', '-C', bootloader64_dir])
+            common.run_cmd([
+                'make', common.Newline,
+                '-C', bootloader64_dir, common.Newline
+            ])
             common.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir)
         common.run_cmd(
             (
                 [
-                    'scons',
-                    '-j', str(args.nproc),
-                    '--ignore-style',
-                    common.gem5_executable
+                    'scons', common.Newline,
+                    '-j', str(args.nproc), common.Newline,
+                    '--ignore-style', common.Newline,
+                    common.gem5_executable, common.Newline,
                 ] +
                 verbose +
-                args.extra_scons_args
+                common.add_newlines(args.extra_scons_args)
             ),
             cwd=common.gem5_src_dir,
             extra_paths=[common.ccache_dir],

build-linux

@@ -66,12 +66,12 @@ Configure the kernel, but don't build it.
         else:
             cc = gcc
         common_make_args = [
-            'make',
-            '-j', str(args.nproc),
-            'ARCH={}'.format(common.linux_arch),
-            'CROSS_COMPILE={}'.format(prefix),
-            'CC={}'.format(cc),
-            'O={}'.format(build_dir),
+            'make', common.Newline,
+            '-j', str(args.nproc), common.Newline,
+            'ARCH={}'.format(common.linux_arch), common.Newline,
+            'CROSS_COMPILE={}'.format(prefix), common.Newline,
+            'CC={}'.format(cc), common.Newline,
+            'O={}'.format(build_dir), common.Newline,
         ]
         if args.verbose:
             verbose = ['V=1']
@@ -99,17 +99,17 @@ Configure the kernel, but don't build it.
         )
         common.run_cmd(
             [
-                os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'),
-                '-m',
-                '-O', build_dir,
-                os.path.join(build_dir, '.config'),
+                os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), common.Newline,
+                '-m', common.Newline,
+                '-O', build_dir, common.Newline,
+                os.path.join(build_dir, '.config'), common.Newline,
             ] +
             config_fragments
         )
         common.run_cmd(
             (
                 common_make_args +
-                ['olddefconfig']
+                ['olddefconfig', common.Newline]
             ),
             **common_args
         )
@@ -117,7 +117,7 @@ Configure the kernel, but don't build it.
             common.run_cmd(
                 (
                     common_make_args +
-                    args.extra_make_args
+                    common.add_newlines(args.extra_make_args)
                 ),
                 **common_args
             )

build-m5

@@ -14,12 +14,12 @@ class M5Component(common.Component):
         else:
             arch = args.arch
         return [
-            'make',
-            '-j', str(args.nproc),
-            '-f', 'Makefile.{}'.format(arch),
-            'CC={}'.format(cc),
-            'LD={}'.format(ld),
-            'PWD={}'.format(common.gem5_m5_src_dir),
+            'make', common.Newline,
+            '-j', str(args.nproc), common.Newline,
+            '-f', 'Makefile.{}'.format(arch), common.Newline,
+            'CC={}'.format(cc), common.Newline,
+            'LD={}'.format(ld), common.Newline,
+            'PWD={}'.format(common.gem5_m5_src_dir), common.Newline,
         ]
 
     def do_build(self, args):
@@ -36,7 +36,7 @@ class M5Component(common.Component):
 
     def clean(self, args):
         common.run_cmd(
-            self.get_make_cmd(args) + ['clean'],
+            self.get_make_cmd(args) + ['clean', common.Newline],
             cwd=common.gem5_m5_src_dir,
         )
 

build-modules

@@ -3,7 +3,6 @@
 import distutils.dir_util
 import os
 import platform
-import shlex
 import shutil
 
 import common
@@ -90,16 +89,16 @@ Use the host packaged cross toolchain.
         common.run_cmd(
             (
                 [
-                    'make',
-                    '-j', str(args.nproc),
-                    'ARCH={}'.format(common.linux_arch),
-                    'CC={}'.format(cc),
-                    'CROSS_COMPILE={}'.format(prefix),
-                    'LINUX_DIR={}'.format(linux_dir),
-                    'M={}'.format(build_subdir),
-                    'OBJECT_FILES={}'.format(' '.join(object_files)),
+                    'make', common.Newline,
+                    '-j', str(args.nproc), common.Newline,
+                    'ARCH={}'.format(common.linux_arch), common.Newline,
+                    'CC={}'.format(cc), common.Newline,
+                    'CROSS_COMPILE={}'.format(prefix), common.Newline,
+                    'LINUX_DIR={}'.format(linux_dir), common.Newline,
+                    'M={}'.format(build_subdir), common.Newline,
+                    'OBJECT_FILES={}'.format(' '.join(object_files)), common.Newline,
                 ] +
-                shlex.split(args.make_args) +
+                common.shlex_split(args.make_args) +
                 verbose
             ),
             cwd=os.path.join(common.kernel_modules_build_subdir),

build-qemu

@@ -32,22 +32,22 @@ class QemuComponent(common.Component):
             target_list = '{}-softmmu'.format(args.arch)
         common.run_cmd(
             [
-                os.path.join(common.qemu_src_dir, 'configure'),
-                '--enable-debug',
-                '--enable-trace-backends=simple',
-                '--target-list={}'.format(target_list),
-                '--enable-sdl',
-                '--with-sdlabi=2.0',
+                os.path.join(common.qemu_src_dir, 'configure'), common.Newline,
+                '--enable-debug', common.Newline,
+                '--enable-trace-backends=simple', common.Newline,
+                '--target-list={}'.format(target_list), common.Newline,
+                '--enable-sdl', common.Newline,
+                '--with-sdlabi=2.0', common.Newline,
             ] +
-            args.extra_config_args,
+            common.add_newlines(args.extra_config_args),
             extra_paths=[common.ccache_dir],
             cwd=build_dir
         )
         common.run_cmd(
             (
                 [
-                    'make',
-                    '-j', str(args.nproc),
+                    'make', common.Newline,
+                    '-j', str(args.nproc), common.Newline,
 
                 ] +
                 verbose

build-userland

@@ -56,17 +56,17 @@ has the OpenBLAS libraries and headers installed.
         common.run_cmd(
             (
                 [
-                    'make',
-                    '-j', str(args.nproc),
-                    'CC={}'.format(cc),
-                    'CXX={}'.format(cxx),
-                    'PKG_CONFIG={}'.format(common.buildroot_pkg_config),
-                    'STAGING_DIR={}'.format(common.buildroot_staging_dir),
-                    'OUT_DIR={}'.format(build_dir),
+                    'make', common.Newline,
+                    '-j', str(args.nproc), common.Newline,
+                    'CC={}'.format(cc), common.Newline,
+                    'CXX={}'.format(cxx), common.Newline,
+                    'PKG_CONFIG={}'.format(common.buildroot_pkg_config), common.Newline,
+                    'STAGING_DIR={}'.format(common.buildroot_staging_dir), common.Newline,
+                    'OUT_DIR={}'.format(build_dir), common.Newline,
                 ] +
-                ['HAS_{}=y'.format(package.upper()) for package in args.has_package] +
+                common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in args.has_package]) +
                 shlex.split(args.make_args) +
-                [os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets]
+                common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets])
             ),
             cwd=common.userland_src_dir,
             extra_paths=[common.ccache_dir],

common.py

@@ -8,6 +8,7 @@ import datetime
 import distutils.file_util
 import glob
 import imp
+import itertools
 import json
 import multiprocessing
 import os
@@ -144,6 +145,13 @@ class Component:
         '''
         return {}
 
+class Newline:
+    '''
+    Singleton class. Can be used in print_cmd to print out nicer command lines
+    with -key on the same line as "-key value".
+    '''
+    pass
+
 def add_dry_run_argument(parser):
     parser.add_argument('--dry-run', default=False, action='store_true', help='''\
 Print the commands that would be run, but don't run them.
@@ -154,6 +162,12 @@ Bash equivalents even for actions taken directly in Python without shelling out.
 mkdir are generally omitted since those are obvious.
 ''')
 
+def add_newlines(cmd):
+    out = []
+    for arg in cmd:
+        out.extend([arg, this_module.Newline])
+    return out
+
 def base64_encode(string):
     return base64.b64encode(string.encode()).decode()
 
@@ -456,19 +470,29 @@ def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None):
     Format a command given as a list of strings so that it can
     be viewed nicely and executed by bash directly and print it to stdout.
     '''
-    newline_separator = ' \\\n'
+    last_newline = ' \\\n'
+    newline_separator = last_newline + '  '
     out = []
     if extra_env is None:
         extra_env = {}
     if cwd is not None:
-        out.append('cd {} &&{}'.format(shlex.quote(cwd), newline_separator))
+        out.append('cd {} &&'.format(shlex.quote(cwd)))
     if extra_paths is not None:
-        out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)) + newline_separator)
+        out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)))
     for key in extra_env:
-        out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])) + newline_separator)
+        out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])))
+    cmd_quote = []
+    has_newline = False
     for arg in cmd:
-        out.append(shlex.quote(arg) + newline_separator)
-    return '  '.join(out) + ';'
+        if arg == this_module.Newline:
+            cmd_quote.append(arg)
+            has_newline = True
+        else:
+            cmd_quote.append(shlex.quote(arg))
+    if has_newline:
+        cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == this_module.Newline) if not x]
+    out.extend(cmd_quote)
+    return newline_separator.join(out) + last_newline + ';'
 
 def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
     '''
@@ -476,6 +500,9 @@ def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
 
     Optionally save the command to cmd_file file, and add extra_env
     environment variables to the command generated.
+
+    If cmd contains at least one common.Newline, newlines are only added on common.Newline.
+    Otherwise, newlines are added automatically after every word.
     '''
     global dry_run
     if type(cmd) is str:
@@ -513,14 +540,14 @@ def raw_to_qcow2(prebuilt=False, reverse=False):
         infile = outfile
         outfile = tmp
     this_module.run_cmd([
-        qemu_img_executable,
+        qemu_img_executable, this_module.Newline,
         # Prevent qemu-img from generating trace files like QEMU. Disgusting.
-        '-T', 'pr_manager_run,file=/dev/null',
-        'convert',
-        '-f', infmt,
-        '-O', outfmt,
-        infile,
-        outfile,
+        '-T', 'pr_manager_run,file=/dev/null', this_module.Newline,
+        'convert', this_module.Newline,
+        '-f', infmt, this_module.Newline,
+        '-O', outfmt, this_module.Newline,
+        infile, this_module.Newline,
+        outfile, this_module.Newline,
     ])
 
 def raise_no_x86(arch):
@@ -563,7 +590,7 @@ def run_cmd(
 
     Wait until the command finishes execution.
 
-    :param cmd: command to run
+    :param cmd: command to run. common.Newline entries are magic get skipped.
     :type cmd: List[str]
 
     :param cmd_file: if not None, write the command to be run to that file
@@ -625,6 +652,7 @@ def run_cmd(
     #sigpipe_old = signal.getsignal(signal.SIGPIPE)
     #signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
+    cmd = [x for x in cmd if x != this_module.Newline]
     if not dry_run and not this_module.dry_run:
         # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
         with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
@@ -886,6 +914,14 @@ def setup(parser):
 def setup_dry_run_arguments(args):
     this_module.dry_run = args.dry_run
 
+def shlex_split(string):
+    '''
+    shlex_split, but also add Newline after every word.
+
+    Not perfect since it does not group arguments, but I don't see a solution.
+    '''
+    return this_module.add_newlines(shlex.split(string))
+
 def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext):
     if os.path.isabs(in_path):
         return in_path

gem5-shell

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import subprocess
 import sys
 
 import common
@@ -10,4 +9,8 @@ parser = common.get_argparse(
     argparse_args={'description':'Connect a terminal to a running gem5 instance'}
 )
 args = common.setup(parser)
-sys.exit(common.run_cmd([str(common.gem5_m5term), 'localhost', str(common.gem5_telnet_port)]))
+sys.exit(common.run_cmd([
+    common.gem5_m5term, common.Newline,
+    'localhost', common.Newline,
+    str(common.gem5_telnet_port), common.Newline,
+]))

qemu-trace2txt

@@ -7,13 +7,12 @@ import sys
 import common
 
 def main():
-    cmd = [
-        os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'),
-        os.path.join(common.qemu_build_dir, 'trace-events-all'),
-        os.path.join(common.qemu_trace_file),
-    ]
     return common.run_cmd(
-        cmd,
+        [
+            os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), common.Newline,
+            os.path.join(common.qemu_build_dir, 'trace-events-all'), common.Newline,
+            os.path.join(common.qemu_trace_file), common.Newline,
+        ],
         cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'),
         out_file=common.qemu_trace_txt_file,
         show_stdout=False,

run

@@ -2,7 +2,6 @@
 
 import os
 import re
-import shlex
 import shutil
 import subprocess
 import sys
@@ -55,18 +54,17 @@ def main(args, extra_args=None):
     extra_emulator_args = []
     extra_qemu_args = []
     if args.debug_vm is not None:
-        print(args.debug_vm)
-        debug_vm = ['gdb', '-q'] + shlex.split(args.debug_vm) + ['--args']
+        debug_vm = ['gdb', common.Newline, '-q', common.Newline] + common.shlex_split(args.debug_vm) + ['--args', common.Newline]
     else:
         debug_vm = []
     if args.wait_gdb:
-        extra_qemu_args.append('-S')
+        extra_qemu_args.extend(['-S', common.Newline])
     if args.eval_busybox is not None:
         kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(common.base64_encode(args.eval_busybox))
     if args.kernel_cli_after_dash is not None:
         kernel_cli_after_dash += ' {}'.format(args.kernel_cli_after_dash)
     if args.vnc:
-        vnc = ['-vnc', ':0']
+        vnc = ['-vnc', ':0', common.Newline]
     else:
         vnc = []
     if args.initrd or args.initramfs:
@@ -81,7 +79,7 @@ def main(args, extra_args=None):
         kernel_cli += ' {}=/eval_base64.sh'.format(initarg)
         kernel_cli_after_dash += ' lkmc_eval="{}"'.format(common.base64_encode(args.eval))
     if not args.graphic:
-        extra_qemu_args.append('-nographic')
+        extra_qemu_args.extend(['-nographic', common.Newline])
     console = None
     console_type = None
     console_count = 0
@@ -144,24 +142,27 @@ def main(args, extra_args=None):
         os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True)
         common.write_string_to_file(common.gem5_readfile, args.gem5_readfile)
         memory = '{}B'.format(args.memory)
-        gem5_exe_args = shlex.split(args.gem5_exe_args)
+        gem5_exe_args = common.shlex_split(args.gem5_exe_args)
         if do_trace:
-            gem5_exe_args.append('--debug-flags={}'.format(trace_type))
+            gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), common.Newline])
         extra_env['M5_PATH'] = common.gem5_system_dir
         # https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071
         extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true'
         cmd.extend(
             [
-                common.executable,
-                '--debug-file=trace.txt',
+                common.executable, common.Newline,
+                '--debug-file=trace.txt', common.Newline
             ] +
             gem5_exe_args +
             [
-                '-d', common.m5out_dir
+                '-d', common.m5out_dir, common.Newline
             ]
         )
         if args.userland is not None:
-            cmd.extend([common.gem5_se_file, '-c', common.resolve_userland(args.userland)])
+            cmd.extend([
+                common.gem5_se_file, common.Newline,
+                '-c', common.resolve_userland(args.userland), common.Newline,
+            ])
         else:
             if args.gem5_script == 'fs':
                 # TODO port
@@ -170,67 +171,67 @@ def main(args, extra_args=None):
                     cpt_dir = cpt_dirs[-args.gem5_restore]
                     extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)])
                 cmd.extend([
-                    common.gem5_fs_file,
-                    '--disk-image', common.disk_image,
-                    '--kernel', common.image,
-                    '--mem-size', memory,
-                    '--num-cpus', str(args.cpus),
-                    '--script', common.gem5_readfile,
+                    common.gem5_fs_file, common.Newline,
+                    '--disk-image', common.disk_image, common.Newline,
+                    '--kernel', common.image, common.Newline,
+                    '--mem-size', memory, common.Newline,
+                    '--num-cpus', str(args.cpus), common.Newline,
+                    '--script', common.gem5_readfile, common.Newline,
                 ])
                 if args.arch == 'x86_64':
                     if args.kvm:
-                        cmd.extend(['--cpu-type', 'X86KvmCPU'])
-                    cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli)])
+                        cmd.extend(['--cpu-type', 'X86KvmCPU', common.Newline])
+                    cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), common.Newline])
                 elif common.is_arm:
                     # TODO why is it mandatory to pass mem= here? Not true for QEMU.
                     # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value?
                     cmd.extend([
-                        '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli),
-                        '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)),
-                        '--machine-type', common.machine,
-                        '--param', 'system.panic_on_panic = True',
+                        '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), common.Newline,
+                        '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)), common.Newline,
+                        '--machine-type', common.machine, common.Newline,
+                        '--param', 'system.panic_on_panic = True', common.Newline,
                     ])
                     if not args.baremetal is None:
-                        cmd.append('--bare-metal')
+                        cmd.extend(['--bare-metal', common.Newline])
                         if args.arch == 'aarch64':
                             # https://stackoverflow.com/questions/43682311/uart-communication-in-gem5-with-arm-bare-metal/50983650#50983650
-                            cmd.extend(['--param', 'system.highest_el_is_64 = True'])
-                            cmd.extend(['--param', 'system.auto_reset_addr_64 = True'])
+                            cmd.extend(['--param', 'system.highest_el_is_64 = True', common.Newline])
+                            cmd.extend(['--param', 'system.auto_reset_addr_64 = True', common.Newline])
             elif args.gem5_script == 'biglittle':
                 if args.gem5_restore is not None:
                     cpt_dir = common.gem_list_checkpoint_dirs()[-args.gem5_restore]
                     extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)])
                 cmd.extend([
-                    os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'),
-                    '--big-cpus', '2',
-                    '--cpu-type', 'atomic',
-                    '--disk', common.disk_image,
-                    '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'),
-                    '--kernel', common.image,
-                    '--little-cpus', '2'
+                    os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), common.Newline,
+                    '--big-cpus', '2', common.Newline,
+                    '--cpu-type', 'atomic', common.Newline,
+                    '--disk', common.disk_image, common.Newline,
+                    '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), common.Newline,
+                    '--kernel', common.image, common.Newline,
+                    '--little-cpus', '2', common.Newline,
                 ])
         if args.wait_gdb:
             # https://stackoverflow.com/questions/49296092/how-to-make-gem5-wait-for-gdb-to-connect-to-reliably-break-at-start-kernel-of-th
-            cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True'])
+            cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', common.Newline])
     else:
         qemu_user_and_system_options = [
-            '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file),
+            '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), common.Newline,
         ]
         if args.userland is not None:
             if args.wait_gdb:
-                debug_args = ['-g', str(common.gdb_port)]
+                debug_args = ['-g', str(common.gdb_port), common.Newline]
             else:
                 debug_args = []
             cmd.extend(
                 [
-                    os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)),
-                    '-L', common.target_dir,
+                    os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)), common.Newline,
+                    '-L', common.target_dir, common.Newline
                 ] +
                 qemu_user_and_system_options +
-                shlex.split(args.userland_before) +
+                common.shlex_split(args.userland_before) +
                 debug_args +
                 [
-                    common.resolve_userland(args.userland)
+                    common.resolve_userland(args.userland), common.Newline
                 ]
             )
         else:
@@ -250,10 +251,10 @@ def main(args, extra_args=None):
             if args.debug_vm:
                 serial_monitor = []
             else:
-                serial_monitor = ['-serial', 'mon:stdio']
+                serial_monitor = ['-serial', 'mon:stdio', common.Newline]
             if args.kvm:
-                extra_emulator_args.append('-enable-kvm')
-            extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port)])
+                extra_emulator_args.extend(['-enable-kvm', common.Newline])
+            extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port), common.Newline])
             virtfs_data = [
                 (common.p9_dir, 'host_data'),
                 (common.out_dir, 'host_out'),
@@ -266,19 +267,20 @@ def main(args, extra_args=None):
                     virtfs_cmd.extend([
                         '-virtfs',
                         'local,path={virtfs_dir},mount_tag={virtfs_tag},security_model=mapped,id={virtfs_tag}' \
-                            .format(virtfs_dir=virtfs_dir, virtfs_tag=virtfs_tag
-                    )])
+                            .format(virtfs_dir=virtfs_dir, virtfs_tag=virtfs_tag),
+                        common.Newline,
+                    ])
             cmd.extend(
                 [
-                    qemu_executable,
-                    '-device', 'rtl8139,netdev=net0',
-                    '-gdb', 'tcp::{}'.format(common.gdb_port),
-                    '-kernel', common.image,
-                    '-m', args.memory,
-                    '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port),
-                    '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port),
-                    '-no-reboot',
-                    '-smp', str(args.cpus),
+                    qemu_executable, common.Newline,
+                    '-device', 'rtl8139,netdev=net0', common.Newline,
+                    '-gdb', 'tcp::{}'.format(common.gdb_port), common.Newline,
+                    '-kernel', common.image, common.Newline,
+                    '-m', args.memory, common.Newline,
+                    '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), common.Newline,
+                    '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), common.Newline,
+                    '-no-reboot', common.Newline,
+                    '-smp', str(args.cpus), common.Newline,
                 ] +
                 virtfs_cmd +
                 qemu_user_and_system_options +
@@ -309,12 +311,13 @@ def main(args, extra_args=None):
                         common.raw_to_qcow2(prebuilt=args.prebuilt)
                     extra_emulator_args.extend([
                         '-drive',
-                        'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid)
+                        'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid),
+                        common.Newline,
                     ])
                     if rr:
                         extra_emulator_args.extend([
-                        '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay',
-                        '-device', 'ide-hd,drive=img-blkreplay'
+                        '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', common.Newline,
+                        '-device', 'ide-hd,drive=img-blkreplay', common.Newline,
                     ])
             if rr:
                 extra_emulator_args.extend([
@@ -325,13 +328,13 @@ def main(args, extra_args=None):
             else:
                 virtio_gpu_pci = ['-device', 'virtio-gpu-pci']
             if args.arch == 'x86_64':
-                append = ['-append', '{} nopat {}'.format(root, kernel_cli)]
+                append = ['-append', '{} nopat {}'.format(root, kernel_cli), common.Newline]
                 cmd.extend([
-                    '-M', common.machine,
-                    '-device', 'edu',
+                    '-M', common.machine, common.Newline,
+                    '-device', 'edu', common.Newline,
                 ])
             elif common.is_arm:
-                extra_emulator_args.append('-semihosting')
+                extra_emulator_args.extend(['-semihosting', common.Newline])
                 if args.arch == 'arm':
                     cpu = 'cortex-a15'
                 else:
@@ -341,8 +344,8 @@ def main(args, extra_args=None):
                     [
                         # highmem=off needed since v3.0.0 due to:
                         # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
-                        '-M', '{},highmem=off'.format(common.machine),
-                        '-cpu', cpu,
+                        '-M', '{},highmem=off'.format(common.machine), common.Newline,
+                        '-cpu', cpu, common.Newline,
                     ] +
                     virtio_gpu_pci
                 )

run-gdb

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import os
-import shlex
 import sys
 import signal
 import subprocess
@@ -33,10 +32,10 @@ def main(args, extra_args=None):
     '''
     global defaults
     args = common.resolve_args(defaults, args, extra_args)
-    after = shlex.split(args.after)
-    before = shlex.split(args.before)
+    after = common.shlex_split(args.after)
+    before = common.shlex_split(args.before)
     if args.break_at is not None:
-        break_at = ['-ex', 'break {}'.format(args.break_at)]
+        break_at = ['-ex', 'break {}'.format(args.break_at), common.Newline]
     else:
         break_at = []
     linux_full_system = (args.baremetal is None and args.userland is None)
@@ -51,12 +50,12 @@ def main(args, extra_args=None):
     else:
         allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
     cmd = (
-        [common.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains)] +
+        [common.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains), common.Newline] +
         before +
-        ['-q']
+        ['-q', common.Newline]
     )
     if linux_full_system:
-        cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir)])
+        cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), common.Newline])
     if args.sim:
         target = 'sim'
     else:
@@ -66,8 +65,8 @@ def main(args, extra_args=None):
             port = common.gdb_port
         target = 'remote localhost:{}'.format(port)
     cmd.extend([
-        '-ex', 'file {}'.format(image),
-        '-ex', 'target {}'.format(target),
+        '-ex', 'file {}'.format(image), common.Newline,
+        '-ex', 'target {}'.format(target), common.Newline,
     ])
     if not args.kgdb:
         cmd.extend(break_at)
@@ -91,9 +90,9 @@ def main(args, extra_args=None):
         #
         # The lx-symbols commands gets loaded through the file vmlinux-gdb.py
         # which gets put on the kernel build root when python debugging scripts are enabled.
-        cmd.extend(['-ex', 'continue'])
+        cmd.extend(['-ex', 'continue', common.Newline])
         if not args.no_lxsymbols and linux_full_system:
-            cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir)])
+            cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), common.Newline])
     cmd.extend(after)
     # I would rather have cwd be out_rootfs_overlay_dir,
     # but then lx-symbols cannot fine the vmlinux and fails with:

run-toolchain

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import os
-import subprocess
 import sys
 
 import common
@@ -45,6 +44,7 @@ if args.dry:
     print(tool)
 else:
     sys.exit(common.run_cmd(
-        [tool] + args.extra_args,
+        [tool, common.Newline]
+        + common.add_newlines(args.extra_args),
         cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'),
     ))