linux-kernel-module-cheat/common.py

#!/usr/bin/env python3

import argparse
import base64
import collections
import copy
import datetime
import distutils.file_util
import glob
import imp
import json
import os
import re
import shlex
import shutil
import signal
import stat
import subprocess
import sys
import time
import urllib
import urllib.request

this_module = sys.modules[__name__]
root_dir = os.path.dirname(os.path.abspath(__file__))
data_dir = os.path.join(root_dir, 'data')
p9_dir = os.path.join(data_dir, '9p')
gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
out_dir = os.path.join(root_dir, 'out')
bench_boot = os.path.join(out_dir, 'bench-boot.txt')
packages_dir = os.path.join(root_dir, 'packages')
lkmc_package_src_dir = os.path.join(this_module.packages_dir, 'lkmc')
kernel_modules_subdir = 'kernel_modules'
kernel_modules_src_dir = os.path.join(this_module.lkmc_package_src_dir, this_module.kernel_modules_subdir)
userland_subdir = 'userland'
userland_src_dir = os.path.join(this_module.lkmc_package_src_dir, this_module.userland_subdir)
submodules_dir = os.path.join(root_dir, 'submodules')
buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng')
linux_src_dir = os.path.join(submodules_dir, 'linux')
linux_config_dir = os.path.join(this_module.root_dir, 'kernel_config')
rootfs_overlay_dir = os.path.join(this_module.root_dir, 'rootfs_overlay')
extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux')
qemu_src_dir = os.path.join(submodules_dir, 'qemu')
parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark')
ccache_dir = os.path.join('/usr', 'lib', 'ccache')
default_build_id = 'default'
arch_short_to_long_dict = collections.OrderedDict([
    ('x', 'x86_64'),
    ('a', 'arm'),
    ('A', 'aarch64'),
])
all_archs = [arch_short_to_long_dict[k] for k in arch_short_to_long_dict]
arch_choices = []
for key in this_module.arch_short_to_long_dict:
    arch_choices.append(key)
    arch_choices.append(this_module.arch_short_to_long_dict[key])
default_arch = 'x86_64'
gem5_cpt_prefix = '^cpt\.'
sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
release_dir = os.path.join(this_module.out_dir, 'release')
release_zip_file = os.path.join(this_module.release_dir, 'lkmc-{}.zip'.format(this_module.sha))
github_repo_id = 'cirosantilli/linux-kernel-module-cheat'
asm_ext = '.S'
c_ext = '.c'
header_ext = '.h'
kernel_module_ext = '.ko'
obj_ext = '.o'
executable_ext = '.out'
config_file = os.path.join(data_dir, 'config')
command_prefix = '+ '
if os.path.exists(config_file):
    config = imp.load_source('config', config_file)
    configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')}

class Component:
    def __init__(self):
        pass

    def build(self):
        parser = this_module.get_argparse(argparse_args=self.get_argparse_args())
        self.add_parser_arguments(parser)
        this_module.add_build_arguments(parser)
        args = this_module.setup(parser)
        if not this_module.dry_run:
            start_time = time.time()
        if args.clean:
            self.clean(args)
        else:
            self.do_build(args)
        if not this_module.dry_run:
            end_time = time.time()
            this_module.print_time(end_time - start_time)

    def add_parser_arguments(self, parser):
        pass

    def clean(self, args):
        this_module.rmrf(self.get_build_dir(args))

    def do_build(self, args):
        raise NotImplementedError()

    def get_argparse_args(self):
        return {}

    def get_build_dir(self, args):
        raise NotImplementedError()

def add_build_arguments(parser):
    parser.add_argument(
        '--clean',
        help='Clean the build instead of building.',
        action='store_true',
    )

def add_dry_run_argument(parser):
    parser.add_argument('--dry-run', default=False, action='store_true', help='''\
    Print the commands that would be run, but don't run them.
''')

def base64_encode(string):
    return base64.b64encode(string.encode()).decode()

def copy_dir_if_update_non_recursive(srcdir, destdir, filter_ext=None):
    os.makedirs(destdir, exist_ok=True)
    for basename in os.listdir(srcdir):
        src = os.path.join(srcdir, basename)
        if os.path.isfile(src):
            noext, ext = os.path.splitext(basename)
            if filter_ext is not None and ext == filter_ext:
                distutils.file_util.copy_file(
                    src,
                    os.path.join(destdir, basename),
                    update=1,
                )

def gem_list_checkpoint_dirs():
    '''
    List checkpoint directory, oldest first.
    '''
    global this_module
    prefix_re = re.compile(this_module.gem5_cpt_prefix)
    files = list(filter(lambda x: os.path.isdir(os.path.join(this_module.m5out_dir, x)) and prefix_re.search(x), os.listdir(this_module.m5out_dir)))
    files.sort(key=lambda x: os.path.getmtime(os.path.join(this_module.m5out_dir, x)))
    return files

def get_argparse(default_args=None, argparse_args=None):
    '''
    Return an argument parser with common arguments set.

    :type default_args: Dict[str,str]
    :type argparse_args: Dict
    '''
    global this_module
    if default_args is None:
        default_args = {}
    if argparse_args is None:
        argparse_args = {}
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter,
        **argparse_args
    )
    this_module.add_dry_run_argument(parser)
    parser.add_argument(
        '-a', '--arch', choices=this_module.arch_choices, default=this_module.default_arch,
        help='CPU architecture. Default: %(default)s'
    )
    parser.add_argument(
        '--baremetal',
        help='Use Baremetal examples instead of Linux kernel ones'
    )
    parser.add_argument(
        '--buildroot-build-id',
        default=default_build_id,
        help='Buildroot build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
    )
    parser.add_argument(
        '--crosstool-ng-build-id', default=default_build_id,
        help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds. Default: %(default)s'
    )
    parser.add_argument(
        '--docker', default=False, action='store_true',
        help='''\
Use the docker download Ubuntu root filesystem instead of the default Buildroot one.
'''
    )
    parser.add_argument(
        '-g', '--gem5', default=False, action='store_true',
        help='Use gem5 instead of QEMU'
    )
    parser.add_argument(
        '--gem5-src',
        help='''\
Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`.
'''
    )
    parser.add_argument(
        '-L', '--linux-build-id', default=default_build_id,
        help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s'
    )
    parser.add_argument(
        '--machine',
        help='''Machine type.
QEMU default: virt
gem5 default: VExpress_GEM5_V1
See the documentation for other values known to work.
'''
    )
    parser.add_argument(
        '-M', '--gem5-build-id',
        help='gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'.format(default_build_id)
    )
    parser.add_argument(
        '-N', '--gem5-worktree',
        help='''\
Create and use a git worktree of the gem5 submodule.
See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree
'''
    )
    parser.add_argument(
        '--gem5-build-dir',
        help='''\
Use the given directory as the gem5 build directory.
'''
    )
    parser.add_argument(
        '--gem5-source-dir',
        help='''\
Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`.
'''
    )
    parser.add_argument(
        '-n', '--run-id', default='0',
        help='''\
ID for run outputs such as gem5's m5out. Allows you to do multiple runs,
and then inspect separate outputs later in different output directories.
Default: %(default)s
'''
    )
    parser.add_argument(
        '-P', '--prebuilt', default=False, action='store_true',
        help='''\
Use prebuilt packaged host utilities as much as possible instead
of the ones we built ourselves. Saves build time, but decreases
the likelihood of compatibility.
'''
    )
    parser.add_argument(
        '--port-offset', type=int,
        help='''\
Increase the ports to be used such as for GDB by an offset to run multiple
instances in parallel.
Default: the run ID (-n) if that is an integer, otherwise 0.
'''
    )
    parser.add_argument(
        '-Q', '--qemu-build-id', default=default_build_id,
        help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
    )
    parser.add_argument(
        '-t', '--gem5-build-type', default='opt',
        help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
    )
    parser.add_argument(
        '-v', '--verbose', default=False, action='store_true',
        help='Show full compilation commands when they are not shown by default.'
    )
    if hasattr(this_module, 'configs'):
        defaults = this_module.configs.copy()
    else:
        defaults = {}
    defaults.update(default_args)
    # A bit ugly as it actually changes the defaults shown on --help, but we can't do any better
    # because it is impossible to check if arguments were given or not...
    # - https://stackoverflow.com/questions/30487767/check-if-argparse-optional-argument-is-set-or-not
    # - https://stackoverflow.com/questions/3609852/which-is-the-best-way-to-allow-configuration-options-be-overridden-at-the-comman
    parser.set_defaults(**defaults)
    return parser

def get_elf_entry(elf_file_path):
    global this_module
    readelf_header = subprocess.check_output([
        this_module.get_toolchain_tool('readelf'),
        '-h',
        elf_file_path
    ])
    for line in readelf_header.decode().split('\n'):
        split = line.split()
        if line.startswith('  Entry point address:'):
            addr = line.split()[-1]
            break
    return int(addr, 0)

def get_stats(stat_re=None, stats_file=None):
    global this_module
    if stat_re is None:
        stat_re = '^system.cpu[0-9]*.numCycles$'
    if stats_file is None:
        stats_file = this_module.stats_file
    stat_re = re.compile(stat_re)
    ret = []
    with open(stats_file, 'r') as statfile:
        for line in statfile:
            if line[0] != '-':
                cols = line.split()
                if len(cols) > 1 and stat_re.search(cols[0]):
                    ret.append(cols[1])
    return ret

def get_toolchain_prefix(tool, allowed_toolchains=None):
    buildroot_full_prefix = os.path.join(this_module.host_bin_dir, this_module.buildroot_toolchain_prefix)
    buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
    crosstool_ng_full_prefix = os.path.join(this_module.crosstool_ng_bin_dir, this_module.crosstool_ng_toolchain_prefix)
    crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
    host_tool = '{}-{}'.format(this_module.ubuntu_toolchain_prefix, tool)
    host_path = shutil.which(host_tool)
    if host_path is not None:
        host_exists = True
        host_full_prefix = host_path[:-(len(tool)+1)]
    else:
        host_exists = False
        host_full_prefix = None
    known_toolchains = {
        'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix),
        'buildroot': (buildroot_exists, buildroot_full_prefix),
        'host': (host_exists, host_full_prefix),
    }
    if allowed_toolchains is None:
        if this_module.baremetal is None:
            allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
        else:
            allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
    tried = []
    for toolchain in allowed_toolchains:
        exists, prefix = known_toolchains[toolchain]
        tried.append('{}-{}'.format(prefix, tool))
        if exists:
            return prefix
    raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))

def get_toolchain_tool(tool, allowed_toolchains=None):
    return '{}-{}'.format(this_module.get_toolchain_prefix(tool, allowed_toolchains), tool)

def github_make_request(
        authenticate=False,
        data=None,
        extra_headers=None,
        path='',
        subdomain='api',
        url_params=None,
        **extra_request_args
    ):
    global this_module
    if extra_headers is None:
        extra_headers = {}
    headers = {'Accept': 'application/vnd.github.v3+json'}
    headers.update(extra_headers)
    if authenticate:
        headers['Authorization'] = 'token ' + os.environ['LKMC_GITHUB_TOKEN']
    if url_params is not None:
        path += '?' + urllib.parse.urlencode(url_params)
    request = urllib.request.Request(
        'https://' + subdomain + '.github.com/repos/' + github_repo_id + path,
        headers=headers,
        data=data,
        **extra_request_args
    )
    response_body = urllib.request.urlopen(request).read().decode()
    if response_body:
        _json = json.loads(response_body)
    else:
        _json = {}
    return _json

def log_error(msg):
    print('error: {}'.format(msg), file=sys.stderr)

def make_build_dirs():
    global this_module
    os.makedirs(this_module.build_dir, exist_ok=True)
    os.makedirs(this_module.gem5_build_dir, exist_ok=True)
    os.makedirs(this_module.out_rootfs_overlay_dir, exist_ok=True)

def make_run_dirs():
    '''
    Make directories required for the run.
    The user could nuke those anytime between runs to try and clean things up.
    '''
    global this_module
    os.makedirs(this_module.gem5_run_dir, exist_ok=True)
    os.makedirs(this_module.p9_dir, exist_ok=True)
    os.makedirs(this_module.qemu_run_dir, exist_ok=True)

def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
    '''
    Format a command given as a list of strings so that it can
    be viewed nicely and executed by bash directly and print it to stdout.

    Optionally save the command to cmd_file file, and add extra_env
    environment variables to the command generated.
    '''
    newline_separator = ' \\\n'
    out = []
    if extra_env is None:
        extra_env = {}
    if cwd is not None:
        out.append('cd {} &&{}'.format(shlex.quote(cwd), newline_separator))
    if extra_paths is not None:
        out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)) + newline_separator)
    for key in extra_env:
        out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])) + newline_separator)
    for arg in cmd:
        out.append(shlex.quote(arg) + newline_separator)
    out = '  '.join(out) + ';\n'
    print(this_module.command_prefix + out, end='')
    if cmd_file is not None:
        with open(cmd_file, 'w') as f:
            f.write('#!/usr/bin/env bash\n')
            f.write(out)
        st = os.stat(cmd_file)
        os.chmod(cmd_file, st.st_mode | stat.S_IXUSR)

def print_time(ellapsed_seconds):
    hours, rem = divmod(ellapsed_seconds, 3600)
    minutes, seconds = divmod(rem, 60)
    print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))

def raw_to_qcow2(prebuilt=False, reverse=False):
    global this_module
    if prebuilt:
        qemu_img_executable = this_module.qemu_img_basename
    else:
        qemu_img_executable = this_module.qemu_img_executable
    infmt = 'raw'
    outfmt = 'qcow2'
    infile = this_module.rootfs_raw_file
    outfile = this_module.qcow2_file
    if reverse:
        tmp = infmt
        infmt = outfmt
        outfmt = tmp
        tmp = infile
        infile = outfile
        outfile = tmp
    assert this_module.run_cmd([
        qemu_img_executable,
        # Prevent qemu-img from generating trace files like QEMU. Disgusting.
        '-T', 'pr_manager_run,file=/dev/null',
        'convert',
        '-f', infmt,
        '-O', outfmt,
        infile,
        outfile,
    ]) == 0

def raise_no_x86(arch):
    if (arch == 'x86_64'):
        raise Exception('x86_64 not yet supported')

def resolve_args(defaults, args, extra_args):
    if extra_args is None:
        extra_args = {}
    argcopy = copy.copy(args)
    argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items()))
    return argcopy

def cp(src, dest):
    print_cmd(['cp', src, dest])
    if not this_module.dry_run:
        shutil.copy2(src, dest)

def rmrf(path):
    print_cmd(['rm', '-r', '-f', path])
    if not this_module.dry_run and os.path.exists(path):
        shutil.rmtree(path)

def run_cmd(
        cmd,
        cmd_file=None,
        out_file=None,
        show_stdout=True,
        show_cmd=True,
        extra_env=None,
        extra_paths=None,
        delete_env=None,
        dry_run=False,
        **kwargs
    ):
    '''
    Run a command. Write the command to stdout before running it.

    Wait until the command finishes execution.

    :param cmd: command to run
    :type cmd: List[str]

    :param cmd_file: if not None, write the command to be run to that file
    :type cmd_file: str

    :param out_file: if not None, write the stdout and stderr of the command the file
    :type out_file: str

    :param show_stdout: wether to show stdout and stderr on the terminal or not
    :type show_stdout: bool

    :param extra_env: extra environment variables to add when running the command
    :type extra_env: Dict[str,str]

    :param dry_run: don't run the commands, just potentially print them. Debug aid.
    :type dry_run: Bool
    '''
    if out_file is not None:
        stdout = subprocess.PIPE
        stderr = subprocess.STDOUT
    else:
        if show_stdout:
            stdout = None
            stderr = None
        else:
            stdout = subprocess.DEVNULL
            stderr = subprocess.DEVNULL
    if extra_env is None:
        extra_env = {}
    if delete_env is None:
        delete_env = []
    if 'cwd' in kwargs:
        cwd = kwargs['cwd']
    else:
        cwd = None
    env = os.environ.copy()
    env.update(extra_env)
    if extra_paths is not None:
        path = ':'.join(extra_paths)
        if 'PATH' in os.environ:
            path += ':' + os.environ['PATH']
        env['PATH'] = path
    for key in delete_env:
        if key in env:
            del env[key]
    if show_cmd:
        print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths)

    # Otherwise Ctrl + C gives:
    # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
    # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
    sigint_old = signal.getsignal(signal.SIGINT)
    signal.signal(signal.SIGINT, signal.SIG_IGN)

    # Otherwise BrokenPipeError when piping through | grep
    # But if I do this_module, my terminal gets broken at the end. Why, why, why.
    # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python
    # Ignoring the exception is not enough as it prints a warning anyways.
    #sigpipe_old = signal.getsignal(signal.SIGPIPE)
    #signal.signal(signal.SIGPIPE, signal.SIG_DFL)

    if not dry_run and not this_module.dry_run:
        # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
        with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
            if out_file is not None:
                os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True)
                with open(out_file, 'bw') as logfile:
                    while True:
                        byte = proc.stdout.read(1)
                        if byte:
                            if show_stdout:
                                sys.stdout.buffer.write(byte)
                                try:
                                    sys.stdout.flush()
                                except BlockingIOError:
                                    # TODO understand. Why, Python, why.
                                    pass
                            logfile.write(byte)
                        else:
                            break
        signal.signal(signal.SIGINT, sigint_old)
        #signal.signal(signal.SIGPIPE, sigpipe_old)
        return proc.returncode
    else:
        return 0

def setup(parser):
    '''
    Parse the command line arguments, and setup several variables based on them.
    Typically done after getting inputs from the command line arguments.
    '''
    global this_module
    args = parser.parse_args()
    if args.arch in this_module.arch_short_to_long_dict:
        args.arch = this_module.arch_short_to_long_dict[args.arch]
    if args.gem5_build_id is None:
        args.gem5_build_id = default_build_id
        gem5_build_id_given = False
    else:
        gem5_build_id_given = True
    if args.gem5_worktree is not None and not gem5_build_id_given:
        args.gem5_build_id = args.gem5_worktree
    this_module.machine = args.machine
    this_module.dry_run = args.dry_run
    if args.arch == 'arm':
        this_module.armv = 7
        this_module.gem5_arch = 'ARM'
        this_module.mcpu = 'cortex-a15'
        this_module.buildroot_toolchain_prefix = 'arm-buildroot-linux-uclibcgnueabihf'
        this_module.crosstool_ng_toolchain_prefix = 'arm-unknown-eabi'
        this_module.ubuntu_toolchain_prefix = 'arm-linux-gnueabihf'
        if args.gem5:
            if this_module.machine is None:
                this_module.machine = 'VExpress_GEM5_V1'
        else:
            if this_module.machine is None:
                this_module.machine = 'virt'
    elif args.arch == 'aarch64':
        this_module.armv = 8
        this_module.gem5_arch = 'ARM'
        this_module.mcpu = 'cortex-a57'
        this_module.buildroot_toolchain_prefix = 'aarch64-buildroot-linux-uclibc'
        this_module.crosstool_ng_toolchain_prefix = 'aarch64-unknown-elf'
        this_module.ubuntu_toolchain_prefix = 'aarch64-linux-gnu'
        if args.gem5:
            if this_module.machine is None:
                this_module.machine = 'VExpress_GEM5_V1'
        else:
            if this_module.machine is None:
                this_module.machine = 'virt'
    elif args.arch == 'x86_64':
        this_module.crosstool_ng_toolchain_prefix = 'x86_64-unknown-elf'
        this_module.gem5_arch = 'X86'
        this_module.buildroot_toolchain_prefix = 'x86_64-buildroot-linux-uclibc'
        this_module.ubuntu_toolchain_prefix = 'x86_64-linux-gnu'
        if args.gem5:
            if this_module.machine is None:
                this_module.machine = 'TODO'
        else:
            if this_module.machine is None:
                this_module.machine = 'pc'
    this_module.buildroot_out_dir = os.path.join(this_module.out_dir, 'buildroot')
    this_module.buildroot_build_dir = os.path.join(this_module.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch)
    this_module.buildroot_download_dir = os.path.join(this_module.buildroot_out_dir, 'download')
    this_module.buildroot_config_file = os.path.join(this_module.buildroot_build_dir, '.config')
    this_module.build_dir = os.path.join(this_module.buildroot_build_dir, 'build')
    this_module.qemu_build_dir = os.path.join(this_module.out_dir, 'qemu', args.qemu_build_id)
    this_module.qemu_executable_basename = 'qemu-system-{}'.format(args.arch)
    this_module.qemu_executable = os.path.join(this_module.qemu_build_dir, '{}-softmmu'.format(args.arch), this_module.qemu_executable_basename)
    this_module.qemu_img_basename = 'qemu-img'
    this_module.qemu_img_executable = os.path.join(this_module.qemu_build_dir, this_module.qemu_img_basename)
    this_module.qemu_guest_build_dir = os.path.join(this_module.build_dir, 'qemu-custom')
    this_module.host_dir = os.path.join(this_module.buildroot_build_dir, 'host')
    this_module.host_bin_dir = os.path.join(this_module.host_dir, 'usr', 'bin')
    this_module.buildroot_pkg_config = os.path.join(this_module.host_bin_dir, 'pkg-config')
    this_module.buildroot_images_dir = os.path.join(this_module.buildroot_build_dir, 'images')
    this_module.buildroot_rootfs_raw_file = os.path.join(this_module.buildroot_images_dir, 'rootfs.ext2')
    this_module.buildroot_qcow2_file = this_module.buildroot_rootfs_raw_file + '.qcow2'
    this_module.staging_dir = os.path.join(this_module.out_dir, 'staging', args.arch)
    this_module.buildroot_staging_dir = os.path.join(this_module.buildroot_build_dir, 'staging')
    this_module.target_dir = os.path.join(this_module.buildroot_build_dir, 'target')
    this_module.run_dir_base = os.path.join(this_module.out_dir, 'run')
    this_module.gem5_run_dir = os.path.join(this_module.run_dir_base, 'gem5', args.arch, str(args.run_id))
    this_module.m5out_dir = os.path.join(this_module.gem5_run_dir, 'm5out')
    this_module.stats_file = os.path.join(this_module.m5out_dir, 'stats.txt')
    this_module.trace_txt_file = os.path.join(this_module.m5out_dir, 'trace.txt')
    this_module.gem5_readfile = os.path.join(this_module.gem5_run_dir, 'readfile')
    this_module.gem5_termout_file = os.path.join(this_module.gem5_run_dir, 'termout.txt')
    this_module.qemu_run_dir = os.path.join(this_module.run_dir_base, 'qemu', args.arch, str(args.run_id))
    this_module.qemu_trace_basename = 'trace.bin'
    this_module.qemu_trace_file = os.path.join(this_module.qemu_run_dir, 'trace.bin')
    this_module.qemu_trace_txt_file = os.path.join(this_module.qemu_run_dir, 'trace.txt')
    this_module.qemu_termout_file = os.path.join(this_module.qemu_run_dir, 'termout.txt')
    this_module.qemu_rrfile = os.path.join(this_module.qemu_run_dir, 'rrfile')
    this_module.gem5_out_dir = os.path.join(this_module.out_dir, 'gem5')
    if args.gem5_build_dir is None:
        this_module.gem5_build_dir = os.path.join(this_module.gem5_out_dir, args.gem5_build_id, args.gem5_build_type)
    else:
        this_module.gem5_build_dir = args.gem5_build_dir
    this_module.gem5_fake_iso = os.path.join(this_module.gem5_out_dir, 'fake.iso')
    this_module.gem5_m5term = os.path.join(this_module.gem5_build_dir, 'm5term')
    this_module.gem5_build_build_dir = os.path.join(this_module.gem5_build_dir, 'build')
    this_module.gem5_executable = os.path.join(this_module.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type))
    this_module.gem5_system_dir = os.path.join(this_module.gem5_build_dir, 'system')
    this_module.crosstool_ng_out_dir = os.path.join(this_module.out_dir, 'crosstool-ng')
    this_module.crosstool_ng_buildid_dir = os.path.join(this_module.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id)
    this_module.crosstool_ng_install_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'install', args.arch)
    this_module.crosstool_ng_bin_dir = os.path.join(this_module.crosstool_ng_install_dir, 'bin')
    this_module.crosstool_ng_util_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'util')
    this_module.crosstool_ng_config = os.path.join(this_module.crosstool_ng_util_dir, '.config')
    this_module.crosstool_ng_defconfig = os.path.join(this_module.crosstool_ng_util_dir, 'defconfig')
    this_module.crosstool_ng_executable = os.path.join(this_module.crosstool_ng_util_dir, 'ct-ng')
    this_module.crosstool_ng_build_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'build')
    this_module.crosstool_ng_download_dir = os.path.join(this_module.crosstool_ng_out_dir, 'download')
    this_module.gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
    if args.gem5_source_dir is not None:
        this_module.gem5_src_dir = args.gem5_source_dir
        assert(os.path.exists(args.gem5_source_dir))
    else:
        if args.gem5_worktree is not None:
            this_module.gem5_src_dir = os.path.join(this_module.gem5_non_default_src_root_dir, args.gem5_worktree)
        else:
            this_module.gem5_src_dir = this_module.gem5_default_src_dir
    this_module.gem5_m5_src_dir = os.path.join(this_module.gem5_src_dir, 'util', 'm5')
    this_module.gem5_m5_build_dir = os.path.join(this_module.out_dir, 'util', 'm5')
    if args.gem5:
        this_module.executable = this_module.gem5_executable
        this_module.run_dir = this_module.gem5_run_dir
        this_module.termout_file = this_module.gem5_termout_file
    else:
        this_module.executable = this_module.qemu_executable
        this_module.run_dir = this_module.qemu_run_dir
        this_module.termout_file = this_module.qemu_termout_file
    this_module.gem5_config_dir = os.path.join(this_module.gem5_src_dir, 'configs')
    this_module.gem5_se_file = os.path.join(this_module.gem5_config_dir, 'example', 'se.py')
    this_module.gem5_fs_file = os.path.join(this_module.gem5_config_dir, 'example', 'fs.py')
    this_module.run_cmd_file = os.path.join(this_module.run_dir, 'run.sh')

    # Linux
    this_module.linux_buildroot_build_dir = os.path.join(this_module.build_dir, 'linux-custom')
    this_module.linux_build_dir = os.path.join(this_module.out_dir, 'linux', args.linux_build_id, args.arch)
    this_module.vmlinux = os.path.join(this_module.linux_build_dir, "vmlinux")
    if args.arch == 'arm':
        this_module.linux_arch = 'arm'
        this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'zImage')
    elif args.arch == 'aarch64':
        this_module.linux_arch = 'arm64'
        this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'Image')
    elif args.arch == 'x86_64':
        this_module.linux_arch = 'x86'
        this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'bzImage')
    this_module.linux_image = os.path.join(this_module.linux_build_dir, linux_image)

    # Kernel modules.
    this_module.kernel_modules_build_base_dir = os.path.join(this_module.out_dir, 'kernel_modules')
    this_module.kernel_modules_build_dir = os.path.join(this_module.kernel_modules_build_base_dir, args.arch)
    this_module.kernel_modules_build_host_dir = os.path.join(this_module.kernel_modules_build_base_dir, 'host')
    this_module.userland_build_dir = os.path.join(this_module.out_dir, 'userland', args.arch)
    this_module.out_rootfs_overlay_dir = os.path.join(this_module.out_dir, 'rootfs_overlay', args.arch)
    this_module.out_rootfs_overlay_bin_dir = os.path.join(this_module.out_rootfs_overlay_dir, 'bin')

    # Ports
    if args.port_offset is None:
        try:
            args.port_offset = int(args.run_id)
        except ValueError:
            args.port_offset = 0
    if args.gem5:
        this_module.gem5_telnet_port = 3456 + args.port_offset
        this_module.gdb_port = 7000 + args.port_offset
    else:
        this_module.qemu_base_port = 45454 + 10 * args.port_offset
        this_module.qemu_monitor_port = this_module.qemu_base_port + 0
        this_module.qemu_hostfwd_generic_port = this_module.qemu_base_port + 1
        this_module.qemu_hostfwd_ssh_port = this_module.qemu_base_port + 2
        this_module.qemu_gdb_port = this_module.qemu_base_port + 3
        this_module.gdb_port = this_module.qemu_gdb_port

    # Baremetal.
    this_module.baremetal = args.baremetal
    this_module.baremetal_lib_basename = 'lib'
    this_module.baremetal_src_dir = os.path.join(this_module.root_dir, 'baremetal')
    this_module.baremetal_src_lib_dir = os.path.join(this_module.baremetal_src_dir, this_module.baremetal_lib_basename)
    if args.gem5:
        this_module.simulator_name = 'gem5'
    else:
        this_module.simulator_name = 'qemu'
    this_module.baremetal_out_dir = os.path.join(out_dir, 'baremetal', args.arch, this_module.simulator_name, this_module.machine)
    this_module.baremetal_out_lib_dir = os.path.join(this_module.baremetal_out_dir, this_module.baremetal_lib_basename)
    this_module.baremetal_out_ext = '.elf'


    # Docker
    this_module.docker_build_dir = os.path.join(this_module.out_dir, 'docker', args.arch)
    this_module.docker_tar_dir = os.path.join(this_module.docker_build_dir, 'export')
    this_module.docker_tar_file = os.path.join(this_module.docker_build_dir, 'export.tar')
    this_module.docker_rootfs_raw_file = os.path.join(this_module.docker_build_dir, 'export.ext2')
    this_module.docker_qcow2_file = os.path.join(this_module.docker_rootfs_raw_file + '.qcow2')
    if args.docker:
        this_module.rootfs_raw_file = this_module.docker_rootfs_raw_file
        this_module.qcow2_file = this_module.docker_qcow2_file
    else:
        this_module.rootfs_raw_file = this_module.buildroot_rootfs_raw_file
        this_module.qcow2_file = this_module.buildroot_qcow2_file

    # Image.
    if args.baremetal is None:
        if args.gem5:
            this_module.image = this_module.vmlinux
            this_module.disk_image = this_module.rootfs_raw_file
        else:
            this_module.image = this_module.linux_image
            this_module.disk_image = this_module.qcow2_file
    else:
        this_module.disk_image = this_module.gem5_fake_iso
        paths = [
            os.path.join(this_module.baremetal_out_dir, this_module.baremetal),
            os.path.join(
                this_module.baremetal_out_dir,
                os.path.relpath(this_module.baremetal, this_module.baremetal_src_dir),
            )
        ]
        paths[:] = [os.path.splitext(path)[0] + this_module.baremetal_out_ext for path in paths]
        found = False
        for path in paths:
            if os.path.exists(path):
                found = True
                break
        if not found and this_module.baremetal != 'all':
            raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
        this_module.image = path
    return args

def write_configs(config_path, configs, config_fragments=None):
    """
    Write extra configs into the Buildroot config file.
    TODO Can't get rid of these for now with nice fragments:
    http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
    """
    if config_fragments is None:
        config_fragments = []
    with open(config_path, 'a') as config_file:
        for config_fragment in config_fragments:
            with open(config_fragment, 'r') as config_fragment:
                for line in config_fragment:
                    config_file.write(line)
        for config in configs:
            config_file.write(config + '\n')