3rd-party/linux-kernel-module-cheat
1
0
Fork 0
mirror of https://github.com/cirosantilli/linux-kernel-module-cheat.git synced 2026-01-26 03:31:36 +01:00
Code Issues Releases Wiki Activity

build-linux and build-gem5 seem to work

Browse Source
This commit is contained in:
Ciro Santilli 六四事件 法轮功
2018-12-09 00:00:00 +00:00
parent 1768421dbd
commit 5e20ba833b
33 changed files with 702 additions and 707 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.adoc
View File

@@ -10301,7 +10301,7 @@ The most important things that we setup in the bootloaders are:
The C functions that become available as a result are:
* Newlib functions implemented at link:baremetal/lib/syscalls.c[]
* non-Newlib functions implemented at link:common.c[]
* non-Newlib functions implemented at link:kwargs['c'][]
It is not possible to call those C functions from the examples that don't use a bootloader.
@@ -10333,7 +10333,7 @@ svc 0x00123456
and we can see from the docs that `0x18` stands for the `SYS_EXIT` command.
This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/common.c[].
This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/kwargs['c'][].
Other magic operations we can do with semihosting besides exiting the on the host include:

3
arm
View File

@@ -1,3 +0,0 @@
#!/usr/bin/env bash
build-crosstool-ng \
;

6
bisect-linux-boot-gem5
View File

@@ -6,8 +6,8 @@ import shutil
import sys
import common
build_linux = imp.load_source('build-linux', os.path.join(common.root_dir, 'build_linux'))
run = imp.load_source('run', os.path.join(common.root_dir, 'run'))
build_linux = imp.load_source('build-linux', os.path.join(kwargs['root_dir'], 'build_linux'))
run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run'))
parser = common.get_argparse(
argparse_args={
@@ -24,7 +24,7 @@ args = common.setup(parser)
# We need a clean rebuild because rebuilds at different revisions:
# - may fail
# - may not actually rebuild all files, e.g. on header changes
common.rmrf(common.linux_build_dir)
common.rmrf(kwargs['linux_build_dir'])
build_linux.LinuxComponent().do_build(args)
status = run.main(args, {
'eval': 'm5 exit',

42
build
View File

@@ -49,7 +49,7 @@ class Component:
def run_cmd(cmd, arch):
global args
cmd_abs = cmd.copy()
cmd_abs[0] = os.path.join(common.root_dir, cmd[0])
cmd_abs[0] = os.path.join(kwargs['root_dir'], cmd[0])
cmd_abs.extend(['--arch', arch])
if kwargs['extra_args']:
cmd_abs.append(kwargs['extra_args'])
@@ -86,15 +86,15 @@ name_to_component_map = {
# Leaves without dependencies.
'baremetal-qemu': Component(
lambda arch: run_cmd(['build-baremetal', '--qemu'], arch),
supported_archs=common.crosstool_ng_supported_archs,
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'baremetal-gem5': Component(
lambda arch: run_cmd(['build-baremetal', '--gem5'], arch),
supported_archs=common.crosstool_ng_supported_archs,
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'baremetal-gem5-pbx': Component(
lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
supported_archs=common.crosstool_ng_supported_archs,
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'buildroot': buildroot_component,
'buildroot-gcc': buildroot_component,
@@ -103,7 +103,7 @@ name_to_component_map = {
),
'crosstool-ng': Component(
lambda arch: run_cmd(['build-crosstool-ng'], arch),
supported_archs=common.crosstool_ng_supported_archs,
supported_archs=kwargs['crosstool_ng_supported_archs'],
# http://crosstool-ng.github.io/docs/os-setup/
apt_get_pkgs={
'bison',
@@ -197,7 +197,7 @@ name_to_component_map = {
'gem5-baremetal',
'baremetal-gem5-pbx',
],
supported_archs=common.crosstool_ng_supported_archs,
supported_archs=kwargs['crosstool_ng_supported_archs'],
),
'all-linux': Component(dependencies=[
'qemu-gem5-buildroot',
@@ -296,10 +296,10 @@ group = parser.add_mutually_exclusive_group(required=False)
group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\
Build the selected components for all archs.
''')
group.add_argument('-a', '--arch', choices=common.arch_choices, default=[], action='append', help='''\
group.add_argument('-a', '--arch', choices=kwargs['arch_choices'], default=[], action='append', help='''\
Build the selected components for this arch. Select multiple archs by
passing this option multiple times. Default: [{}]
'''.format(common.default_arch))
'''.format(kwargs['default_arch']))
parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\
Also download all dependencies required for a given build: Ubuntu packages,
Python packages and git submodules.
@@ -314,7 +314,7 @@ Extra args to pass to all scripts.
)
parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\
Which components to build. Default: qemu-buildroot
'''.format(common.default_arch))
'''.format(kwargs['default_arch']))
common.add_dry_run_argument(parser)
args = parser.parse_args()
common.setup_dry_run_arguments(args)
@@ -322,14 +322,14 @@ common.setup_dry_run_arguments(args)
# Decide archs.
if kwargs['arch'] == []:
if kwargs['all'] or kwargs['all_archs']:
archs = common.all_archs.copy()
archs = kwargs['all_archs'].copy()
else:
archs = set([common.default_arch])
archs = set([kwargs['default_arch']])
else:
archs = set()
for arch in kwargs['arch']:
if arch in common.arch_short_to_long_dict:
arch = common.arch_short_to_long_dict[arch]
if arch in kwargs['arch_short_to_long_dict']:
arch = kwargs['arch_short_to_long_dict'][arch]
archs.add(arch)
# Decide components.
@@ -393,7 +393,7 @@ if kwargs['download_dependencies']:
'libsdl2-dev',
}
apt_get_pkgs.difference_update(interacive_pkgs)
if common.in_docker:
if kwargs['in_docker']:
sudo = []
# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
@@ -406,7 +406,7 @@ if kwargs['download_dependencies']:
f.write(sources_txt)
else:
sudo = ['sudo']
if common.in_docker or kwargs['travis']:
if kwargs['in_docker'] or kwargs['travis']:
y = ['-y']
else:
y = []
@@ -416,25 +416,25 @@ if kwargs['download_dependencies']:
if apt_get_pkgs:
self.sh.run_cmd(
sudo + ['apt-get', 'install'] + y + [LF] +
common.add_newlines(sorted(apt_get_pkgs))
self.sh.add_newlines(sorted(apt_get_pkgs))
)
if apt_build_deps:
self.sh.run_cmd(
sudo +
['apt-get', 'build-dep'] + y + [LF] +
common.add_newlines(sorted(apt_build_deps))
self.sh.add_newlines(sorted(apt_build_deps))
)
if python2_pkgs:
self.sh.run_cmd(
['python', '-m', 'pip', 'install', '--user', LF] +
common.add_newlines(sorted(python2_pkgs))
self.sh.add_newlines(sorted(python2_pkgs))
)
if python3_pkgs:
# Not with pip executable directly:
# https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
self.sh.run_cmd(
['python3', '-m', 'pip', 'install', '--user', LF] +
common.add_newlines(sorted(python3_pkgs))
self.sh.add_newlines(sorted(python3_pkgs))
)
git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
if submodules:
@@ -450,7 +450,7 @@ if kwargs['download_dependencies']:
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
self.sh.run_cmd(
git_cmd_common + ['--', LF] +
common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules)])
self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules)])
)
if submodules_shallow:
# == Shallow cloning.
@@ -474,7 +474,7 @@ if kwargs['download_dependencies']:
#
self.sh.run_cmd(
git_cmd_common + ['--depth', '1', '--', LF] +
common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules_shallow)])
self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules_shallow)])
)
# Do the build.

52
build-baremetal
View File

@@ -8,43 +8,43 @@ class BaremetalComponent(common.Component):
def do_build(self, args):
common.assert_crosstool_ng_supports_arch(kwargs['arch'])
build_dir = self.get_build_dir(args)
bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext))
bootloader_obj = os.path.join(kwargs['baremetal_build_lib_dir'], 'bootloader{}'.format(kwargs['obj_ext']))
common_basename_noext = 'common'
common_src = os.path.join(common.root_dir, common_basename_noext + common.c_ext)
common_obj = os.path.join(common.baremetal_build_lib_dir, common_basename_noext + common.obj_ext)
common_src = os.path.join(kwargs['root_dir'], common_basename_noext + kwargs['c_ext'])
common_obj = os.path.join(kwargs['baremetal_build_lib_dir'], common_basename_noext + kwargs['obj_ext'])
syscalls_basename_noext = 'syscalls'
syscalls_src = os.path.join(common.baremetal_src_lib_dir, syscalls_basename_noext + common.c_ext)
syscalls_obj = os.path.join(common.baremetal_build_lib_dir, syscalls_basename_noext + common.obj_ext)
syscalls_src = os.path.join(kwargs['baremetal_src_lib_dir'], syscalls_basename_noext + kwargs['c_ext'])
syscalls_obj = os.path.join(kwargs['baremetal_build_lib_dir'], syscalls_basename_noext + kwargs['obj_ext'])
common_objs = [common_obj, syscalls_obj]
cflags = [
'-I', common.baremetal_src_lib_dir, LF,
'-I', common.root_dir, LF,
'-I', kwargs['baremetal_src_lib_dir'], LF,
'-I', kwargs['root_dir'], LF,
'-O0', LF,
'-ggdb3', LF,
'-mcpu={}'.format(common.mcpu), LF,
'-mcpu={}'.format(kwargs['mcpu']), LF,
'-nostartfiles', LF,
]
if kwargs['prebuilt']:
gcc = 'arm-none-eabi-gcc'
else:
os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH']
os.environ['PATH'] = kwargs['crosstool_ng_bin_dir'] + os.environ['PATH']
gcc = common.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng'])
if common.emulator == 'gem5':
if common.machine == 'VExpress_GEM5_V1':
if kwargs['emulator'] == 'gem5':
if kwargs['machine'] == 'VExpress_GEM5_V1':
entry_address = 0x80000000
uart_address = 0x1c090000
elif common.machine == 'RealViewPBX':
elif kwargs['machine'] == 'RealViewPBX':
entry_address = 0x10000
uart_address = 0x10009000
else:
raise Exception('unknown machine: ' + common.machine)
raise Exception('unknown machine: ' + kwargs['machine'])
cflags.extend(['-D', 'GEM5'.format(uart_address), LF])
else:
entry_address = 0x40000000
uart_address = 0x09000000
os.makedirs(build_dir, exist_ok=True)
os.makedirs(common.baremetal_build_lib_dir, exist_ok=True)
src = os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(kwargs['arch'], common.asm_ext))
os.makedirs(kwargs['baremetal_build_lib_dir'], exist_ok=True)
src = os.path.join(kwargs['baremetal_src_lib_dir'], '{}{}'.format(kwargs['arch'], kwargs['asm_ext']))
if common.need_rebuild([src], bootloader_obj):
self.sh.run_cmd(
[gcc, LF] +
@@ -87,7 +87,7 @@ class BaremetalComponent(common.Component):
common_objs=common_objs,
)
arch_dir = os.path.join('arch', kwargs['arch'])
if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)):
if os.path.isdir(os.path.join(kwargs['baremetal_src_dir'], arch_dir)):
self._build_dir(
arch_dir,
gcc=gcc,
@@ -97,7 +97,7 @@ class BaremetalComponent(common.Component):
common_objs=common_objs,
)
arch_dir = os.path.join('arch', kwargs['arch'], 'no_bootloader')
if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)):
if os.path.isdir(os.path.join(kwargs['baremetal_src_dir'], arch_dir)):
self._build_dir(
arch_dir,
gcc=gcc,
@@ -116,7 +116,7 @@ Build the baremetal examples with crosstool-NG.
}
def get_build_dir(self, args):
return common.baremetal_build_dir
return kwargs['baremetal_build_dir']
def get_default_args(self):
return {'baremetal': 'all'}
@@ -137,18 +137,18 @@ Build the baremetal examples with crosstool-NG.
Place outputs on the same subpath or the output directory.
"""
in_dir = os.path.join(common.baremetal_src_dir, subpath)
out_dir = os.path.join(common.baremetal_build_dir, subpath)
in_dir = os.path.join(kwargs['baremetal_src_dir'], subpath)
out_dir = os.path.join(kwargs['baremetal_build_dir'], subpath)
os.makedirs(out_dir, exist_ok=True)
common_objs = common_objs.copy()
if bootloader:
common_objs.append(bootloader_obj)
for in_basename in os.listdir(in_dir):
in_path = os.path.join(in_dir, in_basename)
if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext):
if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (kwargs['c_ext'], kwargs['asm_ext']):
in_name = os.path.splitext(in_basename)[0]
main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext))
src = os.path.join(common.baremetal_src_dir, in_path)
main_obj = os.path.join(kwargs['baremetal_build_dir'], subpath, '{}{}'.format(in_name, kwargs['obj_ext']))
src = os.path.join(kwargs['baremetal_src_dir'], in_path)
if common.need_rebuild([src], main_obj):
self.sh.run_cmd(
[gcc, LF] +
@@ -160,8 +160,8 @@ Build the baremetal examples with crosstool-NG.
]
)
objs = common_objs + [main_obj]
out = os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext)
link_script = os.path.join(common.baremetal_src_dir, 'link.ld')
out = os.path.join(kwargs['baremetal_build_dir'], subpath, in_name + kwargs['baremetal_build_ext'])
link_script = os.path.join(kwargs['baremetal_src_dir'], 'link.ld')
if common.need_rebuild(objs + [link_script], out):
self.sh.run_cmd(
[gcc, LF] +
@@ -171,7 +171,7 @@ Build the baremetal examples with crosstool-NG.
'-o', out, LF,
'-T', link_script, LF,
] +
common.add_newlines(objs)
self.sh.add_newlines(objs)
)
if __name__ == '__main__':

52
build-buildroot
View File

@@ -67,11 +67,11 @@ usually extra Buildroot targets.
def do_build(self, args):
build_dir = self.get_build_dir(args)
os.makedirs(common.out_dir, exist_ok=True)
extra_make_args = common.add_newlines(kwargs['extra_make_args'])
os.makedirs(kwargs['out_dir'], exist_ok=True)
extra_make_args = self.sh.add_newlines(kwargs['extra_make_args'])
if kwargs['build_linux']:
extra_make_args.extend(['linux-reconfigure', LF])
if common.emulator == 'gem5':
if kwargs['emulator'] == 'gem5':
extra_make_args.extend(['gem5-reconfigure', LF])
if kwargs['arch'] == 'x86_64':
defconfig = 'qemu_x86_64_defconfig'
@@ -80,24 +80,24 @@ usually extra Buildroot targets.
elif kwargs['arch'] == 'aarch64':
defconfig = 'qemu_aarch64_virt_defconfig'
br2_external_dirs = []
for package_dir in os.listdir(common.packages_dir):
package_dir_abs = os.path.join(common.packages_dir, package_dir)
for package_dir in os.listdir(kwargs['packages_dir']):
package_dir_abs = os.path.join(kwargs['packages_dir'], package_dir)
if os.path.isdir(package_dir_abs):
br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs))
br2_external_str = ':'.join(br2_external_dirs)
self.sh.run_cmd(
[
'make', LF,
'O={}'.format(common.buildroot_build_dir), LF,
'O={}'.format(kwargs['buildroot_build_dir']), LF,
'BR2_EXTERNAL={}'.format(br2_external_str), LF,
defconfig, LF,
],
cwd=common.buildroot_src_dir,
cwd=kwargs['buildroot_src_dir'],
)
configs = kwargs['config']
configs.extend([
'BR2_JLEVEL={}'.format(kwargs['nproc']),
'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir),
'BR2_DL_DIR="{}"'.format(kwargs['buildroot_download_dir']),
])
if not kwargs['build_linux']:
configs.extend([
@@ -107,38 +107,38 @@ usually extra Buildroot targets.
if not kwargs['baseline']:
configs.extend([
'BR2_GLOBAL_PATCH_DIR="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global'))
self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'patches', 'global'))
),
'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment'))
self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'busybox_config_fragment'))
),
'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override'))
self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'buildroot_override'))
),
'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script'))
self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'rootfs-post-build-script'))
),
'BR2_ROOTFS_USERS_TABLES="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table'))
self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'user_table'))
),
])
if not kwargs['no_overlay']:
configs.append('BR2_ROOTFS_OVERLAY="{}"'.format(
self._path_relative_to_buildroot(common.out_rootfs_overlay_dir)
self._path_relative_to_buildroot(kwargs['out_rootfs_overlay_dir'])
))
config_fragments = [
os.path.join(common.root_dir, 'buildroot_config', 'default')
os.path.join(kwargs['root_dir'], 'buildroot_config', 'default')
] + kwargs['config_fragment']
# TODO Can't get rid of these for now with nice fragments on Buildroot:
# http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
self.sh.write_configs(common.buildroot_config_file, configs, config_fragments)
self.sh.write_configs(kwargs['buildroot_config_file'], configs, config_fragments)
self.sh.run_cmd(
[
'make', LF,
'O={}'.format(common.buildroot_build_dir), LF,
'O={}'.format(kwargs['buildroot_build_dir']), LF,
'olddefconfig', LF,
],
cwd=common.buildroot_src_dir,
cwd=kwargs['buildroot_src_dir'],
)
common.make_build_dirs()
if not kwargs['no_all']:
@@ -146,21 +146,21 @@ usually extra Buildroot targets.
self.sh.run_cmd(
[
'make', LF,
'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_source_dir), LF,
'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), LF,
'O={}'.format(common.buildroot_build_dir), LF,
'LKMC_GEM5_SRCDIR="{}"'.format(kwargs['gem5_source_dir']), LF,
'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(kwargs['parsec_benchmark_src_dir']), LF,
'O={}'.format(kwargs['buildroot_build_dir']), LF,
'V={}'.format(int(kwargs['verbose'])), LF,
] +
extra_make_args
,
out_file=os.path.join(common.buildroot_build_dir, 'lkmc.log'),
out_file=os.path.join(kwargs['buildroot_build_dir'], 'lkmc.log'),
delete_env=['LD_LIBRARY_PATH'],
cwd=common.buildroot_src_dir,
cwd=kwargs['buildroot_src_dir'],
)
# Create the qcow2 from ext2.
# Skip if qemu is not present, because gem5 does not need the qcow2.
# so we don't force a QEMU build for gem5.
if not kwargs['no_all'] and os.path.exists(common.qemu_img_executable):
if not kwargs['no_all'] and os.path.exists(kwargs['qemu_img_executable']):
common.raw_to_qcow2()
def get_argparse_args(self):
@@ -171,7 +171,7 @@ Run Linux on an emulator
}
def get_build_dir(self, args):
return common.buildroot_build_dir
return kwargs['buildroot_build_dir']
_defaults = {
'baseline': False,
@@ -184,7 +184,7 @@ Run Linux on an emulator
}
def _path_relative_to_buildroot(self, abspath):
return os.path.relpath(abspath, common.buildroot_src_dir)
return os.path.relpath(abspath, kwargs['buildroot_src_dir'])
if __name__ == '__main__':
BuildrootComponent().build()

30
build-crosstool-ng
View File

@@ -8,20 +8,20 @@ class CrosstoolNgComponent(common.Component):
def do_build(self, args):
common.assert_crosstool_ng_supports_arch(kwargs['arch'])
build_dir = self.get_build_dir(args)
defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig')
os.makedirs(common.crosstool_ng_util_dir, exist_ok=True)
os.makedirs(common.crosstool_ng_download_dir, exist_ok=True)
defconfig_dest = os.path.join(kwargs['crosstool_ng_util_dir'], 'defconfig')
os.makedirs(kwargs['crosstool_ng_util_dir'], exist_ok=True)
os.makedirs(kwargs['crosstool_ng_download_dir'], exist_ok=True)
# Bootstrap out-ot-tree WONTFIX. I've tried.
# https://github.com/crosstool-ng/crosstool-ng/issues/1021
os.chdir(common.crosstool_ng_src_dir)
os.chdir(kwargs['crosstool_ng_src_dir'])
self.sh.run_cmd(
[os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), LF],
[os.path.join(kwargs['crosstool_ng_src_dir'], 'bootstrap'), LF],
)
os.chdir(common.crosstool_ng_util_dir)
os.chdir(kwargs['crosstool_ng_util_dir'])
self.sh.run_cmd(
[
os.path.join(common.crosstool_ng_src_dir, 'configure'), LF,
os.path.join(kwargs['crosstool_ng_src_dir'], 'configure'), LF,
'--enable-local', LF,
],
)
@@ -34,33 +34,33 @@ class CrosstoolNgComponent(common.Component):
# Build the toolchain.
self.sh.cp(
os.path.join(common.root_dir, 'crosstool_ng_config', kwargs['arch']),
os.path.join(kwargs['root_dir'], 'crosstool_ng_config', kwargs['arch']),
defconfig_dest
)
common.write_configs(
common.crosstool_ng_defconfig,
kwargs['crosstool_ng_defconfig'],
[
'CT_PREFIX_DIR="{}"'.format(common.crosstool_ng_install_dir),
'CT_PREFIX_DIR="{}"'.format(kwargs['crosstool_ng_install_dir']),
'CT_WORK_DIR="{}"'.format(build_dir),
'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir),
'CT_LOCAL_TARBALLS_DIR="{}"'.format(kwargs['crosstool_ng_download_dir']),
]
)
self.sh.run_cmd(
[
common.crosstool_ng_executable, LF,
kwargs['crosstool_ng_executable'], LF,
'defconfig', LF,
],
)
os.unlink(defconfig_dest)
self.sh.run_cmd(
[
common.crosstool_ng_executable, LF,
kwargs['crosstool_ng_executable'], LF,
'build', LF,
'CT_JOBS={}'.format(str(kwargs['nproc'])), LF,
],
out_file=os.path.join(build_dir, 'lkmc.log'),
delete_env=['LD_LIBRARY_PATH'],
extra_paths=[common.ccache_dir],
extra_paths=[kwargs['ccache_dir']],
)
def get_argparse_args(self):
@@ -71,7 +71,7 @@ Build crosstool-NG with Newlib for bare metal compilation'
}
def get_build_dir(self, args):
return common.crosstool_ng_build_dir
return kwargs['crosstool_ng_build_dir']
if __name__ == '__main__':
CrosstoolNgComponent().build()

14
build-docker
View File

@@ -44,7 +44,7 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
'--privileged',
'-t',
'-w', target_dir,
'-v', '{}:{}'.format(common.root_dir, target_dir),
'-v', '{}:{}'.format(kwargs['root_dir'], target_dir),
'ubuntu:18.04',
'bash',
])
@@ -52,11 +52,11 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
'docker',
'export',
'-o',
common.docker_tar_file,
kwargs['docker_tar_file'],
container_name,
])
tar = tarfile.open(common.docker_tar_file)
tar.extractall(common.docker_tar_dir)
tar = tarfile.open(kwargs['docker_tar_file'])
tar.extractall(kwargs['docker_tar_dir'])
tar.close()
# sudo not required in theory
# https://askubuntu.com/questions/1046828/how-to-run-libguestfs-tools-tools-such-as-virt-make-fs-without-sudo
@@ -65,13 +65,13 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
'--format', 'raw',
'--size', '+1G',
'--type', 'ext2',
common.docker_tar_dir,
common.docker_rootfs_raw_file,
kwargs['docker_tar_dir'],
kwargs['docker_rootfs_raw_file'],
])
common.raw_to_qcow2(prebuilt=True)
def get_build_dir(self, args):
return common.docker_build_dir
return kwargs['docker_build_dir']
def get_default_args(self):
return {'docker': True}

68
build-gem5
View File

@@ -16,49 +16,46 @@ class Main(common.BuildCliFunction):
nargs='*',
)
def build(self, **kwargs):
build_dir = self.get_build_dir(**kwargs)
binaries_dir = os.path.join(kwargs['gem5_system_dir'], 'binaries')
disks_dir = os.path.join(kwargs['gem5_system_dir'], 'disks')
def build(self):
build_dir = self.get_build_dir()
binaries_dir = os.path.join(self.env['gem5_system_dir'], 'binaries')
disks_dir = os.path.join(self.env['gem5_system_dir'], 'disks')
os.makedirs(binaries_dir, exist_ok=True)
os.makedirs(disks_dir, exist_ok=True)
if kwargs['gem5_source_dir'] is None:
if not os.path.exists(os.path.join(kwargs['gem5_source_dir'], '.git')):
if kwargs['gem5_source_dir'] == kwargs['gem5_default_src_dir']:
if self.env['gem5_source_dir'] is None:
if not os.path.exists(os.path.join(self.env['gem5_source_dir'], '.git')):
if self.env['gem5_source_dir'] == self.env['gem5_default_src_dir']:
raise Exception('gem5 submodule not checked out')
self.sh.run_cmd([
'git', LF,
'-C', kwargs['gem5_default_src_dir'], LF,
'-C', self.env['gem5_default_src_dir'], LF,
'worktree', 'add', LF,
'-b', os.path.join('wt', kwargs['gem5_build_id']), LF,
kwargs['gem5_source_dir'], LF,
'-b', os.path.join('wt', self.env['gem5_build_id']), LF,
self.env['gem5_source_dir'], LF,
])
if kwargs['verbose']:
if self.env['verbose']:
verbose = ['--verbose', LF]
else:
verbose = []
if kwargs['arch'] == 'x86_64':
if self.env['arch'] == 'x86_64':
dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img')
with open(dummy_img_path, 'wb') as dummy_img_file:
zeroes = b'\x00' * (2 ** 16)
for i in range(2 ** 10):
dummy_img_file.write(zeroes)
self.sh.run_cmd(['mkswap', dummy_img_path])
self.sh.run_cmd(['mkswap', dummy_img_path, LF])
with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'):
# This file must always be present, despite --kernel overriding that default and selecting the kernel.
# I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present.
pass
elif kwargs['arch'] == 'arm' or kwargs['arch'] == 'aarch64':
gem5_system_src_dir = os.path.join(kwargs['gem5_source_dir'], 'system')
elif self.env['arch'] == 'arm' or self.env['arch'] == 'aarch64':
gem5_system_src_dir = os.path.join(self.env['gem5_source_dir'], 'system')
# dtb
dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt')
dt_build_dir = os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt')
self.sh.run_cmd([
'make', LF,
'-C', dt_src_dir, LF,
])
common.copy_dir_if_update_non_recursive(
dt_build_dir = os.path.join(self.env['gem5_system_dir'], 'arm', 'dt')
self.sh.run_cmd(['make', '-C', dt_src_dir, LF])
self.sh.copy_dir_if_update_non_recursive(
srcdir=dt_src_dir,
destdir=dt_build_dir,
filter_ext='.dtb',
@@ -78,38 +75,35 @@ class Main(common.BuildCliFunction):
# Bootloader 64.
bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader')
# TODO cross_compile is ignored because the make does not use CC...
self.sh.run_cmd([
'make', LF,
'-C', bootloader64_dir, LF
])
self.sh.run_cmd(['make', '-C', bootloader64_dir, LF])
self.sh.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir)
self.sh.run_cmd(
(
[
'scons', LF,
'-j', str(kwargs['nproc']), LF,
'-j', str(self.env['nproc']), LF,
'--gold-linker', LF,
'--ignore-style', LF,
kwargs['gem5_executable'], LF,
self.env['gem5_executable'], LF,
] +
verbose +
self.sh.add_newlines(kwargs['extra_scons_args'])
self.sh.add_newlines(self.env['extra_scons_args'])
),
cwd=kwargs['gem5_source_dir'],
extra_paths=[kwargs['ccache_dir']],
cwd=self.env['gem5_source_dir'],
extra_paths=[self.env['ccache_dir']],
)
term_src_dir = os.path.join(kwargs['gem5_source_dir'], 'util/term')
term_src_dir = os.path.join(self.env['gem5_source_dir'], 'util/term')
m5term_build = os.path.join(term_src_dir, 'm5term')
self.sh.run_cmd(['make', '-C', term_src_dir])
if os.path.exists(kwargs['gem5_m5term']):
self.sh.run_cmd(['make', '-C', term_src_dir, LF])
if os.path.exists(self.env['gem5_m5term']):
# Otherwise self.sh.cp would fail with "Text file busy" if you
# tried to rebuild while running m5term:
# https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512
os.unlink(kwargs['gem5_m5term'])
self.sh.cp(m5term_build, kwargs['gem5_m5term'])
os.unlink(self.env['gem5_m5term'])
self.sh.cp(m5term_build, self.env['gem5_m5term'])
def get_build_dir(self, **kwargs):
return kwargs['gem5_build_dir']
def get_build_dir(self):
return self.env['gem5_build_dir']
if __name__ == '__main__':
Main().cli()

66
build-linux
View File

@@ -6,8 +6,13 @@ import shutil
import common
from shell_helpers import LF
class LinuxComponent(common.Component):
def __init__(self, parser):
class Main(common.BuildCliFunction):
def __init__(self):
super().__init__(
description='''\
Build the Linux kernel.
'''
)
self.add_argument(
'--config', default=[], action='append',
help='''\
@@ -44,48 +49,48 @@ Configure the kernel, but don't build it.
nargs='*'
)
def build(self, **kwargs):
build_dir = self.get_build_dir(**kwargs)
if kwargs['initrd'] or kwargs['initramfs']:
def build(self):
build_dir = self.get_build_dir()
if self.env['initrd'] or self.env['initramfs']:
raise Exception('just trolling, --initrd and --initramfs are broken for now')
os.makedirs(build_dir, exist_ok=True)
tool = 'gcc'
gcc = common.get_toolchain_tool(tool)
gcc = self.get_toolchain_tool(tool)
prefix = gcc[:-len(tool)]
common_args = {
'cwd': common.linux_src_dir,
'cwd': self.env['linux_src_dir'],
}
ccache = shutil.which('ccache')
if ccache is not None:
cc = '{} {}'.format(ccache, gcc)
else:
cc = gcc
if kwargs['verbose']:
if self.env['verbose']:
verbose = ['V=1']
else:
verbose = []
common_make_args = [
'make', LF,
'-j', str(kwargs['nproc']), LF,
'ARCH={}'.format(common.linux_arch), LF,
'-j', str(self.env['nproc']), LF,
'ARCH={}'.format(self.env['linux_arch']), LF,
'CROSS_COMPILE={}'.format(prefix), LF,
'CC={}'.format(cc), LF,
'O={}'.format(build_dir), LF,
] + verbose
if kwargs['custom_config_file'] is not None:
if not os.path.exists(kwargs['custom_config_file']):
raise Exception('config fragment file does not exist: {}'.format(kwargs['custom_config_file']))
base_config_file = kwargs['custom_config_file']
if self.env['custom_config_file'] is not None:
if not os.path.exists(self.env['custom_config_file']):
raise Exception('config fragment file does not exist: {}'.format(self.env['custom_config_file']))
base_config_file = self.env['custom_config_file']
config_fragments = []
else:
base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(kwargs['arch']))
base_config_file = os.path.join(self.env['linux_config_dir'], 'buildroot-{}'.format(self.env['arch']))
config_fragments = ['min', 'default']
for i, config_fragment in enumerate(config_fragments):
config_fragments[i] = os.path.join(common.linux_config_dir, config_fragment)
config_fragments.extend(kwargs['config_fragment'])
if kwargs['config'] != []:
config_fragments[i] = os.path.join(self.env['linux_config_dir'], config_fragment)
config_fragments.extend(self.env['config_fragment'])
if self.env['config'] != []:
cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment')
cli_config_str = '\n'.join(kwargs['config'])
cli_config_str = '\n'.join(self.env['config'])
common.write_string_to_file(cli_config_fragment_path, cli_config_str)
config_fragments.append(cli_config_fragment_path)
self.sh.cp(
@@ -94,12 +99,12 @@ Configure the kernel, but don't build it.
)
self.sh.run_cmd(
[
os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), LF,
os.path.join(self.env['linux_src_dir'], 'scripts', 'kconfig', 'merge_config.sh'), LF,
'-m', LF,
'-O', build_dir, LF,
os.path.join(build_dir, '.config'), LF,
] +
common.add_newlines(config_fragments)
self.sh.add_newlines(config_fragments)
)
self.sh.run_cmd(
(
@@ -108,11 +113,11 @@ Configure the kernel, but don't build it.
),
**common_args
)
if not kwargs['config_only']:
if not self.env['config_only']:
self.sh.run_cmd(
(
common_make_args +
common.add_newlines(kwargs['extra_make_args'])
self.sh.add_newlines(self.env['extra_make_args'])
),
**common_args
)
@@ -120,7 +125,7 @@ Configure the kernel, but don't build it.
(
common_make_args +
[
'INSTALL_MOD_PATH={}'.format(common.out_rootfs_overlay_dir), LF,
'INSTALL_MOD_PATH={}'.format(self.env['out_rootfs_overlay_dir']), LF,
'modules_install', LF,
]
),
@@ -130,15 +135,8 @@ Configure the kernel, but don't build it.
# TODO Basically all kernel modules also basically leak full host paths. Just terrible. Buildroot deals with that stuff nicely for us.
# common.rmrf()
def get_argparse_args(self):
return {
'description': '''\
Build the Linux kernel.
'''
}
def get_build_dir(self, args):
return common.linux_build_dir
def get_build_dir(self):
return self.env['linux_build_dir']
if __name__ == '__main__':
LinuxComponent().build()
Main().cli()

12
build-m5
View File

@@ -19,25 +19,25 @@ class M5Component(common.Component):
'-f', 'Makefile.{}'.format(arch), LF,
'CC={}'.format(cc), LF,
'LD={}'.format(ld), LF,
'PWD={}'.format(common.gem5_m5_source_dir), LF,
'PWD={}'.format(kwargs['gem5_m5_source_dir']), LF,
]
def do_build(self, args):
os.makedirs(common.gem5_m5_build_dir, exist_ok=True)
os.makedirs(kwargs['gem5_m5_build_dir'], exist_ok=True)
# We must clean first or else the build outputs of one arch can conflict with the other.
# I should stop being lazy and go actually patch gem5 to support out of tree m5 build...
self.clean(args)
self.sh.run_cmd(
self.get_make_cmd(args),
cwd=common.gem5_m5_source_dir,
cwd=kwargs['gem5_m5_source_dir'],
)
os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True)
self.sh.cp(os.path.join(common.gem5_m5_source_dir, 'm5'), common.out_rootfs_overlay_bin_dir)
os.makedirs(kwargs['out_rootfs_overlay_bin_dir'], exist_ok=True)
self.sh.cp(os.path.join(kwargs['gem5_m5_source_dir'], 'm5'), kwargs['out_rootfs_overlay_bin_dir'])
def clean(self, args):
self.sh.run_cmd(
self.get_make_cmd(args) + ['clean', LF],
cwd=common.gem5_m5_source_dir,
cwd=kwargs['gem5_m5_source_dir'],
)
if __name__ == '__main__':

36
build-modules
View File

@@ -42,34 +42,34 @@ Use the host packaged cross toolchain.
# This copies only modified files as per:
# https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory
distutils.dir_util.copy_tree(
common.kernel_modules_src_dir,
os.path.join(build_dir, common.kernel_modules_subdir),
kwargs['kernel_modules_src_dir'],
os.path.join(build_dir, kwargs['kernel_modules_subdir']),
update=1,
)
distutils.dir_util.copy_tree(
common.include_src_dir,
os.path.join(build_dir, common.include_subdir),
kwargs['include_src_dir'],
os.path.join(build_dir, kwargs['include_subdir']),
update=1,
)
all_kernel_modules = []
for basename in os.listdir(common.kernel_modules_src_dir):
src = os.path.join(common.kernel_modules_src_dir, basename)
for basename in os.listdir(kwargs['kernel_modules_src_dir']):
src = os.path.join(kwargs['kernel_modules_src_dir'], basename)
if os.path.isfile(src):
noext, ext = os.path.splitext(basename)
if ext == common.c_ext:
if ext == kwargs['c_ext']:
all_kernel_modules.append(noext)
if kwargs['kernel_modules'] == []:
kernel_modules = all_kernel_modules
else:
kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], kwargs['kernel_modules'])
object_files = map(lambda x: x + common.obj_ext, kernel_modules)
object_files = map(lambda x: x + kwargs['obj_ext'], kernel_modules)
tool = 'gcc'
if kwargs['host']:
allowed_toolchains = ['host']
build_subdir = common.kernel_modules_build_host_subdir
build_subdir = kwargs['kernel_modules_build_host_subdir']
else:
allowed_toolchains = None
build_subdir = common.kernel_modules_build_subdir
build_subdir = kwargs['kernel_modules_build_subdir']
gcc = common.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains)
prefix = gcc[:-len(tool)]
ccache = shutil.which('ccache')
@@ -84,13 +84,13 @@ Use the host packaged cross toolchain.
if kwargs['host']:
linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build')
else:
linux_dir = common.linux_build_dir
linux_dir = kwargs['linux_build_dir']
self.sh.run_cmd(
(
[
'make', LF,
'-j', str(kwargs['nproc']), LF,
'ARCH={}'.format(common.linux_arch), LF,
'ARCH={}'.format(kwargs['linux_arch']), LF,
'CC={}'.format(cc), LF,
'CROSS_COMPILE={}'.format(prefix), LF,
'LINUX_DIR={}'.format(linux_dir), LF,
@@ -100,13 +100,13 @@ Use the host packaged cross toolchain.
common.shlex_split(kwargs['make_args']) +
verbose
),
cwd=os.path.join(common.kernel_modules_build_subdir),
cwd=os.path.join(kwargs['kernel_modules_build_subdir']),
)
if not kwargs['host']:
common.copy_dir_if_update_non_recursive(
srcdir=common.kernel_modules_build_subdir,
destdir=common.out_rootfs_overlay_dir,
filter_ext=common.kernel_module_ext,
srcdir=kwargs['kernel_modules_build_subdir'],
destdir=kwargs['out_rootfs_overlay_dir'],
filter_ext=kwargs['kernel_module_ext'],
)
def get_argparse_args(self):
@@ -120,9 +120,9 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host
def get_build_dir(self, args):
if kwargs['host']:
return os.path.join(common.kernel_modules_build_host_dir)
return os.path.join(kwargs['kernel_modules_build_host_dir'])
else:
return os.path.join(common.kernel_modules_build_dir)
return os.path.join(kwargs['kernel_modules_build_dir'])
if __name__ == '__main__':
ModulesComponent().build()

10
build-qemu
View File

@@ -32,15 +32,15 @@ class QemuComponent(common.Component):
target_list = '{}-softmmu'.format(kwargs['arch'])
self.sh.run_cmd(
[
os.path.join(common.qemu_src_dir, 'configure'), LF,
os.path.join(kwargs['qemu_src_dir'], 'configure'), LF,
'--enable-debug', LF,
'--enable-trace-backends=simple', LF,
'--target-list={}'.format(target_list), LF,
'--enable-sdl', LF,
'--with-sdlabi=2.0', LF,
] +
common.add_newlines(kwargs['extra_config_args']),
extra_paths=[common.ccache_dir],
self.sh.add_newlines(kwargs['extra_config_args']),
extra_paths=[kwargs['ccache_dir']],
cwd=build_dir
)
self.sh.run_cmd(
@@ -53,11 +53,11 @@ class QemuComponent(common.Component):
verbose
),
cwd=build_dir,
extra_paths=[common.ccache_dir],
extra_paths=[kwargs['ccache_dir']],
)
def get_build_dir(self, args):
return common.qemu_build_dir
return kwargs['qemu_build_dir']
if __name__ == '__main__':
QemuComponent().build()

22
build-userland
View File

@@ -59,25 +59,25 @@ has the OpenBLAS libraries and headers installed.
'make', LF,
'-j', str(kwargs['nproc']), LF,
'ARCH={}'.format(kwargs['arch']), LF,
'CCFLAGS_SCRIPT={} {}'.format('-I', common.userland_src_dir), LF,
'COMMON_DIR={}'.format(common.root_dir), LF,
'CCFLAGS_SCRIPT={} {}'.format('-I', kwargs['userland_src_dir']), LF,
'COMMON_DIR={}'.format(kwargs['root_dir']), LF,
'CC={}'.format(cc), LF,
'CXX={}'.format(cxx), LF,
'PKG_CONFIG={}'.format(common.buildroot_pkg_config), LF,
'STAGING_DIR={}'.format(common.buildroot_staging_dir), LF,
'PKG_CONFIG={}'.format(kwargs['buildroot_pkg_config']), LF,
'STAGING_DIR={}'.format(kwargs['buildroot_staging_dir']), LF,
'OUT_DIR={}'.format(build_dir), LF,
] +
common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in kwargs['has_package']]) +
self.sh.add_newlines(['HAS_{}=y'.format(package.upper()) for package in kwargs['has_package']]) +
shlex.split(kwargs['make_args']) +
common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in kwargs['targets']])
self.sh.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + kwargs['userland_build_ext'] for target in kwargs['targets']])
),
cwd=common.userland_src_dir,
extra_paths=[common.ccache_dir],
cwd=kwargs['userland_src_dir'],
extra_paths=[kwargs['ccache_dir']],
)
common.copy_dir_if_update_non_recursive(
srcdir=build_dir,
destdir=common.out_rootfs_overlay_dir,
filter_ext=common.userland_build_ext,
destdir=kwargs['out_rootfs_overlay_dir'],
filter_ext=kwargs['userland_build_ext'],
)
def get_argparse_args(self):
@@ -86,7 +86,7 @@ has the OpenBLAS libraries and headers installed.
}
def get_build_dir(self, args):
return common.userland_build_dir
return kwargs['userland_build_dir']
if __name__ == '__main__':
UserlandComponent().build()

18
cli_function.py
View File

@@ -119,11 +119,11 @@ class CliFunction:
raise Exception('Value not given for mandatory argument: ' + key)
return self.main(**args_with_defaults)
def __init__(self, config_file=None):
def __init__(self, config_file=None, description=None):
self._all_keys = set()
self._arguments = []
self.description = None
self._config_file = config_file
self._description = description
if self._config_file is not None:
self.add_argument(
'--config-file',
@@ -146,7 +146,7 @@ class CliFunction:
to get all arguments.
'''
parser = argparse.ArgumentParser(
description=self.description,
description=self._description,
formatter_class=argparse.RawTextHelpFormatter,
)
for argument in self._arguments:
@@ -177,7 +177,13 @@ class CliFunction:
if __name__ == '__main__':
class OneCliFunction(CliFunction):
def __init__(self):
super().__init__(config_file='cli_function_test_config.py')
super().__init__(
config_file='cli_function_test_config.py',
description = '''\
Description of this
amazing function!
''',
)
self.add_argument('-a', '--asdf', default='A', help='Help for asdf'),
self.add_argument('-q', '--qwer', default='Q', help='Help for qwer'),
self.add_argument('-b', '--bool', default=True, help='Help for bool'),
@@ -186,10 +192,6 @@ if __name__ == '__main__':
self.add_argument('pos-mandatory', help='Help for pos-mandatory', type=int),
self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int),
self.add_argument('args-star', help='Help for args-star', nargs='*'),
self.description = '''\
Description of this
amazing function!
'''
def main(self, **kwargs):
del kwargs['config_file']
return kwargs

684
common.py
View File

@@ -5,7 +5,6 @@ import base64
import collections
import copy
import datetime
import distutils.file_util
import glob
import imp
import json
@@ -99,8 +98,9 @@ class LkmcCliFunction(cli_function.CliFunction):
* command timing
* some common flags, e.g.: --arch, --dry-run, --verbose
'''
def __init__(self):
super().__init__(config_file=common.consts['config_file'])
def __init__(self, *args, **kwargs):
kwargs['config_file'] = consts['config_file']
super().__init__(*args, **kwargs)
# Args for all scripts.
self.add_argument(
@@ -262,24 +262,327 @@ to allow overriding configs from the CLI.
'''
)
def _init_env(self, env):
'''
Update the kwargs from the command line with derived arguments.
'''
def join(*paths):
return os.path.join(*paths)
if env['qemu'] or not env['gem5']:
env['emulator'] = 'qemu'
else:
env['emulator'] = 'gem5'
if env['arch'] in env['arch_short_to_long_dict']:
env['arch'] = env['arch_short_to_long_dict'][env['arch']]
if env['userland_build_id'] is None:
env['userland_build_id'] = env['default_build_id']
env['userland_build_id_given'] = False
else:
env['userland_build_id_given'] = True
if env['gem5_worktree'] is not None and env['gem5_build_id'] is None:
env['gem5_build_id'] = env['gem5_worktree']
env['is_arm'] = False
if env['arch'] == 'arm':
env['armv'] = 7
env['gem5_arch'] = 'ARM'
env['mcpu'] = 'cortex-a15'
env['buildroot_toolchain_prefix'] = 'arm-buildroot-linux-uclibcgnueabihf'
env['crosstool_ng_toolchain_prefix'] = 'arm-unknown-eabi'
env['ubuntu_toolchain_prefix'] = 'arm-linux-gnueabihf'
if env['emulator'] == 'gem5':
if env['machine'] is None:
env['machine'] = 'VExpress_GEM5_V1'
else:
if env['machine'] is None:
env['machine'] = 'virt'
env['is_arm'] = True
elif env['arch'] == 'aarch64':
env['armv'] = 8
env['gem5_arch'] = 'ARM'
env['mcpu'] = 'cortex-a57'
env['buildroot_toolchain_prefix'] = 'aarch64-buildroot-linux-uclibc'
env['crosstool_ng_toolchain_prefix'] = 'aarch64-unknown-elf'
env['ubuntu_toolchain_prefix'] = 'aarch64-linux-gnu'
if env['emulator'] == 'gem5':
if env['machine'] is None:
env['machine'] = 'VExpress_GEM5_V1'
else:
if env['machine'] is None:
env['machine'] = 'virt'
env['is_arm'] = True
elif env['arch'] == 'x86_64':
env['crosstool_ng_toolchain_prefix'] = 'x86_64-unknown-elf'
env['gem5_arch'] = 'X86'
env['buildroot_toolchain_prefix'] = 'x86_64-buildroot-linux-uclibc'
env['ubuntu_toolchain_prefix'] = 'x86_64-linux-gnu'
if env['emulator'] == 'gem5':
if env['machine'] is None:
env['machine'] = 'TODO'
else:
if env['machine'] is None:
env['machine'] = 'pc'
# Buildroot
env['buildroot_build_dir'] = join(env['buildroot_out_dir'], 'build', env['buildroot_build_id'], env['arch'])
env['buildroot_download_dir'] = join(env['buildroot_out_dir'], 'download')
env['buildroot_config_file'] = join(env['buildroot_build_dir'], '.config')
env['buildroot_build_build_dir'] = join(env['buildroot_build_dir'], 'build')
env['buildroot_linux_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom')
env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], "vmlinux")
env['host_dir'] = join(env['buildroot_build_dir'], 'host')
env['host_bin_dir'] = join(env['host_dir'], 'usr', 'bin')
env['buildroot_pkg_config'] = join(env['host_bin_dir'], 'pkg-config')
env['buildroot_images_dir'] = join(env['buildroot_build_dir'], 'images')
env['buildroot_rootfs_raw_file'] = join(env['buildroot_images_dir'], 'rootfs.ext2')
env['buildroot_qcow2_file'] = env['buildroot_rootfs_raw_file'] + '.qcow2'
env['staging_dir'] = join(env['out_dir'], 'staging', env['arch'])
env['buildroot_staging_dir'] = join(env['buildroot_build_dir'], 'staging')
env['target_dir'] = join(env['buildroot_build_dir'], 'target')
env['linux_buildroot_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom')
# QEMU
env['qemu_build_dir'] = join(env['out_dir'], 'qemu', env['qemu_build_id'])
env['qemu_executable_basename'] = 'qemu-system-{}'.format(env['arch'])
env['qemu_executable'] = join(env['qemu_build_dir'], '{}-softmmu'.format(env['arch']), env['qemu_executable_basename'])
env['qemu_img_basename'] = 'qemu-img'
env['qemu_img_executable'] = join(env['qemu_build_dir'], env['qemu_img_basename'])
# gem5
if env['gem5_build_dir'] is None:
env['gem5_build_dir'] = join(env['gem5_out_dir'], env['gem5_build_id'], env['gem5_build_type'])
env['gem5_fake_iso'] = join(env['gem5_out_dir'], 'fake.iso')
env['gem5_m5term'] = join(env['gem5_build_dir'], 'm5term')
env['gem5_build_build_dir'] = join(env['gem5_build_dir'], 'build')
env['gem5_executable'] = join(env['gem5_build_build_dir'], env['gem5_arch'], 'gem5.{}'.format(env['gem5_build_type']))
env['gem5_system_dir'] = join(env['gem5_build_dir'], 'system')
# gem5 source
if env['gem5_source_dir'] is not None:
assert os.path.exists(env['gem5_source_dir'])
else:
if env['gem5_worktree'] is not None:
env['gem5_source_dir'] = join(env['gem5_non_default_src_root_dir'], env['gem5_worktree'])
else:
env['gem5_source_dir'] = env['gem5_default_src_dir']
env['gem5_m5_source_dir'] = join(env['gem5_source_dir'], 'util', 'm5')
env['gem5_config_dir'] = join(env['gem5_source_dir'], 'configs')
env['gem5_se_file'] = join(env['gem5_config_dir'], 'example', 'se.py')
env['gem5_fs_file'] = join(env['gem5_config_dir'], 'example', 'fs.py')
# crosstool-ng
env['crosstool_ng_buildid_dir'] = join(env['crosstool_ng_out_dir'], 'build', env['crosstool_ng_build_id'])
env['crosstool_ng_install_dir'] = join(env['crosstool_ng_buildid_dir'], 'install', env['arch'])
env['crosstool_ng_bin_dir'] = join(env['crosstool_ng_install_dir'], 'bin')
env['crosstool_ng_util_dir'] = join(env['crosstool_ng_buildid_dir'], 'util')
env['crosstool_ng_config'] = join(env['crosstool_ng_util_dir'], '.config')
env['crosstool_ng_defconfig'] = join(env['crosstool_ng_util_dir'], 'defconfig')
env['crosstool_ng_executable'] = join(env['crosstool_ng_util_dir'], 'ct-ng')
env['crosstool_ng_build_dir'] = join(env['crosstool_ng_buildid_dir'], 'build')
env['crosstool_ng_download_dir'] = join(env['crosstool_ng_out_dir'], 'download')
# run
env['gem5_run_dir'] = join(env['run_dir_base'], 'gem5', env['arch'], str(env['run_id']))
env['m5out_dir'] = join(env['gem5_run_dir'], 'm5out')
env['stats_file'] = join(env['m5out_dir'], 'stats.txt')
env['gem5_trace_txt_file'] = join(env['m5out_dir'], 'trace.txt')
env['gem5_guest_terminal_file'] = join(env['m5out_dir'], 'system.terminal')
env['gem5_readfile'] = join(env['gem5_run_dir'], 'readfile')
env['gem5_termout_file'] = join(env['gem5_run_dir'], 'termout.txt')
env['qemu_run_dir'] = join(env['run_dir_base'], 'qemu', env['arch'], str(env['run_id']))
env['qemu_termout_file'] = join(env['qemu_run_dir'], 'termout.txt')
env['qemu_trace_basename'] = 'trace.bin'
env['qemu_trace_file'] = join(env['qemu_run_dir'], 'trace.bin')
env['qemu_trace_txt_file'] = join(env['qemu_run_dir'], 'trace.txt')
env['qemu_rrfile'] = join(env['qemu_run_dir'], 'rrfile')
env['gem5_out_dir'] = join(env['out_dir'], 'gem5')
# Ports
if env['port_offset'] is None:
try:
env['port_offset'] = int(env['run_id'])
except ValueError:
env['port_offset'] = 0
if env['emulator'] == 'gem5':
env['gem5_telnet_port'] = 3456 + env['port_offset']
env['gdb_port'] = 7000 + env['port_offset']
else:
env['qemu_base_port'] = 45454 + 10 * env['port_offset']
env['qemu_monitor_port'] = env['qemu_base_port'] + 0
env['qemu_hostfwd_generic_port'] = env['qemu_base_port'] + 1
env['qemu_hostfwd_ssh_port'] = env['qemu_base_port'] + 2
env['qemu_gdb_port'] = env['qemu_base_port'] + 3
env['extra_serial_port'] = env['qemu_base_port'] + 4
env['gdb_port'] = env['qemu_gdb_port']
env['qemu_background_serial_file'] = join(env['qemu_run_dir'], 'background.log')
# gem5 QEMU polymorphism.
if env['emulator'] == 'gem5':
env['executable'] = env['gem5_executable']
env['run_dir'] = env['gem5_run_dir']
env['termout_file'] = env['gem5_termout_file']
env['guest_terminal_file'] = env['gem5_guest_terminal_file']
env['trace_txt_file'] = env['gem5_trace_txt_file']
else:
env['executable'] = env['qemu_executable']
env['run_dir'] = env['qemu_run_dir']
env['termout_file'] = env['qemu_termout_file']
env['trace_txt_file'] = env['qemu_trace_txt_file']
env['run_cmd_file'] = join(env['run_dir'], 'run.sh')
# Linux kernl.
if 'linux_build_id' in env:
env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch'])
env['lkmc_vmlinux'] = join(env['linux_build_dir'], "vmlinux")
if env['arch'] == 'arm':
env['linux_arch'] = 'arm'
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage')
elif env['arch'] == 'aarch64':
env['linux_arch'] = 'arm64'
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'Image')
elif env['arch'] == 'x86_64':
env['linux_arch'] = 'x86'
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'bzImage')
env['lkmc_linux_image'] = join(env['linux_build_dir'], env['linux_image_prefix'])
env['buildroot_linux_image'] = join(env['buildroot_linux_build_dir'], env['linux_image_prefix'])
if env['buildroot_linux']:
env['vmlinux'] = env['buildroot_vmlinux']
env['linux_image'] = env['buildroot_linux_image']
else:
env['vmlinux'] = env['lkmc_vmlinux']
env['linux_image'] = env['lkmc_linux_image']
# Kernel modules.
env['kernel_modules_build_dir'] = join(env['kernel_modules_build_base_dir'], env['arch'])
env['kernel_modules_build_subdir'] = join(env['kernel_modules_build_dir'], env['kernel_modules_subdir'])
env['kernel_modules_build_host_dir'] = join(env['kernel_modules_build_base_dir'], 'host')
env['kernel_modules_build_host_subdir'] = join(env['kernel_modules_build_host_dir'], env['kernel_modules_subdir'])
env['userland_build_dir'] = join(env['out_dir'], 'userland', env['userland_build_id'], env['arch'])
env['out_rootfs_overlay_dir'] = join(env['out_dir'], 'rootfs_overlay', env['arch'])
env['out_rootfs_overlay_bin_dir'] = join(env['out_rootfs_overlay_dir'], 'bin')
# Baremetal.
env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal')
env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename'])
if env['emulator'] == 'gem5':
env['simulator_name'] = 'gem5'
else:
env['simulator_name'] = 'qemu'
env['baremetal_build_dir'] = join(env['out_dir'], 'baremetal', env['arch'], env['simulator_name'], env['machine'])
env['baremetal_build_lib_dir'] = join(env['baremetal_build_dir'], env['baremetal_lib_basename'])
env['baremetal_build_ext'] = '.elf'
# Docker
env['docker_build_dir'] = join(env['out_dir'], 'docker', env['arch'])
env['docker_tar_dir'] = join(env['docker_build_dir'], 'export')
env['docker_tar_file'] = join(env['docker_build_dir'], 'export.tar')
env['docker_rootfs_raw_file'] = join(env['docker_build_dir'], 'export.ext2')
env['docker_qcow2_file'] = join(env['docker_rootfs_raw_file'] + '.qcow2')
if env['docker']:
env['rootfs_raw_file'] = env['docker_rootfs_raw_file']
env['qcow2_file'] = env['docker_qcow2_file']
else:
env['rootfs_raw_file'] = env['buildroot_rootfs_raw_file']
env['qcow2_file'] = env['buildroot_qcow2_file']
# Image.
if env['baremetal'] is None:
if env['emulator'] == 'gem5':
env['image'] = env['vmlinux']
env['disk_image'] = env['rootfs_raw_file']
else:
env['image'] = env['linux_image']
env['disk_image'] = env['qcow2_file']
else:
env['disk_image'] = env['gem5_fake_iso']
if env['baremetal'] == 'all':
path = env['baremetal']
else:
path = resolve_executable(
env['baremetal'],
env['baremetal_src_dir'],
env['baremetal_build_dir'],
env['baremetal_build_ext'],
)
source_path_noext = os.path.splitext(join(
env['baremetal_src_dir'],
os.path.relpath(path, env['baremetal_build_dir'])
))[0]
for ext in [c_ext, asm_ext]:
source_path = source_path_noext + ext
if os.path.exists(source_path):
env['source_path'] = source_path
break
env['image'] = path
self.env = env
def get_elf_entry(self, elf_file_path):
readelf_header = subprocess.check_output([
self.get_toolchain_tool('readelf'),
'-h',
elf_file_path
])
for line in readelf_header.decode().split('\n'):
split = line.split()
if line.startswith(' Entry point address:'):
addr = line.split()[-1]
break
return int(addr, 0)
def get_toolchain_prefix(self, tool, allowed_toolchains=None):
buildroot_full_prefix = os.path.join(self.env['host_bin_dir'], self.env['buildroot_toolchain_prefix'])
buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
crosstool_ng_full_prefix = os.path.join(self.env['crosstool_ng_bin_dir'], self.env['crosstool_ng_toolchain_prefix'])
crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
host_tool = '{}-{}'.format(self.env['ubuntu_toolchain_prefix'], tool)
host_path = shutil.which(host_tool)
if host_path is not None:
host_exists = True
host_full_prefix = host_path[:-(len(tool)+1)]
else:
host_exists = False
host_full_prefix = None
known_toolchains = {
'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix),
'buildroot': (buildroot_exists, buildroot_full_prefix),
'host': (host_exists, host_full_prefix),
}
if allowed_toolchains is None:
if self.env['baremetal'] is None:
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
else:
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
tried = []
for toolchain in allowed_toolchains:
exists, prefix = known_toolchains[toolchain]
tried.append('{}-{}'.format(prefix, tool))
if exists:
return prefix
raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))
def get_toolchain_tool(self, tool, allowed_toolchains=None):
return '{}-{}'.format(self.get_toolchain_prefix(tool, allowed_toolchains), tool)
def main(self, **kwargs):
'''
Time the main of the derived class.
'''
if not kwargs['dry_run']:
start_time = time.time()
kwargs.update(common.consts)
kwargs = set_kwargs(kwargs)
self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run'])
self.timed_main(**kwargs)
kwargs.update(consts)
self._init_env(kwargs)
self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
self.timed_main()
if not kwargs['dry_run']:
end_time = time.time()
common.print_time(end_time - start_time)
print_time(end_time - start_time)
def run_cmd(self, *args, **kwargs):
self.sh.run_cmd(*args, **kwargs)
def timed_main(self, **kwargs):
def timed_main(self):
'''
Main action of the derived class.
'''
raise NotImplementedError()
class BuildCliFunction(LkmcCliFunction):
@@ -289,8 +592,8 @@ class BuildCliFunction(LkmcCliFunction):
* `--clean` to clean the build directory
* `--nproc` to set he number of build threads
'''
def __init__(self):
super().__init__()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.add_argument(
'--clean',
default=False,
@@ -304,33 +607,33 @@ class BuildCliFunction(LkmcCliFunction):
help='Number of processors to use for the build.',
)
def clean(self, **kwargs):
build_dir = self.get_build_dir(kwargs)
def clean(self):
build_dir = self.get_build_dir()
if build_dir is not None:
common.rmrf(build_dir)
self.sh.rmrf(build_dir)
def build(self, **kwargs):
def build(self):
'''
Do the actual main build work.
'''
raise NotImplementedError()
def get_build_dir(self, **kwargs):
def get_build_dir(self):
return None
def timed_main(self, **kwargs):
def timed_main(self):
'''
Parse CLI, and to the build based on it.
The actual build work is done by do_build in implementing classes.
'''
if kwargs['clean']:
self.clean(kwargs)
if self.env['clean']:
self.clean()
else:
self.build(**kwargs)
self.build()
def assert_crosstool_ng_supports_arch(arch):
if arch not in common.crosstool_ng_supported_archs:
if arch not in kwargs['crosstool_ng_supported_archs']:
raise Exception('arch not yet supported: ' + arch)
def base64_encode(string):
@@ -340,29 +643,16 @@ def gem_list_checkpoint_dirs():
'''
List checkpoint directory, oldest first.
'''
prefix_re = re.compile(common.gem5_cpt_prefix)
files = list(filter(lambda x: os.path.isdir(os.path.join(common.m5out_dir, x)) and prefix_re.search(x), os.listdir(common.m5out_dir)))
files.sort(key=lambda x: os.path.getmtime(os.path.join(common.m5out_dir, x)))
prefix_re = re.compile(kwargs['gem5_cpt_prefix'])
files = list(filter(lambda x: os.path.isdir(os.path.join(kwargs['m5out_dir'], x)) and prefix_re.search(x), os.listdir(kwargs['m5out_dir'])))
files.sort(key=lambda x: os.path.getmtime(os.path.join(kwargs['m5out_dir'], x)))
return files
def get_elf_entry(elf_file_path):
readelf_header = subprocess.check_output([
common.get_toolchain_tool('readelf'),
'-h',
elf_file_path
])
for line in readelf_header.decode().split('\n'):
split = line.split()
if line.startswith(' Entry point address:'):
addr = line.split()[-1]
break
return int(addr, 0)
def get_stats(stat_re=None, stats_file=None):
if stat_re is None:
stat_re = '^system.cpu[0-9]*.numCycles$'
if stats_file is None:
stats_file = common.stats_file
stats_file = kwargs['stats_file']
stat_re = re.compile(stat_re)
ret = []
with open(stats_file, 'r') as statfile:
@@ -373,40 +663,6 @@ def get_stats(stat_re=None, stats_file=None):
ret.append(cols[1])
return ret
def get_toolchain_prefix(tool, allowed_toolchains=None):
buildroot_full_prefix = os.path.join(common.host_bin_dir, common.buildroot_toolchain_prefix)
buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
crosstool_ng_full_prefix = os.path.join(common.crosstool_ng_bin_dir, common.crosstool_ng_toolchain_prefix)
crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
host_tool = '{}-{}'.format(common.ubuntu_toolchain_prefix, tool)
host_path = shutil.which(host_tool)
if host_path is not None:
host_exists = True
host_full_prefix = host_path[:-(len(tool)+1)]
else:
host_exists = False
host_full_prefix = None
known_toolchains = {
'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix),
'buildroot': (buildroot_exists, buildroot_full_prefix),
'host': (host_exists, host_full_prefix),
}
if allowed_toolchains is None:
if common['baremetal'] is None:
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
else:
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
tried = []
for toolchain in allowed_toolchains:
exists, prefix = known_toolchains[toolchain]
tried.append('{}-{}'.format(prefix, tool))
if exists:
return prefix
raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))
def get_toolchain_tool(tool, allowed_toolchains=None):
return '{}-{}'.format(common.get_toolchain_prefix(tool, allowed_toolchains), tool)
def github_make_request(
authenticate=False,
data=None,
@@ -441,18 +697,18 @@ def log_error(msg):
print('error: {}'.format(msg), file=sys.stderr)
def make_build_dirs():
os.makedirs(common.buildroot_build_build_dir, exist_ok=True)
os.makedirs(common.gem5_build_dir, exist_ok=True)
os.makedirs(common.out_rootfs_overlay_dir, exist_ok=True)
os.makedirs(kwargs['buildroot_build_build_dir'], exist_ok=True)
os.makedirs(kwargs['gem5_build_dir'], exist_ok=True)
os.makedirs(kwargs['out_rootfs_overlay_dir'], exist_ok=True)
def make_run_dirs():
'''
Make directories required for the run.
The user could nuke those anytime between runs to try and clean things up.
'''
os.makedirs(common.gem5_run_dir, exist_ok=True)
os.makedirs(common.p9_dir, exist_ok=True)
os.makedirs(common.qemu_run_dir, exist_ok=True)
os.makedirs(kwargs['gem5_run_dir'], exist_ok=True)
os.makedirs(kwargs['p9_dir'], exist_ok=True)
os.makedirs(kwargs['qemu_run_dir'], exist_ok=True)
def need_rebuild(srcs, dst):
if not os.path.exists(dst):
@@ -468,17 +724,17 @@ def print_time(ellapsed_seconds):
print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))
def raw_to_qcow2(prebuilt=False, reverse=False):
if prebuilt or not os.path.exists(common.qemu_img_executable):
if prebuilt or not os.path.exists(kwargs['qemu_img_executable']):
disable_trace = []
qemu_img_executable = common.qemu_img_basename
qemu_img_executable = kwargs['qemu_img_basename']
else:
# Prevent qemu-img from generating trace files like QEMU. Disgusting.
disable_trace = ['-T', 'pr_manager_run,file=/dev/null', LF,]
qemu_img_executable = common.qemu_img_executable
qemu_img_executable = kwargs['qemu_img_executable']
infmt = 'raw'
outfmt = 'qcow2'
infile = common.rootfs_raw_file
outfile = common.qcow2_file
infile = kwargs['rootfs_raw_file']
outfile = kwargs['qcow2_file']
if reverse:
tmp = infmt
infmt = outfmt
@@ -525,263 +781,9 @@ def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext):
raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths))
def resolve_userland(path):
return common.resolve_executable(
return resolve_executable(
path,
common.userland_src_dir,
common.userland_build_dir,
common.userland_build_ext,
kwargs['userland_src_dir'],
kwargs['userland_build_dir'],
kwargs['userland_build_ext'],
)
def set_kwargs(kwargs):
'''
Update the kwargs from the command line with derived arguments.
'''
def join(*paths):
return os.path.join(*paths)
kwargs = collections.defaultdict(lambda: None, **kwargs)
if kwargs['qemu'] or not kwargs['gem5']:
kwargs['emulator'] = 'qemu'
else:
kwargs['emulator'] = 'gem5'
if kwargs['arch'] in kwargs['arch_short_to_long_dict']:
kwargs['arch'] = kwargs['arch_short_to_long_dict'][kwargs['arch']]
if kwargs['userland_build_id'] is None:
kwargs['userland_build_id'] = kwargs['default_build_id']
kwargs['userland_build_id_given'] = False
else:
kwargs['userland_build_id_given'] = True
if kwargs['gem5_worktree'] is not None and kwargs['gem5_build_id'] is None:
kwargs['gem5_build_id'] = kwargs['gem5_worktree']
kwargs['is_arm'] = False
if kwargs['arch'] == 'arm':
kwargs['armv'] = 7
kwargs['gem5_arch'] = 'ARM'
kwargs['mcpu'] = 'cortex-a15'
kwargs['buildroot_toolchain_prefix'] = 'arm-buildroot-linux-uclibcgnueabihf'
kwargs['crosstool_ng_toolchain_prefix'] = 'arm-unknown-eabi'
kwargs['ubuntu_toolchain_prefix'] = 'arm-linux-gnueabihf'
if kwargs['emulator'] == 'gem5':
if kwargs['machine'] is None:
kwargs['machine'] = 'VExpress_GEM5_V1'
else:
if kwargs['machine'] is None:
kwargs['machine'] = 'virt'
kwargs['is_arm'] = True
elif kwargs['arch'] == 'aarch64':
kwargs['armv'] = 8
kwargs['gem5_arch'] = 'ARM'
kwargs['mcpu'] = 'cortex-a57'
kwargs['buildroot_toolchain_prefix'] = 'aarch64-buildroot-linux-uclibc'
kwargs['crosstool_ng_toolchain_prefix'] = 'aarch64-unknown-elf'
kwargs['ubuntu_toolchain_prefix'] = 'aarch64-linux-gnu'
if kwargs['emulator'] == 'gem5':
if kwargs['machine'] is None:
kwargs['machine'] = 'VExpress_GEM5_V1'
else:
if kwargs['machine'] is None:
kwargs['machine'] = 'virt'
kwargs['is_arm'] = True
elif kwargs['arch'] == 'x86_64':
kwargs['crosstool_ng_toolchain_prefix'] = 'x86_64-unknown-elf'
kwargs['gem5_arch'] = 'X86'
kwargs['buildroot_toolchain_prefix'] = 'x86_64-buildroot-linux-uclibc'
kwargs['ubuntu_toolchain_prefix'] = 'x86_64-linux-gnu'
if kwargs['emulator'] == 'gem5':
if kwargs['machine'] is None:
kwargs['machine'] = 'TODO'
else:
if kwargs['machine'] is None:
kwargs['machine'] = 'pc'
# Buildroot
kwargs['buildroot_build_dir'] = join(kwargs['buildroot_out_dir'], 'build', kwargs['buildroot_build_id'], kwargs['arch'])
kwargs['buildroot_download_dir'] = join(kwargs['buildroot_out_dir'], 'download')
kwargs['buildroot_config_file'] = join(kwargs['buildroot_build_dir'], '.config')
kwargs['buildroot_build_build_dir'] = join(kwargs['buildroot_build_dir'], 'build')
kwargs['buildroot_linux_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom')
kwargs['buildroot_vmlinux'] = join(kwargs['buildroot_linux_build_dir'], "vmlinux")
kwargs['host_dir'] = join(kwargs['buildroot_build_dir'], 'host')
kwargs['host_bin_dir'] = join(kwargs['host_dir'], 'usr', 'bin')
kwargs['buildroot_pkg_config'] = join(kwargs['host_bin_dir'], 'pkg-config')
kwargs['buildroot_images_dir'] = join(kwargs['buildroot_build_dir'], 'images')
kwargs['buildroot_rootfs_raw_file'] = join(kwargs['buildroot_images_dir'], 'rootfs.ext2')
kwargs['buildroot_qcow2_file'] = kwargs['buildroot_rootfs_raw_file'] + '.qcow2'
kwargs['staging_dir'] = join(kwargs['out_dir'], 'staging', kwargs['arch'])
kwargs['buildroot_staging_dir'] = join(kwargs['buildroot_build_dir'], 'staging')
kwargs['target_dir'] = join(kwargs['buildroot_build_dir'], 'target')
kwargs['linux_buildroot_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom')
# QEMU
kwargs['qemu_build_dir'] = join(kwargs['out_dir'], 'qemu', kwargs['qemu_build_id'])
kwargs['qemu_executable_basename'] = 'qemu-system-{}'.format(kwargs['arch'])
kwargs['qemu_executable'] = join(kwargs['qemu_build_dir'], '{}-softmmu'.format(kwargs['arch']), kwargs['qemu_executable_basename'])
kwargs['qemu_img_basename'] = 'qemu-img'
kwargs['qemu_img_executable'] = join(kwargs['qemu_build_dir'], kwargs['qemu_img_basename'])
# gem5
if kwargs['gem5_build_dir'] is None:
kwargs['gem5_build_dir'] = join(kwargs['gem5_out_dir'], kwargs['gem5_build_id'], kwargs['gem5_build_type'])
kwargs['gem5_fake_iso'] = join(kwargs['gem5_out_dir'], 'fake.iso')
kwargs['gem5_m5term'] = join(kwargs['gem5_build_dir'], 'm5term')
kwargs['gem5_build_build_dir'] = join(kwargs['gem5_build_dir'], 'build')
kwargs['gem5_executable'] = join(kwargs['gem5_build_build_dir'], kwargs['gem5_arch'], 'gem5.{}'.format(kwargs['gem5_build_type']))
kwargs['gem5_system_dir'] = join(kwargs['gem5_build_dir'], 'system')
# gem5 source
if kwargs['gem5_source_dir'] is not None:
assert os.path.exists(kwargs['gem5_source_dir'])
else:
if kwargs['gem5_worktree'] is not None:
kwargs['gem5_source_dir'] = join(kwargs['gem5_non_default_src_root_dir'], kwargs['gem5_worktree'])
else:
kwargs['gem5_source_dir'] = kwargs['gem5_default_src_dir']
kwargs['gem5_m5_source_dir'] = join(kwargs['gem5_source_dir'], 'util', 'm5')
kwargs['gem5_config_dir'] = join(kwargs['gem5_source_dir'], 'configs')
kwargs['gem5_se_file'] = join(kwargs['gem5_config_dir'], 'example', 'se.py')
kwargs['gem5_fs_file'] = join(kwargs['gem5_config_dir'], 'example', 'fs.py')
# crosstool-ng
kwargs['crosstool_ng_buildid_dir'] = join(kwargs['crosstool_ng_out_dir'], 'build', kwargs['crosstool_ng_build_id'])
kwargs['crosstool_ng_install_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'install', kwargs['arch'])
kwargs['crosstool_ng_bin_dir'] = join(kwargs['crosstool_ng_install_dir'], 'bin')
kwargs['crosstool_ng_util_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'util')
kwargs['crosstool_ng_config'] = join(kwargs['crosstool_ng_util_dir'], '.config')
kwargs['crosstool_ng_defconfig'] = join(kwargs['crosstool_ng_util_dir'], 'defconfig')
kwargs['crosstool_ng_executable'] = join(kwargs['crosstool_ng_util_dir'], 'ct-ng')
kwargs['crosstool_ng_build_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'build')
kwargs['crosstool_ng_download_dir'] = join(kwargs['crosstool_ng_out_dir'], 'download')
# run
kwargs['gem5_run_dir'] = join(kwargs['run_dir_base'], 'gem5', kwargs['arch'], str(kwargs['run_id']))
kwargs['m5out_dir'] = join(kwargs['gem5_run_dir'], 'm5out')
kwargs['stats_file'] = join(kwargs['m5out_dir'], 'stats.txt')
kwargs['gem5_trace_txt_file'] = join(kwargs['m5out_dir'], 'trace.txt')
kwargs['gem5_guest_terminal_file'] = join(kwargs['m5out_dir'], 'system.terminal')
kwargs['gem5_readfile'] = join(kwargs['gem5_run_dir'], 'readfile')
kwargs['gem5_termout_file'] = join(kwargs['gem5_run_dir'], 'termout.txt')
kwargs['qemu_run_dir'] = join(kwargs['run_dir_base'], 'qemu', kwargs['arch'], str(kwargs['run_id']))
kwargs['qemu_termout_file'] = join(kwargs['qemu_run_dir'], 'termout.txt')
kwargs['qemu_trace_basename'] = 'trace.bin'
kwargs['qemu_trace_file'] = join(kwargs['qemu_run_dir'], 'trace.bin')
kwargs['qemu_trace_txt_file'] = join(kwargs['qemu_run_dir'], 'trace.txt')
kwargs['qemu_rrfile'] = join(kwargs['qemu_run_dir'], 'rrfile')
kwargs['gem5_out_dir'] = join(kwargs['out_dir'], 'gem5')
# Ports
if kwargs['port_offset'] is None:
try:
kwargs['port_offset'] = int(kwargs['run_id'])
except ValueError:
kwargs['port_offset'] = 0
if kwargs['emulator'] == 'gem5':
kwargs['gem5_telnet_port'] = 3456 + kwargs['port_offset']
kwargs['gdb_port'] = 7000 + kwargs['port_offset']
else:
kwargs['qemu_base_port'] = 45454 + 10 * kwargs['port_offset']
kwargs['qemu_monitor_port'] = kwargs['qemu_base_port'] + 0
kwargs['qemu_hostfwd_generic_port'] = kwargs['qemu_base_port'] + 1
kwargs['qemu_hostfwd_ssh_port'] = kwargs['qemu_base_port'] + 2
kwargs['qemu_gdb_port'] = kwargs['qemu_base_port'] + 3
kwargs['extra_serial_port'] = kwargs['qemu_base_port'] + 4
kwargs['gdb_port'] = kwargs['qemu_gdb_port']
kwargs['qemu_background_serial_file'] = join(kwargs['qemu_run_dir'], 'background.log')
# gem5 QEMU polymorphism.
if kwargs['emulator'] == 'gem5':
kwargs['executable'] = kwargs['gem5_executable']
kwargs['run_dir'] = kwargs['gem5_run_dir']
kwargs['termout_file'] = kwargs['gem5_termout_file']
kwargs['guest_terminal_file'] = kwargs['gem5_guest_terminal_file']
kwargs['trace_txt_file'] = kwargs['gem5_trace_txt_file']
else:
kwargs['executable'] = kwargs['qemu_executable']
kwargs['run_dir'] = kwargs['qemu_run_dir']
kwargs['termout_file'] = kwargs['qemu_termout_file']
kwargs['trace_txt_file'] = kwargs['qemu_trace_txt_file']
kwargs['run_cmd_file'] = join(kwargs['run_dir'], 'run.sh')
# Linux kernl.
if 'linux_build_id' in kwargs:
kwargs['linux_build_dir'] = join(kwargs['out_dir'], 'linux', kwargs['linux_build_id'], kwargs['arch'])
kwargs['lkmc_vmlinux'] = join(kwargs['linux_build_dir'], "vmlinux")
if kwargs['arch'] == 'arm':
kwargs['linux_arch'] = 'arm'
kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'zImage')
elif kwargs['arch'] == 'aarch64':
kwargs['linux_arch'] = 'arm64'
kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'Image')
elif kwargs['arch'] == 'x86_64':
kwargs['linux_arch'] = 'x86'
kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'bzImage')
kwargs['lkmc_linux_image'] = join(kwargs['linux_build_dir'], kwargs['linux_image_prefix'])
kwargs['buildroot_linux_image'] = join(kwargs['buildroot_linux_build_dir'], kwargs['linux_image_prefix'])
if kwargs['buildroot_linux']:
kwargs['vmlinux'] = kwargs['buildroot_vmlinux']
kwargs['linux_image'] = kwargs['buildroot_linux_image']
else:
kwargs['vmlinux'] = kwargs['lkmc_vmlinux']
kwargs['linux_image'] = kwargs['lkmc_linux_image']
# Kernel modules.
kwargs['kernel_modules_build_dir'] = join(kwargs['kernel_modules_build_base_dir'], kwargs['arch'])
kwargs['kernel_modules_build_subdir'] = join(kwargs['kernel_modules_build_dir'], kwargs['kernel_modules_subdir'])
kwargs['kernel_modules_build_host_dir'] = join(kwargs['kernel_modules_build_base_dir'], 'host')
kwargs['kernel_modules_build_host_subdir'] = join(kwargs['kernel_modules_build_host_dir'], kwargs['kernel_modules_subdir'])
kwargs['userland_build_dir'] = join(kwargs['out_dir'], 'userland', kwargs['userland_build_id'], kwargs['arch'])
kwargs['out_rootfs_overlay_dir'] = join(kwargs['out_dir'], 'rootfs_overlay', kwargs['arch'])
kwargs['out_rootfs_overlay_bin_dir'] = join(kwargs['out_rootfs_overlay_dir'], 'bin')
# Baremetal.
kwargs['baremetal_src_dir'] = join(kwargs['root_dir'], 'baremetal')
kwargs['baremetal_src_lib_dir'] = join(kwargs['baremetal_src_dir'], kwargs['baremetal_lib_basename'])
if kwargs['emulator'] == 'gem5':
kwargs['simulator_name'] = 'gem5'
else:
kwargs['simulator_name'] = 'qemu'
kwargs['baremetal_build_dir'] = join(kwargs['out_dir'], 'baremetal', kwargs['arch'], kwargs['simulator_name'], kwargs['machine'])
kwargs['baremetal_build_lib_dir'] = join(kwargs['baremetal_build_dir'], kwargs['baremetal_lib_basename'])
kwargs['baremetal_build_ext'] = '.elf'
# Docker
kwargs['docker_build_dir'] = join(kwargs['out_dir'], 'docker', kwargs['arch'])
kwargs['docker_tar_dir'] = join(kwargs['docker_build_dir'], 'export')
kwargs['docker_tar_file'] = join(kwargs['docker_build_dir'], 'export.tar')
kwargs['docker_rootfs_raw_file'] = join(kwargs['docker_build_dir'], 'export.ext2')
kwargs['docker_qcow2_file'] = join(kwargs['docker_rootfs_raw_file'] + '.qcow2')
if kwargs['docker']:
kwargs['rootfs_raw_file'] = kwargs['docker_rootfs_raw_file']
kwargs['qcow2_file'] = kwargs['docker_qcow2_file']
else:
kwargs['rootfs_raw_file'] = kwargs['buildroot_rootfs_raw_file']
kwargs['qcow2_file'] = kwargs['buildroot_qcow2_file']
# Image.
if kwargs['baremetal'] is None:
if kwargs['emulator'] == 'gem5':
kwargs['image'] = kwargs['vmlinux']
kwargs['disk_image'] = kwargs['rootfs_raw_file']
else:
kwargs['image'] = kwargs['linux_image']
kwargs['disk_image'] = kwargs['qcow2_file']
else:
kwargs['disk_image'] = kwargs['gem5_fake_iso']
if kwargs['baremetal'] == 'all':
path = kwargs['baremetal']
else:
path = kwargs.resolve_executable(
kwargs['baremetal'],
kwargs['baremetal_src_dir'],
kwargs['baremetal_build_dir'],
kwargs['baremetal_build_ext'],
)
source_path_noext = os.path.splitext(join(
kwargs['baremetal_src_dir'],
os.path.relpath(path, kwargs['baremetal_build_dir'])
))[0]
for ext in [c_ext, asm_ext]:
source_path = source_path_noext + ext
if os.path.exists(source_path):
kwargs['source_path'] = source_path
break
kwargs['image'] = path
return dict(kwargs)

4
copy-overlay
View File

@@ -9,8 +9,8 @@ import common
class CopyOverlayComponent(common.Component):
def do_build(self, args):
distutils.dir_util.copy_tree(
common.rootfs_overlay_dir,
common.out_rootfs_overlay_dir,
kwargs['rootfs_overlay_dir'],
kwargs['out_rootfs_overlay_dir'],
update=1,
)

4
gem5-shell
View File

@@ -10,7 +10,7 @@ parser = common.get_argparse(
)
args = common.setup(parser)
sys.exit(self.sh.run_cmd([
common.gem5_m5term, LF,
kwargs['gem5_m5term'], LF,
'localhost', LF,
str(common.gem5_telnet_port), LF,
str(kwargs['gem5_telnet_port']), LF,
]))

4
getvar
View File

@@ -5,12 +5,12 @@ import types
import common
parser = common.get_argparse(argparse_args={
'description': '''Print the value of a common.py variable.
'description': '''Print the value of a kwargs['py'] variable.
This is useful to:
* give dry commands on the README that don't change when we refactor directory structure
* create simple bash scripts that call use common.py variables
* create simple bash scripts that call use kwargs['py'] variables
For example, to get the Buildroot output directory for an ARM build, use:

2
qemu-monitor
View File

@@ -27,7 +27,7 @@ def write_and_read(tn, cmd, prompt):
tn.write(cmd.encode('utf-8'))
return '\n'.join(tn.read_until(prompt).decode('utf-8').splitlines()[1:])[:-len(prompt)]
with telnetlib.Telnet('localhost', common.qemu_monitor_port) as tn:
with telnetlib.Telnet('localhost', kwargs['qemu_monitor_port']) as tn:
# Couldn't disable server echo, so just removing the write for now.
# https://stackoverflow.com/questions/12421799/how-to-disable-telnet-echo-in-python-telnetlib
# sock = tn.get_socket()

10
qemu-trace2txt
View File

@@ -9,12 +9,12 @@ import common
def main():
return self.sh.run_cmd(
[
os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), LF,
os.path.join(common.qemu_build_dir, 'trace-events-all'), LF,
os.path.join(common.qemu_trace_file), LF,
os.path.join(kwargs['qemu_src_dir'], 'scripts/simpletrace.py'), LF,
os.path.join(kwargs['qemu_build_dir'], 'trace-events-all'), LF,
os.path.join(kwargs['qemu_trace_file']), LF,
],
cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'),
out_file=common.qemu_trace_txt_file,
cmd_file=os.path.join(kwargs['run_dir'], 'qemu-trace2txt'),
out_file=kwargs['qemu_trace_txt_file'],
show_stdout=False,
)

12
release
View File

@@ -10,21 +10,21 @@ import subprocess
import time
import common
release_zip = imp.load_source('release_zip', os.path.join(common.root_dir, 'release-zip'))
release_upload = imp.load_source('release_upload', os.path.join(common.root_dir, 'release-upload'))
release_zip = imp.load_source('release_zip', os.path.join(kwargs['root_dir'], 'release-zip'))
release_upload = imp.load_source('release_upload', os.path.join(kwargs['root_dir'], 'release-upload'))
start_time = time.time()
# TODO factor those out so we don't redo the same thing multiple times.
# subprocess.check_call([os.path.join(common.root_dir, 'test')])
# subprocess.check_call([os.path.join(common.root_dir, ''bench-all', '-A', '-u'])
# subprocess.check_call([os.path.join(kwargs['root_dir'], 'test')])
# subprocess.check_call([os.path.join(kwargs['root_dir'], ''bench-all', '-A', '-u'])
# A clean release requires a full rebuild unless we hack it :-(
# We can't just use our current build as it contains packages we've
# installed in random experiments. And with EXT2: we can't easily
# know what the smallest root filesystem size is and use it either...
# https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot
subprocess.check_call([os.path.join(common.root_dir, 'configure'), '--all'])
subprocess.check_call([os.path.join(common.root_dir, 'build'), '--all-archs', 'release'])
subprocess.check_call([os.path.join(kwargs['root_dir'], 'configure'), '--all'])
subprocess.check_call([os.path.join(kwargs['root_dir'], 'build'), '--all-archs', 'release'])
release_zip.main()
subprocess.check_call(['git', 'push'])
release_upload.main()

6
release-upload
View File

@@ -18,9 +18,9 @@ import urllib.error
import common
def main():
repo = common.github_repo_id
tag = 'sha-{}'.format(common.sha)
upload_path = common.release_zip_file
repo = kwargs['github_repo_id']
tag = 'sha-{}'.format(kwargs['sha'])
upload_path = kwargs['release_zip_file']
# Check the release already exists.
try:

14
release-zip
View File

@@ -11,14 +11,14 @@ import zipfile
import common
def main():
os.makedirs(common.release_dir, exist_ok=True)
if os.path.exists(common.release_zip_file):
os.unlink(common.release_zip_file)
zipf = zipfile.ZipFile(common.release_zip_file, 'w', zipfile.ZIP_DEFLATED)
for arch in common.all_archs:
os.makedirs(kwargs['release_dir'], exist_ok=True)
if os.path.exists(kwargs['release_zip_file']):
os.unlink(kwargs['release_zip_file'])
zipf = zipfile.ZipFile(kwargs['release_zip_file'], 'w', zipfile.ZIP_DEFLATED)
for arch in kwargs['all_archs']:
common.setup(common.get_argparse(default_args={'arch': arch}))
zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir))
zipf.write(common.linux_image, arcname=os.path.relpath(common.linux_image, common.root_dir))
zipf.write(kwargs['qcow2_file'], arcname=os.path.relpath(kwargs['qcow2_file'], kwargs['root_dir']))
zipf.write(kwargs['linux_image'], arcname=os.path.relpath(kwargs['linux_image'], kwargs['root_dir']))
zipf.close()
if __name__ == '__main__':

144
run
View File

@@ -85,7 +85,7 @@ def main(args, extra_args=None):
console_count = 0
if kwargs['arch'] == 'x86_64':
console_type = 'ttyS'
elif common.is_arm:
elif kwargs['is_arm']:
console_type = 'ttyAMA'
console = '{}{}'.format(console_type, console_count)
console_count += 1
@@ -118,36 +118,36 @@ def main(args, extra_args=None):
def raise_rootfs_not_found():
if not kwargs['dry_run']:
raise Exception('Root filesystem not found. Did you build it?\n' \
'Tried to use: ' + common.disk_image)
'Tried to use: ' + kwargs['disk_image'])
def raise_image_not_found():
if not kwargs['dry_run']:
raise Exception('Executable image not found. Did you build it?\n' \
'Tried to use: ' + common.image)
if common.image is None:
'Tried to use: ' + kwargs['image'])
if kwargs['image'] is None:
raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
cmd = debug_vm.copy()
if common.emulator == 'gem5':
if common.baremetal is None:
if not os.path.exists(common.rootfs_raw_file):
if not os.path.exists(common.qcow2_file):
if kwargs['emulator'] == 'gem5':
if kwargs['baremetal'] is None:
if not os.path.exists(kwargs['rootfs_raw_file']):
if not os.path.exists(kwargs['qcow2_file']):
raise_rootfs_not_found()
common.raw_to_qcow2(prebuilt=kwargs['prebuilt'], reverse=True)
else:
if not os.path.exists(common.gem5_fake_iso):
os.makedirs(os.path.dirname(common.gem5_fake_iso), exist_ok=True)
common.write_string_to_file(common.gem5_fake_iso, 'a' * 512)
if not os.path.exists(common.image):
if not os.path.exists(kwargs['gem5_fake_iso']):
os.makedirs(os.path.dirname(kwargs['gem5_fake_iso']), exist_ok=True)
common.write_string_to_file(kwargs['gem5_fake_iso'], 'a' * 512)
if not os.path.exists(kwargs['image']):
# This is to run gem5 from a prebuilt download.
if (not common.baremetal is None) or (not os.path.exists(common.linux_image)):
if (not kwargs['baremetal'] is None) or (not os.path.exists(kwargs['linux_image'])):
raise_image_not_found()
self.sh.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)])
os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True)
common.write_string_to_file(common.gem5_readfile, kwargs['gem5_readfile'])
self.sh.run_cmd([os.path.join(kwargs['extract_vmlinux'], kwargs['linux_image'])])
os.makedirs(os.path.dirname(kwargs['gem5_readfile']), exist_ok=True)
common.write_string_to_file(kwargs['gem5_readfile'], kwargs['gem5_readfile'])
memory = '{}B'.format(kwargs['memory'])
gem5_exe_args = common.shlex_split(kwargs['gem5_exe_args'])
if do_trace:
gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), LF])
extra_env['M5_PATH'] = common.gem5_system_dir
extra_env['M5_PATH'] = kwargs['gem5_system_dir']
# https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071
extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true'
if kwargs['trace_stdout']:
@@ -156,16 +156,16 @@ def main(args, extra_args=None):
debug_file = 'trace.txt'
cmd.extend(
[
common.executable, LF,
kwargs['executable'], LF,
'--debug-file', debug_file, LF,
'--listener-mode', 'on', LF,
'--outdir', common.m5out_dir, LF,
'--outdir', kwargs['m5out_dir'], LF,
] +
gem5_exe_args
)
if kwargs['userland'] is not None:
cmd.extend([
common.gem5_se_file, LF,
kwargs['gem5_se_file'], LF,
'-c', common.resolve_userland(kwargs['userland']), LF,
])
else:
@@ -176,28 +176,28 @@ def main(args, extra_args=None):
cpt_dir = cpt_dirs[-kwargs['gem5_restore']]
extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)])
cmd.extend([
common.gem5_fs_file, LF,
'--disk-image', common.disk_image, LF,
'--kernel', common.image, LF,
kwargs['gem5_fs_file'], LF,
'--disk-image', kwargs['disk_image'], LF,
'--kernel', kwargs['image'], LF,
'--mem-size', memory, LF,
'--num-cpus', str(kwargs['cpus']), LF,
'--script', common.gem5_readfile, LF,
'--script', kwargs['gem5_readfile'], LF,
])
if kwargs['arch'] == 'x86_64':
if kwargs['kvm']:
cmd.extend(['--cpu-type', 'X86KvmCPU', LF])
cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), LF])
elif common.is_arm:
elif kwargs['is_arm']:
if kwargs['kvm']:
cmd.extend(['--cpu-type', 'ArmV8KvmCPU', LF])
cmd.extend([
# TODO why is it mandatory to pass mem= here? Not true for QEMU.
# Anything smaller than physical blows up as expected, but why can't it auto-detect the right value?
'--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), LF,
'--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, kwargs['cpus'])), LF,
'--machine-type', common.machine, LF,
'--dtb-filename', os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(kwargs['armv'], kwargs['cpus'])), LF,
'--machine-type', kwargs['machine'], LF,
])
if common.baremetal is None:
if kwargs['baremetal'] is None:
cmd.extend([
'--param', 'system.panic_on_panic = True', LF])
else:
@@ -215,14 +215,14 @@ def main(args, extra_args=None):
cpu_type = 'atomic'
if kwargs['gem5_restore'] is not None:
cpt_dir = common.gem_list_checkpoint_dirs()[-kwargs['gem5_restore']]
extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)])
extra_emulator_args.extend(['--restore-from', os.path.join(kwargs['m5out_dir'], cpt_dir)])
cmd.extend([
os.path.join(common.gem5_source_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF,
os.path.join(kwargs['gem5_source_dir'], 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF,
'--big-cpus', '2', LF,
'--cpu-type', cpu_type, LF,
'--disk', common.disk_image, LF,
'--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), LF,
'--kernel', common.image, LF,
'--disk', kwargs['disk_image'], LF,
'--dtb', os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), LF,
'--kernel', kwargs['image'], LF,
'--little-cpus', '2', LF,
])
if kwargs['wait_gdb']:
@@ -230,17 +230,17 @@ def main(args, extra_args=None):
cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', LF])
else:
qemu_user_and_system_options = [
'-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), LF,
'-trace', 'enable={},file={}'.format(trace_type, kwargs['qemu_trace_file']), LF,
]
if kwargs['userland'] is not None:
if kwargs['wait_gdb']:
debug_args = ['-g', str(common.gdb_port), LF]
debug_args = ['-g', str(kwargs['gdb_port']), LF]
else:
debug_args = []
cmd.extend(
[
os.path.join(common.qemu_build_dir, '{}-linux-user'.format(kwargs['arch']), 'qemu-{}'.format(kwargs['arch'])), LF,
'-L', common.target_dir, LF
os.path.join(kwargs['qemu_build_dir'], '{}-linux-user'.format(kwargs['arch']), 'qemu-{}'.format(kwargs['arch'])), LF,
'-L', kwargs['target_dir'], LF
] +
qemu_user_and_system_options +
common.shlex_split(kwargs['userland_before']) +
@@ -250,15 +250,15 @@ def main(args, extra_args=None):
]
)
else:
if not os.path.exists(common.image):
if not os.path.exists(kwargs['image']):
raise_image_not_found()
extra_emulator_args.extend(extra_qemu_args)
common.make_run_dirs()
if kwargs['prebuilt'] or not os.path.exists(common.qemu_executable):
qemu_executable = common.qemu_executable_basename
if kwargs['prebuilt'] or not os.path.exists(kwargs['qemu_executable']):
qemu_executable = kwargs['qemu_executable_basename']
qemu_executable_prebuilt = True
else:
qemu_executable = common.qemu_executable
qemu_executable = kwargs['qemu_executable']
qemu_executable_prebuilt = False
qemu_executable = shutil.which(qemu_executable)
if qemu_executable is None:
@@ -268,17 +268,17 @@ def main(args, extra_args=None):
serial_monitor = []
else:
if kwargs['background']:
serial_monitor = ['-serial', 'file:{}'.format(common.qemu_background_serial_file), LF]
serial_monitor = ['-serial', 'file:{}'.format(kwargs['qemu_background_serial_file']), LF]
else:
serial_monitor = ['-serial', 'mon:stdio', LF]
if kwargs['kvm']:
extra_emulator_args.extend(['-enable-kvm', LF])
extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port), LF])
extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(kwargs['extra_serial_port']), LF])
virtfs_data = [
(common.p9_dir, 'host_data'),
(common.out_dir, 'host_out'),
(common.out_rootfs_overlay_dir, 'host_out_rootfs_overlay'),
(common.rootfs_overlay_dir, 'host_rootfs_overlay'),
(kwargs['p9_dir'], 'host_data'),
(kwargs['out_dir'], 'host_out'),
(kwargs['out_rootfs_overlay_dir'], 'host_out_rootfs_overlay'),
(kwargs['rootfs_overlay_dir'], 'host_rootfs_overlay'),
]
virtfs_cmd = []
for virtfs_dir, virtfs_tag in virtfs_data:
@@ -293,11 +293,11 @@ def main(args, extra_args=None):
[
qemu_executable, LF,
'-device', 'rtl8139,netdev=net0', LF,
'-gdb', 'tcp::{}'.format(common.gdb_port), LF,
'-kernel', common.image, LF,
'-gdb', 'tcp::{}'.format(kwargs['gdb_port']), LF,
'-kernel', kwargs['image'], LF,
'-m', kwargs['memory'], LF,
'-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), LF,
'-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), LF,
'-monitor', 'telnet::{},server,nowait'.format(kwargs['qemu_monitor_port']), LF,
'-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(kwargs['qemu_hostfwd_generic_port'], kwargs['qemu_hostfwd_generic_port'], kwargs['qemu_hostfwd_ssh_port']), LF,
'-no-reboot', LF,
'-smp', str(kwargs['cpus']), LF,
] +
@@ -308,7 +308,7 @@ def main(args, extra_args=None):
if not qemu_executable_prebuilt:
cmd.extend(qemu_user_and_system_options)
if kwargs['initrd']:
extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')])
extra_emulator_args.extend(['-initrd', os.path.join(kwargs['buildroot_images_dir'], 'rootfs.cpio')])
rr = kwargs['record'] or kwargs['replay']
if ramfs:
# TODO why is this needed, and why any string works.
@@ -324,14 +324,14 @@ def main(args, extra_args=None):
root = 'root=/dev/vda'
rrid = ''
snapshot = ',snapshot'
if common.baremetal is None:
if not os.path.exists(common.qcow2_file):
if not os.path.exists(common.rootfs_raw_file):
if kwargs['baremetal'] is None:
if not os.path.exists(kwargs['qcow2_file']):
if not os.path.exists(kwargs['rootfs_raw_file']):
raise_rootfs_not_found()
common.raw_to_qcow2(prebuilt=kwargs['prebuilt'])
extra_emulator_args.extend([
'-drive',
'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid),
'file={},format=qcow2,if={}{}{}'.format(kwargs['disk_image'], driveif, snapshot, rrid),
LF,
])
if rr:
@@ -342,7 +342,7 @@ def main(args, extra_args=None):
if rr:
extra_emulator_args.extend([
'-object', 'filter-replay,id=replay,netdev=net0',
'-icount', 'shift=7,rr={},rrfile={}'.format('record' if kwargs['record'] else 'replay', common.qemu_rrfile),
'-icount', 'shift=7,rr={},rrfile={}'.format('record' if kwargs['record'] else 'replay', kwargs['qemu_rrfile']),
])
virtio_gpu_pci = []
else:
@@ -350,10 +350,10 @@ def main(args, extra_args=None):
if kwargs['arch'] == 'x86_64':
append = ['-append', '{} nopat {}'.format(root, kernel_cli), LF]
cmd.extend([
'-M', common.machine, LF,
'-M', kwargs['machine'], LF,
'-device', 'edu', LF,
])
elif common.is_arm:
elif kwargs['is_arm']:
extra_emulator_args.extend(['-semihosting', LF])
if kwargs['arch'] == 'arm':
cpu = 'cortex-a15'
@@ -364,16 +364,16 @@ def main(args, extra_args=None):
[
# highmem=off needed since v3.0.0 due to:
# http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
'-M', '{},highmem=off'.format(common.machine), LF,
'-M', '{},highmem=off'.format(kwargs['machine']), LF,
'-cpu', cpu, LF,
] +
virtio_gpu_pci
)
if common.baremetal is None:
if kwargs['baremetal'] is None:
cmd.extend(append)
if kwargs['tmux'] is not None:
tmux_args = '--run-id {}'.format(kwargs['run_id'])
if common.emulator == 'gem5':
if kwargs['emulator'] == 'gem5':
tmux_cmd = './gem5-shell'
elif kwargs['wait_gdb']:
tmux_cmd = './run-gdb'
@@ -386,13 +386,13 @@ def main(args, extra_args=None):
kwargs['linux_build_id'],
kwargs['run_id'],
)
if common.baremetal:
tmux_args += " --baremetal '{}'".format(common.baremetal)
if kwargs['baremetal']:
tmux_args += " --baremetal '{}'".format(kwargs['baremetal'])
if kwargs['userland']:
tmux_args += " --userland '{}'".format(kwargs['userland'])
tmux_args += ' {}'.format(kwargs['tmux'])
subprocess.Popen([
os.path.join(common.root_dir, 'tmu'),
os.path.join(kwargs['root_dir'], 'tmu'),
"sleep 2;{} {}".format(tmux_cmd, tmux_args)
])
cmd.extend(extra_emulator_args)
@@ -400,10 +400,10 @@ def main(args, extra_args=None):
if debug_vm or kwargs['terminal']:
out_file = None
else:
out_file = common.termout_file
self.sh.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env)
out_file = kwargs['termout_file']
self.sh.run_cmd(cmd, cmd_file=kwargs['run_cmd_file'], out_file=out_file, extra_env=extra_env)
# Check if guest panicked.
if common.emulator == 'gem5':
if kwargs['emulator'] == 'gem5':
# We have to do some parsing here because gem5 exits with status 0 even when panic happens.
# Grepping for '^panic: ' does not work because some errors don't show that message.
panic_msg = b'--- BEGIN LIBC BACKTRACE ---$'
@@ -412,16 +412,16 @@ def main(args, extra_args=None):
panic_re = re.compile(panic_msg)
error_string_found = False
if out_file is not None and not kwargs['dry_run']:
with open(common.termout_file, 'br') as logfile:
with open(kwargs['termout_file'], 'br') as logfile:
for line in logfile:
if panic_re.search(line):
error_string_found = True
if os.path.exists(common.guest_terminal_file):
with open(common.guest_terminal_file, 'br') as logfile:
if os.path.exists(kwargs['guest_terminal_file']):
with open(kwargs['guest_terminal_file'], 'br') as logfile:
lines = logfile.readlines()
if lines:
last_line = lines[-1]
if last_line.rstrip() == common.magic_fail_string:
if last_line.rstrip() == kwargs['magic_fail_string']:
error_string_found = True
if error_string_found:
common.log_error('simulation error detected by parsing logs')

8
run-docker
View File

@@ -5,10 +5,10 @@ import os
import common
container_name = common.repo_short_id
container_hostname = common.repo_short_id
image_name = common.repo_short_id
target_dir = '/root/{}'.format(common.repo_short_id)
container_name = kwargs['repo_short_id']
container_hostname = kwargs['repo_short_id']
image_name = kwargs['repo_short_id']
target_dir = '/root/{}'.format(kwargs['repo_short_id'])
docker = ['sudo', 'docker']
def create(args):
self.sh.run_cmd(docker + ['build', '-t', image_name, '.', LF])

26
run-gdb
View File

@@ -120,15 +120,15 @@ def main(args, extra_args=None):
break_at = ['-ex', 'break {}'.format(kwargs['break_at']), LF]
else:
break_at = []
linux_full_system = (common.baremetal is None and kwargs['userland'] is None)
linux_full_system = (kwargs['baremetal'] is None and kwargs['userland'] is None)
if kwargs['userland']:
image = common.resolve_userland(kwargs['userland'])
elif common.baremetal:
image = common.image
test_script_path = os.path.splitext(common.source_path)[0] + '.py'
elif kwargs['baremetal']:
image = kwargs['image']
test_script_path = os.path.splitext(kwargs['source_path'])[0] + '.py'
else:
image = common.vmlinux
if common.baremetal:
image = kwargs['vmlinux']
if kwargs['baremetal']:
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
else:
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
@@ -138,14 +138,14 @@ def main(args, extra_args=None):
['-q', LF]
)
if linux_full_system:
cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), LF])
cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(kwargs['linux_build_dir']), LF])
if kwargs['sim']:
target = 'sim'
else:
if kwargs['kgdb']:
port = common.extra_serial_port
port = kwargs['extra_serial_port']
else:
port = common.gdb_port
port = kwargs['gdb_port']
target = 'remote localhost:{}'.format(port)
cmd.extend([
'-ex', 'file {}'.format(image), LF,
@@ -175,11 +175,11 @@ def main(args, extra_args=None):
# which gets put on the kernel build root when python debugging scripts are enabled.
cmd.extend(['-ex', 'continue', LF])
if not kwargs['no_lxsymbols'] and linux_full_system:
cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), LF])
cmd.extend(['-ex', 'lx-symbols {}'.format(kwargs['kernel_modules_build_subdir']), LF])
cmd.extend(after)
if kwargs['test']:
GdbTestcase(
common.source_path,
kwargs['source_path'],
test_script_path,
cmd,
verbose=kwargs['verbose'],
@@ -190,8 +190,8 @@ def main(args, extra_args=None):
# vmlinux: No such file or directory.
return self.sh.run_cmd(
cmd,
cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'),
cwd=common.linux_build_dir
cmd_file=os.path.join(kwargs['run_dir'], 'run-gdb.sh'),
cwd=kwargs['linux_build_dir']
)
if __name__ == '__main__':

4
run-gdb-user
View File

@@ -5,7 +5,7 @@ import os
import sys
import common
rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'run-gdb'))
rungdb = imp.load_source('rungdb', os.path.join(kwargs['root_dir'], 'run-gdb'))
parser = common.get_argparse(argparse_args={
'description': '''GDB step debug guest userland processes without gdbserver.
@@ -25,7 +25,7 @@ parser.add_argument(
)
args = common.setup(parser)
executable = common.resolve_userland(kwargs['executable'])
addr = common.get_elf_entry(os.path.join(common.buildroot_build_build_dir, executable))
addr = common.get_elf_entry(os.path.join(kwargs['buildroot_build_build_dir'], executable))
extra_args = {}
extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(addr))
# Or else lx-symbols throws for arm:

6
run-gdbserver
View File

@@ -20,9 +20,9 @@ args = common.setup(parser)
sys.exit(subprocess.Popen([
common.get_toolchain_tool('gdb'),
'-q',
'-ex', 'set sysroot {}'.format(common.buildroot_staging_dir),
'-ex', 'target remote localhost:{}'.format(common.qemu_hostfwd_generic_port),
'-ex', 'set sysroot {}'.format(kwargs['buildroot_staging_dir']),
'-ex', 'target remote localhost:{}'.format(kwargs['qemu_hostfwd_generic_port']),
'-ex', 'tbreak {}'.format(kwargs['break_at']),
'-ex', 'continue',
os.path.join(common.buildroot_build_build_dir, common.resolve_userland(kwargs['executable'])),
os.path.join(kwargs['buildroot_build_build_dir'], common.resolve_userland(kwargs['executable'])),
]).wait())

10
run-toolchain
View File

@@ -35,16 +35,16 @@ parser.add_argument(
nargs='*'
)
args = common.setup(parser)
if common.baremetal is None:
image = common.vmlinux
if kwargs['baremetal'] is None:
image = kwargs['vmlinux']
else:
image = common.image
image = kwargs['image']
tool= common.get_toolchain_tool(kwargs['tool'])
if kwargs['dry']:
print(tool)
else:
sys.exit(self.sh.run_cmd(
[tool, LF]
+ common.add_newlines(kwargs['extra_args']),
cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'),
+ self.sh.add_newlines(kwargs['extra_args']),
cmd_file=os.path.join(kwargs['run_dir'], 'run-toolchain.sh'),
))

6
shell_helpers.py
View File

@@ -1,10 +1,12 @@
#!/usr/bin/env python3
import distutils.file_util
import itertools
import os
import shlex
import shutil
import signal
import subprocess
class LF:
'''
@@ -227,8 +229,8 @@ class ShellHelpers:
def strip_newlines(self, cmd):
return [x for x in cmd if x != LF]
def rmrf(path):
self.print_cmd(['rm', '-r', '-f', path])
def rmrf(self, path):
self.print_cmd(['rm', '-r', '-f', path, LF])
if not self.dry_run and os.path.exists(path):
shutil.rmtree(path)

10
trace-boot
View File

@@ -6,8 +6,8 @@ import subprocess
import re
import common
run = imp.load_source('run', os.path.join(common.root_dir, 'run'))
qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(common.root_dir, 'qemu-trace2txt'))
run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run'))
qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(kwargs['root_dir'], 'qemu-trace2txt'))
parser = common.get_argparse(argparse_args={
'description': '''Trace the PIC addresses executed on a Linux kernel boot.
@@ -23,7 +23,7 @@ args = common.setup(parser)
extra_args = {
'extra_emulator_args': kwargs['extra_emulator_args'],
}
if common.emulator == 'gem5':
if kwargs['emulator'] == 'gem5':
extra_args.update({
'eval': 'm5 exit',
'trace': 'Exec,-ExecSymbol,-ExecMicro',
@@ -39,10 +39,10 @@ else:
# Instruction count.
# We could put this on a separate script, but it just adds more arch boilerplate to a new script.
# So let's just leave it here for now since it did not add a significant processing time.
kernel_entry_addr = hex(common.get_elf_entry(common.vmlinux))
kernel_entry_addr = hex(common.get_elf_entry(kwargs['vmlinux']))
nlines = 0
nlines_firmware = 0
with open(common.qemu_trace_txt_file, 'r') as trace_file:
with open(kwargs['qemu_trace_txt_file'], 'r') as trace_file:
in_firmware = True
for line in trace_file:
line = line.rstrip()

20
trace2line
View File

@@ -19,17 +19,17 @@ parser = common.get_argparse(argparse_args={
})
args = common.setup(parser)
sys.exit(subprocess.Popen([
os.path.join(common.root_dir, 'trace2line.sh'),
'true' if common.emulator == 'gem5' else 'false',
common.trace_txt_file,
os.path.join(kwargs['root_dir'], 'trace2line.sh'),
'true' if kwargs['emulator'] == 'gem5' else 'false',
kwargs['trace_txt_file'],
common.get_toolchain_tool('addr2line'),
common.vmlinux,
common.run_dir,
kwargs['vmlinux'],
kwargs['run_dir'],
]).wait())
# This was the full conversion attempt.
# if common.emulator == 'gem5':
# if kwargs['emulator'] == 'gem5':
# def get_pc(line):
# # TODO
# # stdin = sed -r 's/^.* (0x[^. ]*)[. ].*/\1/' "$common_trace_txt_file")
@@ -42,15 +42,15 @@ sys.exit(subprocess.Popen([
# [
# common.get_toolchain_tool('addr2line'),
# '-e',
# common.vmlinux,
# kwargs['vmlinux'],
# '-f',
# '-p',
# ],
# stdout=subprocess.PIPE,
# stdin=subprocess.PIPE,
# ) as proc, \
# open(common.trace_txt_file, 'r') as infile, \
# open(os.path.join(common.run_dir, 'trace-lines.txt'), 'w') as outfile \
# open(kwargs['trace_txt_file'], 'r') as infile, \
# open(os.path.join(kwargs['run_dir'], 'trace-lines.txt'), 'w') as outfile \
# :
# for in_line in infile:
# proc.stdin.write(get_pc(in_line).encode())
@@ -58,5 +58,5 @@ sys.exit(subprocess.Popen([
# stdout = proc.stdout.read()
# outfile.write(stdout.decode())
# # TODO
# # sed -E "s|at ${common.linux_build_dir}/(\./\|)||"
# # sed -E "s|at ${kwargs['linux_build_dir']}/(\./\|)||"
# # uniq -c