3rd-party/linux-kernel-module-cheat
1
0
Fork 0
mirror of https://github.com/cirosantilli/linux-kernel-module-cheat.git synced 2026-01-27 12:04:27 +01:00
Code Issues Releases Wiki Activity

build-linux and build-gem5 seem to work

Browse Source
This commit is contained in:
Ciro Santilli 六四事件 法轮功
2018-12-09 00:00:00 +00:00
parent 1768421dbd
commit 5e20ba833b
33 changed files with 702 additions and 707 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.adoc
View File

@@ -10301,7 +10301,7 @@ The most important things that we setup in the bootloaders are:
The C functions that become available as a result are: The C functions that become available as a result are:
* Newlib functions implemented at link:baremetal/lib/syscalls.c[] * Newlib functions implemented at link:baremetal/lib/syscalls.c[]
* non-Newlib functions implemented at link:common.c[] * non-Newlib functions implemented at link:kwargs['c'][]
It is not possible to call those C functions from the examples that don't use a bootloader. It is not possible to call those C functions from the examples that don't use a bootloader.
@@ -10333,7 +10333,7 @@ svc 0x00123456
and we can see from the docs that `0x18` stands for the `SYS_EXIT` command. and we can see from the docs that `0x18` stands for the `SYS_EXIT` command.
This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/common.c[]. This is also how we implement the `exit(0)` system call in C for QEMU for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/kwargs['c'][].
Other magic operations we can do with semihosting besides exiting the on the host include: Other magic operations we can do with semihosting besides exiting the on the host include:

3
arm
View File

@@ -1,3 +0,0 @@
#!/usr/bin/env bash
build-crosstool-ng \
;

6
bisect-linux-boot-gem5
View File

@@ -6,8 +6,8 @@ import shutil
import sys import sys
import common import common
build_linux = imp.load_source('build-linux', os.path.join(common.root_dir, 'build_linux')) build_linux = imp.load_source('build-linux', os.path.join(kwargs['root_dir'], 'build_linux'))
run = imp.load_source('run', os.path.join(common.root_dir, 'run')) run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run'))
parser = common.get_argparse( parser = common.get_argparse(
argparse_args={ argparse_args={
@@ -24,7 +24,7 @@ args = common.setup(parser)
# We need a clean rebuild because rebuilds at different revisions: # We need a clean rebuild because rebuilds at different revisions:
# - may fail # - may fail
# - may not actually rebuild all files, e.g. on header changes # - may not actually rebuild all files, e.g. on header changes
common.rmrf(common.linux_build_dir) common.rmrf(kwargs['linux_build_dir'])
build_linux.LinuxComponent().do_build(args) build_linux.LinuxComponent().do_build(args)
status = run.main(args, { status = run.main(args, {
'eval': 'm5 exit', 'eval': 'm5 exit',

42
build
View File

@@ -49,7 +49,7 @@ class Component:
def run_cmd(cmd, arch): def run_cmd(cmd, arch):
global args global args
cmd_abs = cmd.copy() cmd_abs = cmd.copy()
cmd_abs[0] = os.path.join(common.root_dir, cmd[0]) cmd_abs[0] = os.path.join(kwargs['root_dir'], cmd[0])
cmd_abs.extend(['--arch', arch]) cmd_abs.extend(['--arch', arch])
if kwargs['extra_args']: if kwargs['extra_args']:
cmd_abs.append(kwargs['extra_args']) cmd_abs.append(kwargs['extra_args'])
@@ -86,15 +86,15 @@ name_to_component_map = {
# Leaves without dependencies. # Leaves without dependencies.
'baremetal-qemu': Component( 'baremetal-qemu': Component(
lambda arch: run_cmd(['build-baremetal', '--qemu'], arch), lambda arch: run_cmd(['build-baremetal', '--qemu'], arch),
supported_archs=common.crosstool_ng_supported_archs, supported_archs=kwargs['crosstool_ng_supported_archs'],
), ),
'baremetal-gem5': Component( 'baremetal-gem5': Component(
lambda arch: run_cmd(['build-baremetal', '--gem5'], arch), lambda arch: run_cmd(['build-baremetal', '--gem5'], arch),
supported_archs=common.crosstool_ng_supported_archs, supported_archs=kwargs['crosstool_ng_supported_archs'],
), ),
'baremetal-gem5-pbx': Component( 'baremetal-gem5-pbx': Component(
lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch), lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
supported_archs=common.crosstool_ng_supported_archs, supported_archs=kwargs['crosstool_ng_supported_archs'],
), ),
'buildroot': buildroot_component, 'buildroot': buildroot_component,
'buildroot-gcc': buildroot_component, 'buildroot-gcc': buildroot_component,
@@ -103,7 +103,7 @@ name_to_component_map = {
), ),
'crosstool-ng': Component( 'crosstool-ng': Component(
lambda arch: run_cmd(['build-crosstool-ng'], arch), lambda arch: run_cmd(['build-crosstool-ng'], arch),
supported_archs=common.crosstool_ng_supported_archs, supported_archs=kwargs['crosstool_ng_supported_archs'],
# http://crosstool-ng.github.io/docs/os-setup/ # http://crosstool-ng.github.io/docs/os-setup/
apt_get_pkgs={ apt_get_pkgs={
'bison', 'bison',
@@ -197,7 +197,7 @@ name_to_component_map = {
'gem5-baremetal', 'gem5-baremetal',
'baremetal-gem5-pbx', 'baremetal-gem5-pbx',
], ],
supported_archs=common.crosstool_ng_supported_archs, supported_archs=kwargs['crosstool_ng_supported_archs'],
), ),
'all-linux': Component(dependencies=[ 'all-linux': Component(dependencies=[
'qemu-gem5-buildroot', 'qemu-gem5-buildroot',
@@ -296,10 +296,10 @@ group = parser.add_mutually_exclusive_group(required=False)
group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\ group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\
Build the selected components for all archs. Build the selected components for all archs.
''') ''')
group.add_argument('-a', '--arch', choices=common.arch_choices, default=[], action='append', help='''\ group.add_argument('-a', '--arch', choices=kwargs['arch_choices'], default=[], action='append', help='''\
Build the selected components for this arch. Select multiple archs by Build the selected components for this arch. Select multiple archs by
passing this option multiple times. Default: [{}] passing this option multiple times. Default: [{}]
'''.format(common.default_arch)) '''.format(kwargs['default_arch']))
parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\ parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\
Also download all dependencies required for a given build: Ubuntu packages, Also download all dependencies required for a given build: Ubuntu packages,
Python packages and git submodules. Python packages and git submodules.
@@ -314,7 +314,7 @@ Extra args to pass to all scripts.
) )
parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\ parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\
Which components to build. Default: qemu-buildroot Which components to build. Default: qemu-buildroot
'''.format(common.default_arch)) '''.format(kwargs['default_arch']))
common.add_dry_run_argument(parser) common.add_dry_run_argument(parser)
args = parser.parse_args() args = parser.parse_args()
common.setup_dry_run_arguments(args) common.setup_dry_run_arguments(args)
@@ -322,14 +322,14 @@ common.setup_dry_run_arguments(args)
# Decide archs. # Decide archs.
if kwargs['arch'] == []: if kwargs['arch'] == []:
if kwargs['all'] or kwargs['all_archs']: if kwargs['all'] or kwargs['all_archs']:
archs = common.all_archs.copy() archs = kwargs['all_archs'].copy()
else: else:
archs = set([common.default_arch]) archs = set([kwargs['default_arch']])
else: else:
archs = set() archs = set()
for arch in kwargs['arch']: for arch in kwargs['arch']:
if arch in common.arch_short_to_long_dict: if arch in kwargs['arch_short_to_long_dict']:
arch = common.arch_short_to_long_dict[arch] arch = kwargs['arch_short_to_long_dict'][arch]
archs.add(arch) archs.add(arch)
# Decide components. # Decide components.
@@ -393,7 +393,7 @@ if kwargs['download_dependencies']:
'libsdl2-dev', 'libsdl2-dev',
} }
apt_get_pkgs.difference_update(interacive_pkgs) apt_get_pkgs.difference_update(interacive_pkgs)
if common.in_docker: if kwargs['in_docker']:
sudo = [] sudo = []
# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
os.environ['DEBIAN_FRONTEND'] = 'noninteractive' os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
@@ -406,7 +406,7 @@ if kwargs['download_dependencies']:
f.write(sources_txt) f.write(sources_txt)
else: else:
sudo = ['sudo'] sudo = ['sudo']
if common.in_docker or kwargs['travis']: if kwargs['in_docker'] or kwargs['travis']:
y = ['-y'] y = ['-y']
else: else:
y = [] y = []
@@ -416,25 +416,25 @@ if kwargs['download_dependencies']:
if apt_get_pkgs: if apt_get_pkgs:
self.sh.run_cmd( self.sh.run_cmd(
sudo + ['apt-get', 'install'] + y + [LF] + sudo + ['apt-get', 'install'] + y + [LF] +
common.add_newlines(sorted(apt_get_pkgs)) self.sh.add_newlines(sorted(apt_get_pkgs))
) )
if apt_build_deps: if apt_build_deps:
self.sh.run_cmd( self.sh.run_cmd(
sudo + sudo +
['apt-get', 'build-dep'] + y + [LF] + ['apt-get', 'build-dep'] + y + [LF] +
common.add_newlines(sorted(apt_build_deps)) self.sh.add_newlines(sorted(apt_build_deps))
) )
if python2_pkgs: if python2_pkgs:
self.sh.run_cmd( self.sh.run_cmd(
['python', '-m', 'pip', 'install', '--user', LF] + ['python', '-m', 'pip', 'install', '--user', LF] +
common.add_newlines(sorted(python2_pkgs)) self.sh.add_newlines(sorted(python2_pkgs))
) )
if python3_pkgs: if python3_pkgs:
# Not with pip executable directly: # Not with pip executable directly:
# https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
self.sh.run_cmd( self.sh.run_cmd(
['python3', '-m', 'pip', 'install', '--user', LF] + ['python3', '-m', 'pip', 'install', '--user', LF] +
common.add_newlines(sorted(python3_pkgs)) self.sh.add_newlines(sorted(python3_pkgs))
) )
git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive'] git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
if submodules: if submodules:
@@ -450,7 +450,7 @@ if kwargs['download_dependencies']:
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
self.sh.run_cmd( self.sh.run_cmd(
git_cmd_common + ['--', LF] + git_cmd_common + ['--', LF] +
common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules)]) self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules)])
) )
if submodules_shallow: if submodules_shallow:
# == Shallow cloning. # == Shallow cloning.
@@ -474,7 +474,7 @@ if kwargs['download_dependencies']:
# #
self.sh.run_cmd( self.sh.run_cmd(
git_cmd_common + ['--depth', '1', '--', LF] + git_cmd_common + ['--depth', '1', '--', LF] +
common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules_shallow)]) self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules_shallow)])
) )
# Do the build. # Do the build.

52
build-baremetal
View File

@@ -8,43 +8,43 @@ class BaremetalComponent(common.Component):
def do_build(self, args): def do_build(self, args):
common.assert_crosstool_ng_supports_arch(kwargs['arch']) common.assert_crosstool_ng_supports_arch(kwargs['arch'])
build_dir = self.get_build_dir(args) build_dir = self.get_build_dir(args)
bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext)) bootloader_obj = os.path.join(kwargs['baremetal_build_lib_dir'], 'bootloader{}'.format(kwargs['obj_ext']))
common_basename_noext = 'common' common_basename_noext = 'common'
common_src = os.path.join(common.root_dir, common_basename_noext + common.c_ext) common_src = os.path.join(kwargs['root_dir'], common_basename_noext + kwargs['c_ext'])
common_obj = os.path.join(common.baremetal_build_lib_dir, common_basename_noext + common.obj_ext) common_obj = os.path.join(kwargs['baremetal_build_lib_dir'], common_basename_noext + kwargs['obj_ext'])
syscalls_basename_noext = 'syscalls' syscalls_basename_noext = 'syscalls'
syscalls_src = os.path.join(common.baremetal_src_lib_dir, syscalls_basename_noext + common.c_ext) syscalls_src = os.path.join(kwargs['baremetal_src_lib_dir'], syscalls_basename_noext + kwargs['c_ext'])
syscalls_obj = os.path.join(common.baremetal_build_lib_dir, syscalls_basename_noext + common.obj_ext) syscalls_obj = os.path.join(kwargs['baremetal_build_lib_dir'], syscalls_basename_noext + kwargs['obj_ext'])
common_objs = [common_obj, syscalls_obj] common_objs = [common_obj, syscalls_obj]
cflags = [ cflags = [
'-I', common.baremetal_src_lib_dir, LF, '-I', kwargs['baremetal_src_lib_dir'], LF,
'-I', common.root_dir, LF, '-I', kwargs['root_dir'], LF,
'-O0', LF, '-O0', LF,
'-ggdb3', LF, '-ggdb3', LF,
'-mcpu={}'.format(common.mcpu), LF, '-mcpu={}'.format(kwargs['mcpu']), LF,
'-nostartfiles', LF, '-nostartfiles', LF,
] ]
if kwargs['prebuilt']: if kwargs['prebuilt']:
gcc = 'arm-none-eabi-gcc' gcc = 'arm-none-eabi-gcc'
else: else:
os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH'] os.environ['PATH'] = kwargs['crosstool_ng_bin_dir'] + os.environ['PATH']
gcc = common.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng']) gcc = common.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng'])
if common.emulator == 'gem5': if kwargs['emulator'] == 'gem5':
if common.machine == 'VExpress_GEM5_V1': if kwargs['machine'] == 'VExpress_GEM5_V1':
entry_address = 0x80000000 entry_address = 0x80000000
uart_address = 0x1c090000 uart_address = 0x1c090000
elif common.machine == 'RealViewPBX': elif kwargs['machine'] == 'RealViewPBX':
entry_address = 0x10000 entry_address = 0x10000
uart_address = 0x10009000 uart_address = 0x10009000
else: else:
raise Exception('unknown machine: ' + common.machine) raise Exception('unknown machine: ' + kwargs['machine'])
cflags.extend(['-D', 'GEM5'.format(uart_address), LF]) cflags.extend(['-D', 'GEM5'.format(uart_address), LF])
else: else:
entry_address = 0x40000000 entry_address = 0x40000000
uart_address = 0x09000000 uart_address = 0x09000000
os.makedirs(build_dir, exist_ok=True) os.makedirs(build_dir, exist_ok=True)
os.makedirs(common.baremetal_build_lib_dir, exist_ok=True) os.makedirs(kwargs['baremetal_build_lib_dir'], exist_ok=True)
src = os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(kwargs['arch'], common.asm_ext)) src = os.path.join(kwargs['baremetal_src_lib_dir'], '{}{}'.format(kwargs['arch'], kwargs['asm_ext']))
if common.need_rebuild([src], bootloader_obj): if common.need_rebuild([src], bootloader_obj):
self.sh.run_cmd( self.sh.run_cmd(
[gcc, LF] + [gcc, LF] +
@@ -87,7 +87,7 @@ class BaremetalComponent(common.Component):
common_objs=common_objs, common_objs=common_objs,
) )
arch_dir = os.path.join('arch', kwargs['arch']) arch_dir = os.path.join('arch', kwargs['arch'])
if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)): if os.path.isdir(os.path.join(kwargs['baremetal_src_dir'], arch_dir)):
self._build_dir( self._build_dir(
arch_dir, arch_dir,
gcc=gcc, gcc=gcc,
@@ -97,7 +97,7 @@ class BaremetalComponent(common.Component):
common_objs=common_objs, common_objs=common_objs,
) )
arch_dir = os.path.join('arch', kwargs['arch'], 'no_bootloader') arch_dir = os.path.join('arch', kwargs['arch'], 'no_bootloader')
if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)): if os.path.isdir(os.path.join(kwargs['baremetal_src_dir'], arch_dir)):
self._build_dir( self._build_dir(
arch_dir, arch_dir,
gcc=gcc, gcc=gcc,
@@ -116,7 +116,7 @@ Build the baremetal examples with crosstool-NG.
} }
def get_build_dir(self, args): def get_build_dir(self, args):
return common.baremetal_build_dir return kwargs['baremetal_build_dir']
def get_default_args(self): def get_default_args(self):
return {'baremetal': 'all'} return {'baremetal': 'all'}
@@ -137,18 +137,18 @@ Build the baremetal examples with crosstool-NG.
Place outputs on the same subpath or the output directory. Place outputs on the same subpath or the output directory.
""" """
in_dir = os.path.join(common.baremetal_src_dir, subpath) in_dir = os.path.join(kwargs['baremetal_src_dir'], subpath)
out_dir = os.path.join(common.baremetal_build_dir, subpath) out_dir = os.path.join(kwargs['baremetal_build_dir'], subpath)
os.makedirs(out_dir, exist_ok=True) os.makedirs(out_dir, exist_ok=True)
common_objs = common_objs.copy() common_objs = common_objs.copy()
if bootloader: if bootloader:
common_objs.append(bootloader_obj) common_objs.append(bootloader_obj)
for in_basename in os.listdir(in_dir): for in_basename in os.listdir(in_dir):
in_path = os.path.join(in_dir, in_basename) in_path = os.path.join(in_dir, in_basename)
if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext): if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (kwargs['c_ext'], kwargs['asm_ext']):
in_name = os.path.splitext(in_basename)[0] in_name = os.path.splitext(in_basename)[0]
main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext)) main_obj = os.path.join(kwargs['baremetal_build_dir'], subpath, '{}{}'.format(in_name, kwargs['obj_ext']))
src = os.path.join(common.baremetal_src_dir, in_path) src = os.path.join(kwargs['baremetal_src_dir'], in_path)
if common.need_rebuild([src], main_obj): if common.need_rebuild([src], main_obj):
self.sh.run_cmd( self.sh.run_cmd(
[gcc, LF] + [gcc, LF] +
@@ -160,8 +160,8 @@ Build the baremetal examples with crosstool-NG.
] ]
) )
objs = common_objs + [main_obj] objs = common_objs + [main_obj]
out = os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext) out = os.path.join(kwargs['baremetal_build_dir'], subpath, in_name + kwargs['baremetal_build_ext'])
link_script = os.path.join(common.baremetal_src_dir, 'link.ld') link_script = os.path.join(kwargs['baremetal_src_dir'], 'link.ld')
if common.need_rebuild(objs + [link_script], out): if common.need_rebuild(objs + [link_script], out):
self.sh.run_cmd( self.sh.run_cmd(
[gcc, LF] + [gcc, LF] +
@@ -171,7 +171,7 @@ Build the baremetal examples with crosstool-NG.
'-o', out, LF, '-o', out, LF,
'-T', link_script, LF, '-T', link_script, LF,
] + ] +
common.add_newlines(objs) self.sh.add_newlines(objs)
) )
if __name__ == '__main__': if __name__ == '__main__':

52
build-buildroot
View File

@@ -67,11 +67,11 @@ usually extra Buildroot targets.
def do_build(self, args): def do_build(self, args):
build_dir = self.get_build_dir(args) build_dir = self.get_build_dir(args)
os.makedirs(common.out_dir, exist_ok=True) os.makedirs(kwargs['out_dir'], exist_ok=True)
extra_make_args = common.add_newlines(kwargs['extra_make_args']) extra_make_args = self.sh.add_newlines(kwargs['extra_make_args'])
if kwargs['build_linux']: if kwargs['build_linux']:
extra_make_args.extend(['linux-reconfigure', LF]) extra_make_args.extend(['linux-reconfigure', LF])
if common.emulator == 'gem5': if kwargs['emulator'] == 'gem5':
extra_make_args.extend(['gem5-reconfigure', LF]) extra_make_args.extend(['gem5-reconfigure', LF])
if kwargs['arch'] == 'x86_64': if kwargs['arch'] == 'x86_64':
defconfig = 'qemu_x86_64_defconfig' defconfig = 'qemu_x86_64_defconfig'
@@ -80,24 +80,24 @@ usually extra Buildroot targets.
elif kwargs['arch'] == 'aarch64': elif kwargs['arch'] == 'aarch64':
defconfig = 'qemu_aarch64_virt_defconfig' defconfig = 'qemu_aarch64_virt_defconfig'
br2_external_dirs = [] br2_external_dirs = []
for package_dir in os.listdir(common.packages_dir): for package_dir in os.listdir(kwargs['packages_dir']):
package_dir_abs = os.path.join(common.packages_dir, package_dir) package_dir_abs = os.path.join(kwargs['packages_dir'], package_dir)
if os.path.isdir(package_dir_abs): if os.path.isdir(package_dir_abs):
br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs)) br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs))
br2_external_str = ':'.join(br2_external_dirs) br2_external_str = ':'.join(br2_external_dirs)
self.sh.run_cmd( self.sh.run_cmd(
[ [
'make', LF, 'make', LF,
'O={}'.format(common.buildroot_build_dir), LF, 'O={}'.format(kwargs['buildroot_build_dir']), LF,
'BR2_EXTERNAL={}'.format(br2_external_str), LF, 'BR2_EXTERNAL={}'.format(br2_external_str), LF,
defconfig, LF, defconfig, LF,
], ],
cwd=common.buildroot_src_dir, cwd=kwargs['buildroot_src_dir'],
) )
configs = kwargs['config'] configs = kwargs['config']
configs.extend([ configs.extend([
'BR2_JLEVEL={}'.format(kwargs['nproc']), 'BR2_JLEVEL={}'.format(kwargs['nproc']),
'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir), 'BR2_DL_DIR="{}"'.format(kwargs['buildroot_download_dir']),
]) ])
if not kwargs['build_linux']: if not kwargs['build_linux']:
configs.extend([ configs.extend([
@@ -107,38 +107,38 @@ usually extra Buildroot targets.
if not kwargs['baseline']: if not kwargs['baseline']:
configs.extend([ configs.extend([
'BR2_GLOBAL_PATCH_DIR="{}"'.format( 'BR2_GLOBAL_PATCH_DIR="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global')) self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'patches', 'global'))
), ),
'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format( 'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment')) self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'busybox_config_fragment'))
), ),
'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format( 'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override')) self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'buildroot_override'))
), ),
'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format( 'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script')) self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'rootfs-post-build-script'))
), ),
'BR2_ROOTFS_USERS_TABLES="{}"'.format( 'BR2_ROOTFS_USERS_TABLES="{}"'.format(
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table')) self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'user_table'))
), ),
]) ])
if not kwargs['no_overlay']: if not kwargs['no_overlay']:
configs.append('BR2_ROOTFS_OVERLAY="{}"'.format( configs.append('BR2_ROOTFS_OVERLAY="{}"'.format(
self._path_relative_to_buildroot(common.out_rootfs_overlay_dir) self._path_relative_to_buildroot(kwargs['out_rootfs_overlay_dir'])
)) ))
config_fragments = [ config_fragments = [
os.path.join(common.root_dir, 'buildroot_config', 'default') os.path.join(kwargs['root_dir'], 'buildroot_config', 'default')
] + kwargs['config_fragment'] ] + kwargs['config_fragment']
# TODO Can't get rid of these for now with nice fragments on Buildroot: # TODO Can't get rid of these for now with nice fragments on Buildroot:
# http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
self.sh.write_configs(common.buildroot_config_file, configs, config_fragments) self.sh.write_configs(kwargs['buildroot_config_file'], configs, config_fragments)
self.sh.run_cmd( self.sh.run_cmd(
[ [
'make', LF, 'make', LF,
'O={}'.format(common.buildroot_build_dir), LF, 'O={}'.format(kwargs['buildroot_build_dir']), LF,
'olddefconfig', LF, 'olddefconfig', LF,
], ],
cwd=common.buildroot_src_dir, cwd=kwargs['buildroot_src_dir'],
) )
common.make_build_dirs() common.make_build_dirs()
if not kwargs['no_all']: if not kwargs['no_all']:
@@ -146,21 +146,21 @@ usually extra Buildroot targets.
self.sh.run_cmd( self.sh.run_cmd(
[ [
'make', LF, 'make', LF,
'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_source_dir), LF, 'LKMC_GEM5_SRCDIR="{}"'.format(kwargs['gem5_source_dir']), LF,
'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), LF, 'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(kwargs['parsec_benchmark_src_dir']), LF,
'O={}'.format(common.buildroot_build_dir), LF, 'O={}'.format(kwargs['buildroot_build_dir']), LF,
'V={}'.format(int(kwargs['verbose'])), LF, 'V={}'.format(int(kwargs['verbose'])), LF,
] + ] +
extra_make_args extra_make_args
, ,
out_file=os.path.join(common.buildroot_build_dir, 'lkmc.log'), out_file=os.path.join(kwargs['buildroot_build_dir'], 'lkmc.log'),
delete_env=['LD_LIBRARY_PATH'], delete_env=['LD_LIBRARY_PATH'],
cwd=common.buildroot_src_dir, cwd=kwargs['buildroot_src_dir'],
) )
# Create the qcow2 from ext2. # Create the qcow2 from ext2.
# Skip if qemu is not present, because gem5 does not need the qcow2. # Skip if qemu is not present, because gem5 does not need the qcow2.
# so we don't force a QEMU build for gem5. # so we don't force a QEMU build for gem5.
if not kwargs['no_all'] and os.path.exists(common.qemu_img_executable): if not kwargs['no_all'] and os.path.exists(kwargs['qemu_img_executable']):
common.raw_to_qcow2() common.raw_to_qcow2()
def get_argparse_args(self): def get_argparse_args(self):
@@ -171,7 +171,7 @@ Run Linux on an emulator
} }
def get_build_dir(self, args): def get_build_dir(self, args):
return common.buildroot_build_dir return kwargs['buildroot_build_dir']
_defaults = { _defaults = {
'baseline': False, 'baseline': False,
@@ -184,7 +184,7 @@ Run Linux on an emulator
} }
def _path_relative_to_buildroot(self, abspath): def _path_relative_to_buildroot(self, abspath):
return os.path.relpath(abspath, common.buildroot_src_dir) return os.path.relpath(abspath, kwargs['buildroot_src_dir'])
if __name__ == '__main__': if __name__ == '__main__':
BuildrootComponent().build() BuildrootComponent().build()

30
build-crosstool-ng
View File

@@ -8,20 +8,20 @@ class CrosstoolNgComponent(common.Component):
def do_build(self, args): def do_build(self, args):
common.assert_crosstool_ng_supports_arch(kwargs['arch']) common.assert_crosstool_ng_supports_arch(kwargs['arch'])
build_dir = self.get_build_dir(args) build_dir = self.get_build_dir(args)
defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig') defconfig_dest = os.path.join(kwargs['crosstool_ng_util_dir'], 'defconfig')
os.makedirs(common.crosstool_ng_util_dir, exist_ok=True) os.makedirs(kwargs['crosstool_ng_util_dir'], exist_ok=True)
os.makedirs(common.crosstool_ng_download_dir, exist_ok=True) os.makedirs(kwargs['crosstool_ng_download_dir'], exist_ok=True)
# Bootstrap out-ot-tree WONTFIX. I've tried. # Bootstrap out-ot-tree WONTFIX. I've tried.
# https://github.com/crosstool-ng/crosstool-ng/issues/1021 # https://github.com/crosstool-ng/crosstool-ng/issues/1021
os.chdir(common.crosstool_ng_src_dir) os.chdir(kwargs['crosstool_ng_src_dir'])
self.sh.run_cmd( self.sh.run_cmd(
[os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), LF], [os.path.join(kwargs['crosstool_ng_src_dir'], 'bootstrap'), LF],
) )
os.chdir(common.crosstool_ng_util_dir) os.chdir(kwargs['crosstool_ng_util_dir'])
self.sh.run_cmd( self.sh.run_cmd(
[ [
os.path.join(common.crosstool_ng_src_dir, 'configure'), LF, os.path.join(kwargs['crosstool_ng_src_dir'], 'configure'), LF,
'--enable-local', LF, '--enable-local', LF,
], ],
) )
@@ -34,33 +34,33 @@ class CrosstoolNgComponent(common.Component):
# Build the toolchain. # Build the toolchain.
self.sh.cp( self.sh.cp(
os.path.join(common.root_dir, 'crosstool_ng_config', kwargs['arch']), os.path.join(kwargs['root_dir'], 'crosstool_ng_config', kwargs['arch']),
defconfig_dest defconfig_dest
) )
common.write_configs( common.write_configs(
common.crosstool_ng_defconfig, kwargs['crosstool_ng_defconfig'],
[ [
'CT_PREFIX_DIR="{}"'.format(common.crosstool_ng_install_dir), 'CT_PREFIX_DIR="{}"'.format(kwargs['crosstool_ng_install_dir']),
'CT_WORK_DIR="{}"'.format(build_dir), 'CT_WORK_DIR="{}"'.format(build_dir),
'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir), 'CT_LOCAL_TARBALLS_DIR="{}"'.format(kwargs['crosstool_ng_download_dir']),
] ]
) )
self.sh.run_cmd( self.sh.run_cmd(
[ [
common.crosstool_ng_executable, LF, kwargs['crosstool_ng_executable'], LF,
'defconfig', LF, 'defconfig', LF,
], ],
) )
os.unlink(defconfig_dest) os.unlink(defconfig_dest)
self.sh.run_cmd( self.sh.run_cmd(
[ [
common.crosstool_ng_executable, LF, kwargs['crosstool_ng_executable'], LF,
'build', LF, 'build', LF,
'CT_JOBS={}'.format(str(kwargs['nproc'])), LF, 'CT_JOBS={}'.format(str(kwargs['nproc'])), LF,
], ],
out_file=os.path.join(build_dir, 'lkmc.log'), out_file=os.path.join(build_dir, 'lkmc.log'),
delete_env=['LD_LIBRARY_PATH'], delete_env=['LD_LIBRARY_PATH'],
extra_paths=[common.ccache_dir], extra_paths=[kwargs['ccache_dir']],
) )
def get_argparse_args(self): def get_argparse_args(self):
@@ -71,7 +71,7 @@ Build crosstool-NG with Newlib for bare metal compilation'
} }
def get_build_dir(self, args): def get_build_dir(self, args):
return common.crosstool_ng_build_dir return kwargs['crosstool_ng_build_dir']
if __name__ == '__main__': if __name__ == '__main__':
CrosstoolNgComponent().build() CrosstoolNgComponent().build()

14
build-docker
View File

@@ -44,7 +44,7 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
'--privileged', '--privileged',
'-t', '-t',
'-w', target_dir, '-w', target_dir,
'-v', '{}:{}'.format(common.root_dir, target_dir), '-v', '{}:{}'.format(kwargs['root_dir'], target_dir),
'ubuntu:18.04', 'ubuntu:18.04',
'bash', 'bash',
]) ])
@@ -52,11 +52,11 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
'docker', 'docker',
'export', 'export',
'-o', '-o',
common.docker_tar_file, kwargs['docker_tar_file'],
container_name, container_name,
]) ])
tar = tarfile.open(common.docker_tar_file) tar = tarfile.open(kwargs['docker_tar_file'])
tar.extractall(common.docker_tar_dir) tar.extractall(kwargs['docker_tar_dir'])
tar.close() tar.close()
# sudo not required in theory # sudo not required in theory
# https://askubuntu.com/questions/1046828/how-to-run-libguestfs-tools-tools-such-as-virt-make-fs-without-sudo # https://askubuntu.com/questions/1046828/how-to-run-libguestfs-tools-tools-such-as-virt-make-fs-without-sudo
@@ -65,13 +65,13 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
'--format', 'raw', '--format', 'raw',
'--size', '+1G', '--size', '+1G',
'--type', 'ext2', '--type', 'ext2',
common.docker_tar_dir, kwargs['docker_tar_dir'],
common.docker_rootfs_raw_file, kwargs['docker_rootfs_raw_file'],
]) ])
common.raw_to_qcow2(prebuilt=True) common.raw_to_qcow2(prebuilt=True)
def get_build_dir(self, args): def get_build_dir(self, args):
return common.docker_build_dir return kwargs['docker_build_dir']
def get_default_args(self): def get_default_args(self):
return {'docker': True} return {'docker': True}

68
build-gem5
View File

@@ -16,49 +16,46 @@ class Main(common.BuildCliFunction):
nargs='*', nargs='*',
) )
def build(self, **kwargs): def build(self):
build_dir = self.get_build_dir(**kwargs) build_dir = self.get_build_dir()
binaries_dir = os.path.join(kwargs['gem5_system_dir'], 'binaries') binaries_dir = os.path.join(self.env['gem5_system_dir'], 'binaries')
disks_dir = os.path.join(kwargs['gem5_system_dir'], 'disks') disks_dir = os.path.join(self.env['gem5_system_dir'], 'disks')
os.makedirs(binaries_dir, exist_ok=True) os.makedirs(binaries_dir, exist_ok=True)
os.makedirs(disks_dir, exist_ok=True) os.makedirs(disks_dir, exist_ok=True)
if kwargs['gem5_source_dir'] is None: if self.env['gem5_source_dir'] is None:
if not os.path.exists(os.path.join(kwargs['gem5_source_dir'], '.git')): if not os.path.exists(os.path.join(self.env['gem5_source_dir'], '.git')):
if kwargs['gem5_source_dir'] == kwargs['gem5_default_src_dir']: if self.env['gem5_source_dir'] == self.env['gem5_default_src_dir']:
raise Exception('gem5 submodule not checked out') raise Exception('gem5 submodule not checked out')
self.sh.run_cmd([ self.sh.run_cmd([
'git', LF, 'git', LF,
'-C', kwargs['gem5_default_src_dir'], LF, '-C', self.env['gem5_default_src_dir'], LF,
'worktree', 'add', LF, 'worktree', 'add', LF,
'-b', os.path.join('wt', kwargs['gem5_build_id']), LF, '-b', os.path.join('wt', self.env['gem5_build_id']), LF,
kwargs['gem5_source_dir'], LF, self.env['gem5_source_dir'], LF,
]) ])
if kwargs['verbose']: if self.env['verbose']:
verbose = ['--verbose', LF] verbose = ['--verbose', LF]
else: else:
verbose = [] verbose = []
if kwargs['arch'] == 'x86_64': if self.env['arch'] == 'x86_64':
dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img') dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img')
with open(dummy_img_path, 'wb') as dummy_img_file: with open(dummy_img_path, 'wb') as dummy_img_file:
zeroes = b'\x00' * (2 ** 16) zeroes = b'\x00' * (2 ** 16)
for i in range(2 ** 10): for i in range(2 ** 10):
dummy_img_file.write(zeroes) dummy_img_file.write(zeroes)
self.sh.run_cmd(['mkswap', dummy_img_path]) self.sh.run_cmd(['mkswap', dummy_img_path, LF])
with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'): with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'):
# This file must always be present, despite --kernel overriding that default and selecting the kernel. # This file must always be present, despite --kernel overriding that default and selecting the kernel.
# I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present. # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present.
pass pass
elif kwargs['arch'] == 'arm' or kwargs['arch'] == 'aarch64': elif self.env['arch'] == 'arm' or self.env['arch'] == 'aarch64':
gem5_system_src_dir = os.path.join(kwargs['gem5_source_dir'], 'system') gem5_system_src_dir = os.path.join(self.env['gem5_source_dir'], 'system')
# dtb # dtb
dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt')
dt_build_dir = os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt') dt_build_dir = os.path.join(self.env['gem5_system_dir'], 'arm', 'dt')
self.sh.run_cmd([ self.sh.run_cmd(['make', '-C', dt_src_dir, LF])
'make', LF, self.sh.copy_dir_if_update_non_recursive(
'-C', dt_src_dir, LF,
])
common.copy_dir_if_update_non_recursive(
srcdir=dt_src_dir, srcdir=dt_src_dir,
destdir=dt_build_dir, destdir=dt_build_dir,
filter_ext='.dtb', filter_ext='.dtb',
@@ -78,38 +75,35 @@ class Main(common.BuildCliFunction):
# Bootloader 64. # Bootloader 64.
bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader')
# TODO cross_compile is ignored because the make does not use CC... # TODO cross_compile is ignored because the make does not use CC...
self.sh.run_cmd([ self.sh.run_cmd(['make', '-C', bootloader64_dir, LF])
'make', LF,
'-C', bootloader64_dir, LF
])
self.sh.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) self.sh.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir)
self.sh.run_cmd( self.sh.run_cmd(
( (
[ [
'scons', LF, 'scons', LF,
'-j', str(kwargs['nproc']), LF, '-j', str(self.env['nproc']), LF,
'--gold-linker', LF, '--gold-linker', LF,
'--ignore-style', LF, '--ignore-style', LF,
kwargs['gem5_executable'], LF, self.env['gem5_executable'], LF,
] + ] +
verbose + verbose +
self.sh.add_newlines(kwargs['extra_scons_args']) self.sh.add_newlines(self.env['extra_scons_args'])
), ),
cwd=kwargs['gem5_source_dir'], cwd=self.env['gem5_source_dir'],
extra_paths=[kwargs['ccache_dir']], extra_paths=[self.env['ccache_dir']],
) )
term_src_dir = os.path.join(kwargs['gem5_source_dir'], 'util/term') term_src_dir = os.path.join(self.env['gem5_source_dir'], 'util/term')
m5term_build = os.path.join(term_src_dir, 'm5term') m5term_build = os.path.join(term_src_dir, 'm5term')
self.sh.run_cmd(['make', '-C', term_src_dir]) self.sh.run_cmd(['make', '-C', term_src_dir, LF])
if os.path.exists(kwargs['gem5_m5term']): if os.path.exists(self.env['gem5_m5term']):
# Otherwise self.sh.cp would fail with "Text file busy" if you # Otherwise self.sh.cp would fail with "Text file busy" if you
# tried to rebuild while running m5term: # tried to rebuild while running m5term:
# https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512 # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512
os.unlink(kwargs['gem5_m5term']) os.unlink(self.env['gem5_m5term'])
self.sh.cp(m5term_build, kwargs['gem5_m5term']) self.sh.cp(m5term_build, self.env['gem5_m5term'])
def get_build_dir(self, **kwargs): def get_build_dir(self):
return kwargs['gem5_build_dir'] return self.env['gem5_build_dir']
if __name__ == '__main__': if __name__ == '__main__':
Main().cli() Main().cli()

66
build-linux
View File

@@ -6,8 +6,13 @@ import shutil
import common import common
from shell_helpers import LF from shell_helpers import LF
class LinuxComponent(common.Component): class Main(common.BuildCliFunction):
def __init__(self, parser): def __init__(self):
super().__init__(
description='''\
Build the Linux kernel.
'''
)
self.add_argument( self.add_argument(
'--config', default=[], action='append', '--config', default=[], action='append',
help='''\ help='''\
@@ -44,48 +49,48 @@ Configure the kernel, but don't build it.
nargs='*' nargs='*'
) )
def build(self, **kwargs): def build(self):
build_dir = self.get_build_dir(**kwargs) build_dir = self.get_build_dir()
if kwargs['initrd'] or kwargs['initramfs']: if self.env['initrd'] or self.env['initramfs']:
raise Exception('just trolling, --initrd and --initramfs are broken for now') raise Exception('just trolling, --initrd and --initramfs are broken for now')
os.makedirs(build_dir, exist_ok=True) os.makedirs(build_dir, exist_ok=True)
tool = 'gcc' tool = 'gcc'
gcc = common.get_toolchain_tool(tool) gcc = self.get_toolchain_tool(tool)
prefix = gcc[:-len(tool)] prefix = gcc[:-len(tool)]
common_args = { common_args = {
'cwd': common.linux_src_dir, 'cwd': self.env['linux_src_dir'],
} }
ccache = shutil.which('ccache') ccache = shutil.which('ccache')
if ccache is not None: if ccache is not None:
cc = '{} {}'.format(ccache, gcc) cc = '{} {}'.format(ccache, gcc)
else: else:
cc = gcc cc = gcc
if kwargs['verbose']: if self.env['verbose']:
verbose = ['V=1'] verbose = ['V=1']
else: else:
verbose = [] verbose = []
common_make_args = [ common_make_args = [
'make', LF, 'make', LF,
'-j', str(kwargs['nproc']), LF, '-j', str(self.env['nproc']), LF,
'ARCH={}'.format(common.linux_arch), LF, 'ARCH={}'.format(self.env['linux_arch']), LF,
'CROSS_COMPILE={}'.format(prefix), LF, 'CROSS_COMPILE={}'.format(prefix), LF,
'CC={}'.format(cc), LF, 'CC={}'.format(cc), LF,
'O={}'.format(build_dir), LF, 'O={}'.format(build_dir), LF,
] + verbose ] + verbose
if kwargs['custom_config_file'] is not None: if self.env['custom_config_file'] is not None:
if not os.path.exists(kwargs['custom_config_file']): if not os.path.exists(self.env['custom_config_file']):
raise Exception('config fragment file does not exist: {}'.format(kwargs['custom_config_file'])) raise Exception('config fragment file does not exist: {}'.format(self.env['custom_config_file']))
base_config_file = kwargs['custom_config_file'] base_config_file = self.env['custom_config_file']
config_fragments = [] config_fragments = []
else: else:
base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(kwargs['arch'])) base_config_file = os.path.join(self.env['linux_config_dir'], 'buildroot-{}'.format(self.env['arch']))
config_fragments = ['min', 'default'] config_fragments = ['min', 'default']
for i, config_fragment in enumerate(config_fragments): for i, config_fragment in enumerate(config_fragments):
config_fragments[i] = os.path.join(common.linux_config_dir, config_fragment) config_fragments[i] = os.path.join(self.env['linux_config_dir'], config_fragment)
config_fragments.extend(kwargs['config_fragment']) config_fragments.extend(self.env['config_fragment'])
if kwargs['config'] != []: if self.env['config'] != []:
cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment') cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment')
cli_config_str = '\n'.join(kwargs['config']) cli_config_str = '\n'.join(self.env['config'])
common.write_string_to_file(cli_config_fragment_path, cli_config_str) common.write_string_to_file(cli_config_fragment_path, cli_config_str)
config_fragments.append(cli_config_fragment_path) config_fragments.append(cli_config_fragment_path)
self.sh.cp( self.sh.cp(
@@ -94,12 +99,12 @@ Configure the kernel, but don't build it.
) )
self.sh.run_cmd( self.sh.run_cmd(
[ [
os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), LF, os.path.join(self.env['linux_src_dir'], 'scripts', 'kconfig', 'merge_config.sh'), LF,
'-m', LF, '-m', LF,
'-O', build_dir, LF, '-O', build_dir, LF,
os.path.join(build_dir, '.config'), LF, os.path.join(build_dir, '.config'), LF,
] + ] +
common.add_newlines(config_fragments) self.sh.add_newlines(config_fragments)
) )
self.sh.run_cmd( self.sh.run_cmd(
( (
@@ -108,11 +113,11 @@ Configure the kernel, but don't build it.
), ),
**common_args **common_args
) )
if not kwargs['config_only']: if not self.env['config_only']:
self.sh.run_cmd( self.sh.run_cmd(
( (
common_make_args + common_make_args +
common.add_newlines(kwargs['extra_make_args']) self.sh.add_newlines(self.env['extra_make_args'])
), ),
**common_args **common_args
) )
@@ -120,7 +125,7 @@ Configure the kernel, but don't build it.
( (
common_make_args + common_make_args +
[ [
'INSTALL_MOD_PATH={}'.format(common.out_rootfs_overlay_dir), LF, 'INSTALL_MOD_PATH={}'.format(self.env['out_rootfs_overlay_dir']), LF,
'modules_install', LF, 'modules_install', LF,
] ]
), ),
@@ -130,15 +135,8 @@ Configure the kernel, but don't build it.
# TODO Basically all kernel modules also basically leak full host paths. Just terrible. Buildroot deals with that stuff nicely for us. # TODO Basically all kernel modules also basically leak full host paths. Just terrible. Buildroot deals with that stuff nicely for us.
# common.rmrf() # common.rmrf()
def get_argparse_args(self): def get_build_dir(self):
return { return self.env['linux_build_dir']
'description': '''\
Build the Linux kernel.
'''
}
def get_build_dir(self, args):
return common.linux_build_dir
if __name__ == '__main__': if __name__ == '__main__':
LinuxComponent().build() Main().cli()

12
build-m5
View File

@@ -19,25 +19,25 @@ class M5Component(common.Component):
'-f', 'Makefile.{}'.format(arch), LF, '-f', 'Makefile.{}'.format(arch), LF,
'CC={}'.format(cc), LF, 'CC={}'.format(cc), LF,
'LD={}'.format(ld), LF, 'LD={}'.format(ld), LF,
'PWD={}'.format(common.gem5_m5_source_dir), LF, 'PWD={}'.format(kwargs['gem5_m5_source_dir']), LF,
] ]
def do_build(self, args): def do_build(self, args):
os.makedirs(common.gem5_m5_build_dir, exist_ok=True) os.makedirs(kwargs['gem5_m5_build_dir'], exist_ok=True)
# We must clean first or else the build outputs of one arch can conflict with the other. # We must clean first or else the build outputs of one arch can conflict with the other.
# I should stop being lazy and go actually patch gem5 to support out of tree m5 build... # I should stop being lazy and go actually patch gem5 to support out of tree m5 build...
self.clean(args) self.clean(args)
self.sh.run_cmd( self.sh.run_cmd(
self.get_make_cmd(args), self.get_make_cmd(args),
cwd=common.gem5_m5_source_dir, cwd=kwargs['gem5_m5_source_dir'],
) )
os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True) os.makedirs(kwargs['out_rootfs_overlay_bin_dir'], exist_ok=True)
self.sh.cp(os.path.join(common.gem5_m5_source_dir, 'm5'), common.out_rootfs_overlay_bin_dir) self.sh.cp(os.path.join(kwargs['gem5_m5_source_dir'], 'm5'), kwargs['out_rootfs_overlay_bin_dir'])
def clean(self, args): def clean(self, args):
self.sh.run_cmd( self.sh.run_cmd(
self.get_make_cmd(args) + ['clean', LF], self.get_make_cmd(args) + ['clean', LF],
cwd=common.gem5_m5_source_dir, cwd=kwargs['gem5_m5_source_dir'],
) )
if __name__ == '__main__': if __name__ == '__main__':

36
build-modules
View File

@@ -42,34 +42,34 @@ Use the host packaged cross toolchain.
# This copies only modified files as per: # This copies only modified files as per:
# https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory # https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory
distutils.dir_util.copy_tree( distutils.dir_util.copy_tree(
common.kernel_modules_src_dir, kwargs['kernel_modules_src_dir'],
os.path.join(build_dir, common.kernel_modules_subdir), os.path.join(build_dir, kwargs['kernel_modules_subdir']),
update=1, update=1,
) )
distutils.dir_util.copy_tree( distutils.dir_util.copy_tree(
common.include_src_dir, kwargs['include_src_dir'],
os.path.join(build_dir, common.include_subdir), os.path.join(build_dir, kwargs['include_subdir']),
update=1, update=1,
) )
all_kernel_modules = [] all_kernel_modules = []
for basename in os.listdir(common.kernel_modules_src_dir): for basename in os.listdir(kwargs['kernel_modules_src_dir']):
src = os.path.join(common.kernel_modules_src_dir, basename) src = os.path.join(kwargs['kernel_modules_src_dir'], basename)
if os.path.isfile(src): if os.path.isfile(src):
noext, ext = os.path.splitext(basename) noext, ext = os.path.splitext(basename)
if ext == common.c_ext: if ext == kwargs['c_ext']:
all_kernel_modules.append(noext) all_kernel_modules.append(noext)
if kwargs['kernel_modules'] == []: if kwargs['kernel_modules'] == []:
kernel_modules = all_kernel_modules kernel_modules = all_kernel_modules
else: else:
kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], kwargs['kernel_modules']) kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], kwargs['kernel_modules'])
object_files = map(lambda x: x + common.obj_ext, kernel_modules) object_files = map(lambda x: x + kwargs['obj_ext'], kernel_modules)
tool = 'gcc' tool = 'gcc'
if kwargs['host']: if kwargs['host']:
allowed_toolchains = ['host'] allowed_toolchains = ['host']
build_subdir = common.kernel_modules_build_host_subdir build_subdir = kwargs['kernel_modules_build_host_subdir']
else: else:
allowed_toolchains = None allowed_toolchains = None
build_subdir = common.kernel_modules_build_subdir build_subdir = kwargs['kernel_modules_build_subdir']
gcc = common.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains) gcc = common.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains)
prefix = gcc[:-len(tool)] prefix = gcc[:-len(tool)]
ccache = shutil.which('ccache') ccache = shutil.which('ccache')
@@ -84,13 +84,13 @@ Use the host packaged cross toolchain.
if kwargs['host']: if kwargs['host']:
linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build') linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build')
else: else:
linux_dir = common.linux_build_dir linux_dir = kwargs['linux_build_dir']
self.sh.run_cmd( self.sh.run_cmd(
( (
[ [
'make', LF, 'make', LF,
'-j', str(kwargs['nproc']), LF, '-j', str(kwargs['nproc']), LF,
'ARCH={}'.format(common.linux_arch), LF, 'ARCH={}'.format(kwargs['linux_arch']), LF,
'CC={}'.format(cc), LF, 'CC={}'.format(cc), LF,
'CROSS_COMPILE={}'.format(prefix), LF, 'CROSS_COMPILE={}'.format(prefix), LF,
'LINUX_DIR={}'.format(linux_dir), LF, 'LINUX_DIR={}'.format(linux_dir), LF,
@@ -100,13 +100,13 @@ Use the host packaged cross toolchain.
common.shlex_split(kwargs['make_args']) + common.shlex_split(kwargs['make_args']) +
verbose verbose
), ),
cwd=os.path.join(common.kernel_modules_build_subdir), cwd=os.path.join(kwargs['kernel_modules_build_subdir']),
) )
if not kwargs['host']: if not kwargs['host']:
common.copy_dir_if_update_non_recursive( common.copy_dir_if_update_non_recursive(
srcdir=common.kernel_modules_build_subdir, srcdir=kwargs['kernel_modules_build_subdir'],
destdir=common.out_rootfs_overlay_dir, destdir=kwargs['out_rootfs_overlay_dir'],
filter_ext=common.kernel_module_ext, filter_ext=kwargs['kernel_module_ext'],
) )
def get_argparse_args(self): def get_argparse_args(self):
@@ -120,9 +120,9 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host
def get_build_dir(self, args): def get_build_dir(self, args):
if kwargs['host']: if kwargs['host']:
return os.path.join(common.kernel_modules_build_host_dir) return os.path.join(kwargs['kernel_modules_build_host_dir'])
else: else:
return os.path.join(common.kernel_modules_build_dir) return os.path.join(kwargs['kernel_modules_build_dir'])
if __name__ == '__main__': if __name__ == '__main__':
ModulesComponent().build() ModulesComponent().build()

10
build-qemu
View File

@@ -32,15 +32,15 @@ class QemuComponent(common.Component):
target_list = '{}-softmmu'.format(kwargs['arch']) target_list = '{}-softmmu'.format(kwargs['arch'])
self.sh.run_cmd( self.sh.run_cmd(
[ [
os.path.join(common.qemu_src_dir, 'configure'), LF, os.path.join(kwargs['qemu_src_dir'], 'configure'), LF,
'--enable-debug', LF, '--enable-debug', LF,
'--enable-trace-backends=simple', LF, '--enable-trace-backends=simple', LF,
'--target-list={}'.format(target_list), LF, '--target-list={}'.format(target_list), LF,
'--enable-sdl', LF, '--enable-sdl', LF,
'--with-sdlabi=2.0', LF, '--with-sdlabi=2.0', LF,
] + ] +
common.add_newlines(kwargs['extra_config_args']), self.sh.add_newlines(kwargs['extra_config_args']),
extra_paths=[common.ccache_dir], extra_paths=[kwargs['ccache_dir']],
cwd=build_dir cwd=build_dir
) )
self.sh.run_cmd( self.sh.run_cmd(
@@ -53,11 +53,11 @@ class QemuComponent(common.Component):
verbose verbose
), ),
cwd=build_dir, cwd=build_dir,
extra_paths=[common.ccache_dir], extra_paths=[kwargs['ccache_dir']],
) )
def get_build_dir(self, args): def get_build_dir(self, args):
return common.qemu_build_dir return kwargs['qemu_build_dir']
if __name__ == '__main__': if __name__ == '__main__':
QemuComponent().build() QemuComponent().build()

22
build-userland
View File

@@ -59,25 +59,25 @@ has the OpenBLAS libraries and headers installed.
'make', LF, 'make', LF,
'-j', str(kwargs['nproc']), LF, '-j', str(kwargs['nproc']), LF,
'ARCH={}'.format(kwargs['arch']), LF, 'ARCH={}'.format(kwargs['arch']), LF,
'CCFLAGS_SCRIPT={} {}'.format('-I', common.userland_src_dir), LF, 'CCFLAGS_SCRIPT={} {}'.format('-I', kwargs['userland_src_dir']), LF,
'COMMON_DIR={}'.format(common.root_dir), LF, 'COMMON_DIR={}'.format(kwargs['root_dir']), LF,
'CC={}'.format(cc), LF, 'CC={}'.format(cc), LF,
'CXX={}'.format(cxx), LF, 'CXX={}'.format(cxx), LF,
'PKG_CONFIG={}'.format(common.buildroot_pkg_config), LF, 'PKG_CONFIG={}'.format(kwargs['buildroot_pkg_config']), LF,
'STAGING_DIR={}'.format(common.buildroot_staging_dir), LF, 'STAGING_DIR={}'.format(kwargs['buildroot_staging_dir']), LF,
'OUT_DIR={}'.format(build_dir), LF, 'OUT_DIR={}'.format(build_dir), LF,
] + ] +
common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in kwargs['has_package']]) + self.sh.add_newlines(['HAS_{}=y'.format(package.upper()) for package in kwargs['has_package']]) +
shlex.split(kwargs['make_args']) + shlex.split(kwargs['make_args']) +
common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in kwargs['targets']]) self.sh.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + kwargs['userland_build_ext'] for target in kwargs['targets']])
), ),
cwd=common.userland_src_dir, cwd=kwargs['userland_src_dir'],
extra_paths=[common.ccache_dir], extra_paths=[kwargs['ccache_dir']],
) )
common.copy_dir_if_update_non_recursive( common.copy_dir_if_update_non_recursive(
srcdir=build_dir, srcdir=build_dir,
destdir=common.out_rootfs_overlay_dir, destdir=kwargs['out_rootfs_overlay_dir'],
filter_ext=common.userland_build_ext, filter_ext=kwargs['userland_build_ext'],
) )
def get_argparse_args(self): def get_argparse_args(self):
@@ -86,7 +86,7 @@ has the OpenBLAS libraries and headers installed.
} }
def get_build_dir(self, args): def get_build_dir(self, args):
return common.userland_build_dir return kwargs['userland_build_dir']
if __name__ == '__main__': if __name__ == '__main__':
UserlandComponent().build() UserlandComponent().build()

18
cli_function.py
View File

@@ -119,11 +119,11 @@ class CliFunction:
raise Exception('Value not given for mandatory argument: ' + key) raise Exception('Value not given for mandatory argument: ' + key)
return self.main(**args_with_defaults) return self.main(**args_with_defaults)
def __init__(self, config_file=None): def __init__(self, config_file=None, description=None):
self._all_keys = set() self._all_keys = set()
self._arguments = [] self._arguments = []
self.description = None
self._config_file = config_file self._config_file = config_file
self._description = description
if self._config_file is not None: if self._config_file is not None:
self.add_argument( self.add_argument(
'--config-file', '--config-file',
@@ -146,7 +146,7 @@ class CliFunction:
to get all arguments. to get all arguments.
''' '''
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description=self.description, description=self._description,
formatter_class=argparse.RawTextHelpFormatter, formatter_class=argparse.RawTextHelpFormatter,
) )
for argument in self._arguments: for argument in self._arguments:
@@ -177,7 +177,13 @@ class CliFunction:
if __name__ == '__main__': if __name__ == '__main__':
class OneCliFunction(CliFunction): class OneCliFunction(CliFunction):
def __init__(self): def __init__(self):
super().__init__(config_file='cli_function_test_config.py') super().__init__(
config_file='cli_function_test_config.py',
description = '''\
Description of this
amazing function!
''',
)
self.add_argument('-a', '--asdf', default='A', help='Help for asdf'), self.add_argument('-a', '--asdf', default='A', help='Help for asdf'),
self.add_argument('-q', '--qwer', default='Q', help='Help for qwer'), self.add_argument('-q', '--qwer', default='Q', help='Help for qwer'),
self.add_argument('-b', '--bool', default=True, help='Help for bool'), self.add_argument('-b', '--bool', default=True, help='Help for bool'),
@@ -186,10 +192,6 @@ if __name__ == '__main__':
self.add_argument('pos-mandatory', help='Help for pos-mandatory', type=int), self.add_argument('pos-mandatory', help='Help for pos-mandatory', type=int),
self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int), self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int),
self.add_argument('args-star', help='Help for args-star', nargs='*'), self.add_argument('args-star', help='Help for args-star', nargs='*'),
self.description = '''\
Description of this
amazing function!
'''
def main(self, **kwargs): def main(self, **kwargs):
del kwargs['config_file'] del kwargs['config_file']
return kwargs return kwargs

684
common.py
View File

@@ -5,7 +5,6 @@ import base64
import collections import collections
import copy import copy
import datetime import datetime
import distutils.file_util
import glob import glob
import imp import imp
import json import json
@@ -99,8 +98,9 @@ class LkmcCliFunction(cli_function.CliFunction):
* command timing * command timing
* some common flags, e.g.: --arch, --dry-run, --verbose * some common flags, e.g.: --arch, --dry-run, --verbose
''' '''
def __init__(self): def __init__(self, *args, **kwargs):
super().__init__(config_file=common.consts['config_file']) kwargs['config_file'] = consts['config_file']
super().__init__(*args, **kwargs)
# Args for all scripts. # Args for all scripts.
self.add_argument( self.add_argument(
@@ -262,24 +262,327 @@ to allow overriding configs from the CLI.
''' '''
) )
def _init_env(self, env):
'''
Update the kwargs from the command line with derived arguments.
'''
def join(*paths):
return os.path.join(*paths)
if env['qemu'] or not env['gem5']:
env['emulator'] = 'qemu'
else:
env['emulator'] = 'gem5'
if env['arch'] in env['arch_short_to_long_dict']:
env['arch'] = env['arch_short_to_long_dict'][env['arch']]
if env['userland_build_id'] is None:
env['userland_build_id'] = env['default_build_id']
env['userland_build_id_given'] = False
else:
env['userland_build_id_given'] = True
if env['gem5_worktree'] is not None and env['gem5_build_id'] is None:
env['gem5_build_id'] = env['gem5_worktree']
env['is_arm'] = False
if env['arch'] == 'arm':
env['armv'] = 7
env['gem5_arch'] = 'ARM'
env['mcpu'] = 'cortex-a15'
env['buildroot_toolchain_prefix'] = 'arm-buildroot-linux-uclibcgnueabihf'
env['crosstool_ng_toolchain_prefix'] = 'arm-unknown-eabi'
env['ubuntu_toolchain_prefix'] = 'arm-linux-gnueabihf'
if env['emulator'] == 'gem5':
if env['machine'] is None:
env['machine'] = 'VExpress_GEM5_V1'
else:
if env['machine'] is None:
env['machine'] = 'virt'
env['is_arm'] = True
elif env['arch'] == 'aarch64':
env['armv'] = 8
env['gem5_arch'] = 'ARM'
env['mcpu'] = 'cortex-a57'
env['buildroot_toolchain_prefix'] = 'aarch64-buildroot-linux-uclibc'
env['crosstool_ng_toolchain_prefix'] = 'aarch64-unknown-elf'
env['ubuntu_toolchain_prefix'] = 'aarch64-linux-gnu'
if env['emulator'] == 'gem5':
if env['machine'] is None:
env['machine'] = 'VExpress_GEM5_V1'
else:
if env['machine'] is None:
env['machine'] = 'virt'
env['is_arm'] = True
elif env['arch'] == 'x86_64':
env['crosstool_ng_toolchain_prefix'] = 'x86_64-unknown-elf'
env['gem5_arch'] = 'X86'
env['buildroot_toolchain_prefix'] = 'x86_64-buildroot-linux-uclibc'
env['ubuntu_toolchain_prefix'] = 'x86_64-linux-gnu'
if env['emulator'] == 'gem5':
if env['machine'] is None:
env['machine'] = 'TODO'
else:
if env['machine'] is None:
env['machine'] = 'pc'
# Buildroot
env['buildroot_build_dir'] = join(env['buildroot_out_dir'], 'build', env['buildroot_build_id'], env['arch'])
env['buildroot_download_dir'] = join(env['buildroot_out_dir'], 'download')
env['buildroot_config_file'] = join(env['buildroot_build_dir'], '.config')
env['buildroot_build_build_dir'] = join(env['buildroot_build_dir'], 'build')
env['buildroot_linux_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom')
env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], "vmlinux")
env['host_dir'] = join(env['buildroot_build_dir'], 'host')
env['host_bin_dir'] = join(env['host_dir'], 'usr', 'bin')
env['buildroot_pkg_config'] = join(env['host_bin_dir'], 'pkg-config')
env['buildroot_images_dir'] = join(env['buildroot_build_dir'], 'images')
env['buildroot_rootfs_raw_file'] = join(env['buildroot_images_dir'], 'rootfs.ext2')
env['buildroot_qcow2_file'] = env['buildroot_rootfs_raw_file'] + '.qcow2'
env['staging_dir'] = join(env['out_dir'], 'staging', env['arch'])
env['buildroot_staging_dir'] = join(env['buildroot_build_dir'], 'staging')
env['target_dir'] = join(env['buildroot_build_dir'], 'target')
env['linux_buildroot_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom')
# QEMU
env['qemu_build_dir'] = join(env['out_dir'], 'qemu', env['qemu_build_id'])
env['qemu_executable_basename'] = 'qemu-system-{}'.format(env['arch'])
env['qemu_executable'] = join(env['qemu_build_dir'], '{}-softmmu'.format(env['arch']), env['qemu_executable_basename'])
env['qemu_img_basename'] = 'qemu-img'
env['qemu_img_executable'] = join(env['qemu_build_dir'], env['qemu_img_basename'])
# gem5
if env['gem5_build_dir'] is None:
env['gem5_build_dir'] = join(env['gem5_out_dir'], env['gem5_build_id'], env['gem5_build_type'])
env['gem5_fake_iso'] = join(env['gem5_out_dir'], 'fake.iso')
env['gem5_m5term'] = join(env['gem5_build_dir'], 'm5term')
env['gem5_build_build_dir'] = join(env['gem5_build_dir'], 'build')
env['gem5_executable'] = join(env['gem5_build_build_dir'], env['gem5_arch'], 'gem5.{}'.format(env['gem5_build_type']))
env['gem5_system_dir'] = join(env['gem5_build_dir'], 'system')
# gem5 source
if env['gem5_source_dir'] is not None:
assert os.path.exists(env['gem5_source_dir'])
else:
if env['gem5_worktree'] is not None:
env['gem5_source_dir'] = join(env['gem5_non_default_src_root_dir'], env['gem5_worktree'])
else:
env['gem5_source_dir'] = env['gem5_default_src_dir']
env['gem5_m5_source_dir'] = join(env['gem5_source_dir'], 'util', 'm5')
env['gem5_config_dir'] = join(env['gem5_source_dir'], 'configs')
env['gem5_se_file'] = join(env['gem5_config_dir'], 'example', 'se.py')
env['gem5_fs_file'] = join(env['gem5_config_dir'], 'example', 'fs.py')
# crosstool-ng
env['crosstool_ng_buildid_dir'] = join(env['crosstool_ng_out_dir'], 'build', env['crosstool_ng_build_id'])
env['crosstool_ng_install_dir'] = join(env['crosstool_ng_buildid_dir'], 'install', env['arch'])
env['crosstool_ng_bin_dir'] = join(env['crosstool_ng_install_dir'], 'bin')
env['crosstool_ng_util_dir'] = join(env['crosstool_ng_buildid_dir'], 'util')
env['crosstool_ng_config'] = join(env['crosstool_ng_util_dir'], '.config')
env['crosstool_ng_defconfig'] = join(env['crosstool_ng_util_dir'], 'defconfig')
env['crosstool_ng_executable'] = join(env['crosstool_ng_util_dir'], 'ct-ng')
env['crosstool_ng_build_dir'] = join(env['crosstool_ng_buildid_dir'], 'build')
env['crosstool_ng_download_dir'] = join(env['crosstool_ng_out_dir'], 'download')
# run
env['gem5_run_dir'] = join(env['run_dir_base'], 'gem5', env['arch'], str(env['run_id']))
env['m5out_dir'] = join(env['gem5_run_dir'], 'm5out')
env['stats_file'] = join(env['m5out_dir'], 'stats.txt')
env['gem5_trace_txt_file'] = join(env['m5out_dir'], 'trace.txt')
env['gem5_guest_terminal_file'] = join(env['m5out_dir'], 'system.terminal')
env['gem5_readfile'] = join(env['gem5_run_dir'], 'readfile')
env['gem5_termout_file'] = join(env['gem5_run_dir'], 'termout.txt')
env['qemu_run_dir'] = join(env['run_dir_base'], 'qemu', env['arch'], str(env['run_id']))
env['qemu_termout_file'] = join(env['qemu_run_dir'], 'termout.txt')
env['qemu_trace_basename'] = 'trace.bin'
env['qemu_trace_file'] = join(env['qemu_run_dir'], 'trace.bin')
env['qemu_trace_txt_file'] = join(env['qemu_run_dir'], 'trace.txt')
env['qemu_rrfile'] = join(env['qemu_run_dir'], 'rrfile')
env['gem5_out_dir'] = join(env['out_dir'], 'gem5')
# Ports
if env['port_offset'] is None:
try:
env['port_offset'] = int(env['run_id'])
except ValueError:
env['port_offset'] = 0
if env['emulator'] == 'gem5':
env['gem5_telnet_port'] = 3456 + env['port_offset']
env['gdb_port'] = 7000 + env['port_offset']
else:
env['qemu_base_port'] = 45454 + 10 * env['port_offset']
env['qemu_monitor_port'] = env['qemu_base_port'] + 0
env['qemu_hostfwd_generic_port'] = env['qemu_base_port'] + 1
env['qemu_hostfwd_ssh_port'] = env['qemu_base_port'] + 2
env['qemu_gdb_port'] = env['qemu_base_port'] + 3
env['extra_serial_port'] = env['qemu_base_port'] + 4
env['gdb_port'] = env['qemu_gdb_port']
env['qemu_background_serial_file'] = join(env['qemu_run_dir'], 'background.log')
# gem5 QEMU polymorphism.
if env['emulator'] == 'gem5':
env['executable'] = env['gem5_executable']
env['run_dir'] = env['gem5_run_dir']
env['termout_file'] = env['gem5_termout_file']
env['guest_terminal_file'] = env['gem5_guest_terminal_file']
env['trace_txt_file'] = env['gem5_trace_txt_file']
else:
env['executable'] = env['qemu_executable']
env['run_dir'] = env['qemu_run_dir']
env['termout_file'] = env['qemu_termout_file']
env['trace_txt_file'] = env['qemu_trace_txt_file']
env['run_cmd_file'] = join(env['run_dir'], 'run.sh')
# Linux kernl.
if 'linux_build_id' in env:
env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch'])
env['lkmc_vmlinux'] = join(env['linux_build_dir'], "vmlinux")
if env['arch'] == 'arm':
env['linux_arch'] = 'arm'
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage')
elif env['arch'] == 'aarch64':
env['linux_arch'] = 'arm64'
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'Image')
elif env['arch'] == 'x86_64':
env['linux_arch'] = 'x86'
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'bzImage')
env['lkmc_linux_image'] = join(env['linux_build_dir'], env['linux_image_prefix'])
env['buildroot_linux_image'] = join(env['buildroot_linux_build_dir'], env['linux_image_prefix'])
if env['buildroot_linux']:
env['vmlinux'] = env['buildroot_vmlinux']
env['linux_image'] = env['buildroot_linux_image']
else:
env['vmlinux'] = env['lkmc_vmlinux']
env['linux_image'] = env['lkmc_linux_image']
# Kernel modules.
env['kernel_modules_build_dir'] = join(env['kernel_modules_build_base_dir'], env['arch'])
env['kernel_modules_build_subdir'] = join(env['kernel_modules_build_dir'], env['kernel_modules_subdir'])
env['kernel_modules_build_host_dir'] = join(env['kernel_modules_build_base_dir'], 'host')
env['kernel_modules_build_host_subdir'] = join(env['kernel_modules_build_host_dir'], env['kernel_modules_subdir'])
env['userland_build_dir'] = join(env['out_dir'], 'userland', env['userland_build_id'], env['arch'])
env['out_rootfs_overlay_dir'] = join(env['out_dir'], 'rootfs_overlay', env['arch'])
env['out_rootfs_overlay_bin_dir'] = join(env['out_rootfs_overlay_dir'], 'bin')
# Baremetal.
env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal')
env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename'])
if env['emulator'] == 'gem5':
env['simulator_name'] = 'gem5'
else:
env['simulator_name'] = 'qemu'
env['baremetal_build_dir'] = join(env['out_dir'], 'baremetal', env['arch'], env['simulator_name'], env['machine'])
env['baremetal_build_lib_dir'] = join(env['baremetal_build_dir'], env['baremetal_lib_basename'])
env['baremetal_build_ext'] = '.elf'
# Docker
env['docker_build_dir'] = join(env['out_dir'], 'docker', env['arch'])
env['docker_tar_dir'] = join(env['docker_build_dir'], 'export')
env['docker_tar_file'] = join(env['docker_build_dir'], 'export.tar')
env['docker_rootfs_raw_file'] = join(env['docker_build_dir'], 'export.ext2')
env['docker_qcow2_file'] = join(env['docker_rootfs_raw_file'] + '.qcow2')
if env['docker']:
env['rootfs_raw_file'] = env['docker_rootfs_raw_file']
env['qcow2_file'] = env['docker_qcow2_file']
else:
env['rootfs_raw_file'] = env['buildroot_rootfs_raw_file']
env['qcow2_file'] = env['buildroot_qcow2_file']
# Image.
if env['baremetal'] is None:
if env['emulator'] == 'gem5':
env['image'] = env['vmlinux']
env['disk_image'] = env['rootfs_raw_file']
else:
env['image'] = env['linux_image']
env['disk_image'] = env['qcow2_file']
else:
env['disk_image'] = env['gem5_fake_iso']
if env['baremetal'] == 'all':
path = env['baremetal']
else:
path = resolve_executable(
env['baremetal'],
env['baremetal_src_dir'],
env['baremetal_build_dir'],
env['baremetal_build_ext'],
)
source_path_noext = os.path.splitext(join(
env['baremetal_src_dir'],
os.path.relpath(path, env['baremetal_build_dir'])
))[0]
for ext in [c_ext, asm_ext]:
source_path = source_path_noext + ext
if os.path.exists(source_path):
env['source_path'] = source_path
break
env['image'] = path
self.env = env
def get_elf_entry(self, elf_file_path):
readelf_header = subprocess.check_output([
self.get_toolchain_tool('readelf'),
'-h',
elf_file_path
])
for line in readelf_header.decode().split('\n'):
split = line.split()
if line.startswith(' Entry point address:'):
addr = line.split()[-1]
break
return int(addr, 0)
def get_toolchain_prefix(self, tool, allowed_toolchains=None):
buildroot_full_prefix = os.path.join(self.env['host_bin_dir'], self.env['buildroot_toolchain_prefix'])
buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
crosstool_ng_full_prefix = os.path.join(self.env['crosstool_ng_bin_dir'], self.env['crosstool_ng_toolchain_prefix'])
crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
host_tool = '{}-{}'.format(self.env['ubuntu_toolchain_prefix'], tool)
host_path = shutil.which(host_tool)
if host_path is not None:
host_exists = True
host_full_prefix = host_path[:-(len(tool)+1)]
else:
host_exists = False
host_full_prefix = None
known_toolchains = {
'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix),
'buildroot': (buildroot_exists, buildroot_full_prefix),
'host': (host_exists, host_full_prefix),
}
if allowed_toolchains is None:
if self.env['baremetal'] is None:
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
else:
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
tried = []
for toolchain in allowed_toolchains:
exists, prefix = known_toolchains[toolchain]
tried.append('{}-{}'.format(prefix, tool))
if exists:
return prefix
raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))
def get_toolchain_tool(self, tool, allowed_toolchains=None):
return '{}-{}'.format(self.get_toolchain_prefix(tool, allowed_toolchains), tool)
def main(self, **kwargs): def main(self, **kwargs):
''' '''
Time the main of the derived class. Time the main of the derived class.
''' '''
if not kwargs['dry_run']: if not kwargs['dry_run']:
start_time = time.time() start_time = time.time()
kwargs.update(common.consts) kwargs.update(consts)
kwargs = set_kwargs(kwargs) self._init_env(kwargs)
self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run']) self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
self.timed_main(**kwargs) self.timed_main()
if not kwargs['dry_run']: if not kwargs['dry_run']:
end_time = time.time() end_time = time.time()
common.print_time(end_time - start_time) print_time(end_time - start_time)
def run_cmd(self, *args, **kwargs): def run_cmd(self, *args, **kwargs):
self.sh.run_cmd(*args, **kwargs) self.sh.run_cmd(*args, **kwargs)
def timed_main(self, **kwargs): def timed_main(self):
'''
Main action of the derived class.
'''
raise NotImplementedError() raise NotImplementedError()
class BuildCliFunction(LkmcCliFunction): class BuildCliFunction(LkmcCliFunction):
@@ -289,8 +592,8 @@ class BuildCliFunction(LkmcCliFunction):
* `--clean` to clean the build directory * `--clean` to clean the build directory
* `--nproc` to set he number of build threads * `--nproc` to set he number of build threads
''' '''
def __init__(self): def __init__(self, *args, **kwargs):
super().__init__() super().__init__(*args, **kwargs)
self.add_argument( self.add_argument(
'--clean', '--clean',
default=False, default=False,
@@ -304,33 +607,33 @@ class BuildCliFunction(LkmcCliFunction):
help='Number of processors to use for the build.', help='Number of processors to use for the build.',
) )
def clean(self, **kwargs): def clean(self):
build_dir = self.get_build_dir(kwargs) build_dir = self.get_build_dir()
if build_dir is not None: if build_dir is not None:
common.rmrf(build_dir) self.sh.rmrf(build_dir)
def build(self, **kwargs): def build(self):
''' '''
Do the actual main build work. Do the actual main build work.
''' '''
raise NotImplementedError() raise NotImplementedError()
def get_build_dir(self, **kwargs): def get_build_dir(self):
return None return None
def timed_main(self, **kwargs): def timed_main(self):
''' '''
Parse CLI, and to the build based on it. Parse CLI, and to the build based on it.
The actual build work is done by do_build in implementing classes. The actual build work is done by do_build in implementing classes.
''' '''
if kwargs['clean']: if self.env['clean']:
self.clean(kwargs) self.clean()
else: else:
self.build(**kwargs) self.build()
def assert_crosstool_ng_supports_arch(arch): def assert_crosstool_ng_supports_arch(arch):
if arch not in common.crosstool_ng_supported_archs: if arch not in kwargs['crosstool_ng_supported_archs']:
raise Exception('arch not yet supported: ' + arch) raise Exception('arch not yet supported: ' + arch)
def base64_encode(string): def base64_encode(string):
@@ -340,29 +643,16 @@ def gem_list_checkpoint_dirs():
''' '''
List checkpoint directory, oldest first. List checkpoint directory, oldest first.
''' '''
prefix_re = re.compile(common.gem5_cpt_prefix) prefix_re = re.compile(kwargs['gem5_cpt_prefix'])
files = list(filter(lambda x: os.path.isdir(os.path.join(common.m5out_dir, x)) and prefix_re.search(x), os.listdir(common.m5out_dir))) files = list(filter(lambda x: os.path.isdir(os.path.join(kwargs['m5out_dir'], x)) and prefix_re.search(x), os.listdir(kwargs['m5out_dir'])))
files.sort(key=lambda x: os.path.getmtime(os.path.join(common.m5out_dir, x))) files.sort(key=lambda x: os.path.getmtime(os.path.join(kwargs['m5out_dir'], x)))
return files return files
def get_elf_entry(elf_file_path):
readelf_header = subprocess.check_output([
common.get_toolchain_tool('readelf'),
'-h',
elf_file_path
])
for line in readelf_header.decode().split('\n'):
split = line.split()
if line.startswith(' Entry point address:'):
addr = line.split()[-1]
break
return int(addr, 0)
def get_stats(stat_re=None, stats_file=None): def get_stats(stat_re=None, stats_file=None):
if stat_re is None: if stat_re is None:
stat_re = '^system.cpu[0-9]*.numCycles$' stat_re = '^system.cpu[0-9]*.numCycles$'
if stats_file is None: if stats_file is None:
stats_file = common.stats_file stats_file = kwargs['stats_file']
stat_re = re.compile(stat_re) stat_re = re.compile(stat_re)
ret = [] ret = []
with open(stats_file, 'r') as statfile: with open(stats_file, 'r') as statfile:
@@ -373,40 +663,6 @@ def get_stats(stat_re=None, stats_file=None):
ret.append(cols[1]) ret.append(cols[1])
return ret return ret
def get_toolchain_prefix(tool, allowed_toolchains=None):
buildroot_full_prefix = os.path.join(common.host_bin_dir, common.buildroot_toolchain_prefix)
buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
crosstool_ng_full_prefix = os.path.join(common.crosstool_ng_bin_dir, common.crosstool_ng_toolchain_prefix)
crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
host_tool = '{}-{}'.format(common.ubuntu_toolchain_prefix, tool)
host_path = shutil.which(host_tool)
if host_path is not None:
host_exists = True
host_full_prefix = host_path[:-(len(tool)+1)]
else:
host_exists = False
host_full_prefix = None
known_toolchains = {
'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix),
'buildroot': (buildroot_exists, buildroot_full_prefix),
'host': (host_exists, host_full_prefix),
}
if allowed_toolchains is None:
if common['baremetal'] is None:
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
else:
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
tried = []
for toolchain in allowed_toolchains:
exists, prefix = known_toolchains[toolchain]
tried.append('{}-{}'.format(prefix, tool))
if exists:
return prefix
raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))
def get_toolchain_tool(tool, allowed_toolchains=None):
return '{}-{}'.format(common.get_toolchain_prefix(tool, allowed_toolchains), tool)
def github_make_request( def github_make_request(
authenticate=False, authenticate=False,
data=None, data=None,
@@ -441,18 +697,18 @@ def log_error(msg):
print('error: {}'.format(msg), file=sys.stderr) print('error: {}'.format(msg), file=sys.stderr)
def make_build_dirs(): def make_build_dirs():
os.makedirs(common.buildroot_build_build_dir, exist_ok=True) os.makedirs(kwargs['buildroot_build_build_dir'], exist_ok=True)
os.makedirs(common.gem5_build_dir, exist_ok=True) os.makedirs(kwargs['gem5_build_dir'], exist_ok=True)
os.makedirs(common.out_rootfs_overlay_dir, exist_ok=True) os.makedirs(kwargs['out_rootfs_overlay_dir'], exist_ok=True)
def make_run_dirs(): def make_run_dirs():
''' '''
Make directories required for the run. Make directories required for the run.
The user could nuke those anytime between runs to try and clean things up. The user could nuke those anytime between runs to try and clean things up.
''' '''
os.makedirs(common.gem5_run_dir, exist_ok=True) os.makedirs(kwargs['gem5_run_dir'], exist_ok=True)
os.makedirs(common.p9_dir, exist_ok=True) os.makedirs(kwargs['p9_dir'], exist_ok=True)
os.makedirs(common.qemu_run_dir, exist_ok=True) os.makedirs(kwargs['qemu_run_dir'], exist_ok=True)
def need_rebuild(srcs, dst): def need_rebuild(srcs, dst):
if not os.path.exists(dst): if not os.path.exists(dst):
@@ -468,17 +724,17 @@ def print_time(ellapsed_seconds):
print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))
def raw_to_qcow2(prebuilt=False, reverse=False): def raw_to_qcow2(prebuilt=False, reverse=False):
if prebuilt or not os.path.exists(common.qemu_img_executable): if prebuilt or not os.path.exists(kwargs['qemu_img_executable']):
disable_trace = [] disable_trace = []
qemu_img_executable = common.qemu_img_basename qemu_img_executable = kwargs['qemu_img_basename']
else: else:
# Prevent qemu-img from generating trace files like QEMU. Disgusting. # Prevent qemu-img from generating trace files like QEMU. Disgusting.
disable_trace = ['-T', 'pr_manager_run,file=/dev/null', LF,] disable_trace = ['-T', 'pr_manager_run,file=/dev/null', LF,]
qemu_img_executable = common.qemu_img_executable qemu_img_executable = kwargs['qemu_img_executable']
infmt = 'raw' infmt = 'raw'
outfmt = 'qcow2' outfmt = 'qcow2'
infile = common.rootfs_raw_file infile = kwargs['rootfs_raw_file']
outfile = common.qcow2_file outfile = kwargs['qcow2_file']
if reverse: if reverse:
tmp = infmt tmp = infmt
infmt = outfmt infmt = outfmt
@@ -525,263 +781,9 @@ def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext):
raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths)) raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths))
def resolve_userland(path): def resolve_userland(path):
return common.resolve_executable( return resolve_executable(
path, path,
common.userland_src_dir, kwargs['userland_src_dir'],
common.userland_build_dir, kwargs['userland_build_dir'],
common.userland_build_ext, kwargs['userland_build_ext'],
) )
def set_kwargs(kwargs):
'''
Update the kwargs from the command line with derived arguments.
'''
def join(*paths):
return os.path.join(*paths)
kwargs = collections.defaultdict(lambda: None, **kwargs)
if kwargs['qemu'] or not kwargs['gem5']:
kwargs['emulator'] = 'qemu'
else:
kwargs['emulator'] = 'gem5'
if kwargs['arch'] in kwargs['arch_short_to_long_dict']:
kwargs['arch'] = kwargs['arch_short_to_long_dict'][kwargs['arch']]
if kwargs['userland_build_id'] is None:
kwargs['userland_build_id'] = kwargs['default_build_id']
kwargs['userland_build_id_given'] = False
else:
kwargs['userland_build_id_given'] = True
if kwargs['gem5_worktree'] is not None and kwargs['gem5_build_id'] is None:
kwargs['gem5_build_id'] = kwargs['gem5_worktree']
kwargs['is_arm'] = False
if kwargs['arch'] == 'arm':
kwargs['armv'] = 7
kwargs['gem5_arch'] = 'ARM'
kwargs['mcpu'] = 'cortex-a15'
kwargs['buildroot_toolchain_prefix'] = 'arm-buildroot-linux-uclibcgnueabihf'
kwargs['crosstool_ng_toolchain_prefix'] = 'arm-unknown-eabi'
kwargs['ubuntu_toolchain_prefix'] = 'arm-linux-gnueabihf'
if kwargs['emulator'] == 'gem5':
if kwargs['machine'] is None:
kwargs['machine'] = 'VExpress_GEM5_V1'
else:
if kwargs['machine'] is None:
kwargs['machine'] = 'virt'
kwargs['is_arm'] = True
elif kwargs['arch'] == 'aarch64':
kwargs['armv'] = 8
kwargs['gem5_arch'] = 'ARM'
kwargs['mcpu'] = 'cortex-a57'
kwargs['buildroot_toolchain_prefix'] = 'aarch64-buildroot-linux-uclibc'
kwargs['crosstool_ng_toolchain_prefix'] = 'aarch64-unknown-elf'
kwargs['ubuntu_toolchain_prefix'] = 'aarch64-linux-gnu'
if kwargs['emulator'] == 'gem5':
if kwargs['machine'] is None:
kwargs['machine'] = 'VExpress_GEM5_V1'
else:
if kwargs['machine'] is None:
kwargs['machine'] = 'virt'
kwargs['is_arm'] = True
elif kwargs['arch'] == 'x86_64':
kwargs['crosstool_ng_toolchain_prefix'] = 'x86_64-unknown-elf'
kwargs['gem5_arch'] = 'X86'
kwargs['buildroot_toolchain_prefix'] = 'x86_64-buildroot-linux-uclibc'
kwargs['ubuntu_toolchain_prefix'] = 'x86_64-linux-gnu'
if kwargs['emulator'] == 'gem5':
if kwargs['machine'] is None:
kwargs['machine'] = 'TODO'
else:
if kwargs['machine'] is None:
kwargs['machine'] = 'pc'
# Buildroot
kwargs['buildroot_build_dir'] = join(kwargs['buildroot_out_dir'], 'build', kwargs['buildroot_build_id'], kwargs['arch'])
kwargs['buildroot_download_dir'] = join(kwargs['buildroot_out_dir'], 'download')
kwargs['buildroot_config_file'] = join(kwargs['buildroot_build_dir'], '.config')
kwargs['buildroot_build_build_dir'] = join(kwargs['buildroot_build_dir'], 'build')
kwargs['buildroot_linux_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom')
kwargs['buildroot_vmlinux'] = join(kwargs['buildroot_linux_build_dir'], "vmlinux")
kwargs['host_dir'] = join(kwargs['buildroot_build_dir'], 'host')
kwargs['host_bin_dir'] = join(kwargs['host_dir'], 'usr', 'bin')
kwargs['buildroot_pkg_config'] = join(kwargs['host_bin_dir'], 'pkg-config')
kwargs['buildroot_images_dir'] = join(kwargs['buildroot_build_dir'], 'images')
kwargs['buildroot_rootfs_raw_file'] = join(kwargs['buildroot_images_dir'], 'rootfs.ext2')
kwargs['buildroot_qcow2_file'] = kwargs['buildroot_rootfs_raw_file'] + '.qcow2'
kwargs['staging_dir'] = join(kwargs['out_dir'], 'staging', kwargs['arch'])
kwargs['buildroot_staging_dir'] = join(kwargs['buildroot_build_dir'], 'staging')
kwargs['target_dir'] = join(kwargs['buildroot_build_dir'], 'target')
kwargs['linux_buildroot_build_dir'] = join(kwargs['buildroot_build_build_dir'], 'linux-custom')
# QEMU
kwargs['qemu_build_dir'] = join(kwargs['out_dir'], 'qemu', kwargs['qemu_build_id'])
kwargs['qemu_executable_basename'] = 'qemu-system-{}'.format(kwargs['arch'])
kwargs['qemu_executable'] = join(kwargs['qemu_build_dir'], '{}-softmmu'.format(kwargs['arch']), kwargs['qemu_executable_basename'])
kwargs['qemu_img_basename'] = 'qemu-img'
kwargs['qemu_img_executable'] = join(kwargs['qemu_build_dir'], kwargs['qemu_img_basename'])
# gem5
if kwargs['gem5_build_dir'] is None:
kwargs['gem5_build_dir'] = join(kwargs['gem5_out_dir'], kwargs['gem5_build_id'], kwargs['gem5_build_type'])
kwargs['gem5_fake_iso'] = join(kwargs['gem5_out_dir'], 'fake.iso')
kwargs['gem5_m5term'] = join(kwargs['gem5_build_dir'], 'm5term')
kwargs['gem5_build_build_dir'] = join(kwargs['gem5_build_dir'], 'build')
kwargs['gem5_executable'] = join(kwargs['gem5_build_build_dir'], kwargs['gem5_arch'], 'gem5.{}'.format(kwargs['gem5_build_type']))
kwargs['gem5_system_dir'] = join(kwargs['gem5_build_dir'], 'system')
# gem5 source
if kwargs['gem5_source_dir'] is not None:
assert os.path.exists(kwargs['gem5_source_dir'])
else:
if kwargs['gem5_worktree'] is not None:
kwargs['gem5_source_dir'] = join(kwargs['gem5_non_default_src_root_dir'], kwargs['gem5_worktree'])
else:
kwargs['gem5_source_dir'] = kwargs['gem5_default_src_dir']
kwargs['gem5_m5_source_dir'] = join(kwargs['gem5_source_dir'], 'util', 'm5')
kwargs['gem5_config_dir'] = join(kwargs['gem5_source_dir'], 'configs')
kwargs['gem5_se_file'] = join(kwargs['gem5_config_dir'], 'example', 'se.py')
kwargs['gem5_fs_file'] = join(kwargs['gem5_config_dir'], 'example', 'fs.py')
# crosstool-ng
kwargs['crosstool_ng_buildid_dir'] = join(kwargs['crosstool_ng_out_dir'], 'build', kwargs['crosstool_ng_build_id'])
kwargs['crosstool_ng_install_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'install', kwargs['arch'])
kwargs['crosstool_ng_bin_dir'] = join(kwargs['crosstool_ng_install_dir'], 'bin')
kwargs['crosstool_ng_util_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'util')
kwargs['crosstool_ng_config'] = join(kwargs['crosstool_ng_util_dir'], '.config')
kwargs['crosstool_ng_defconfig'] = join(kwargs['crosstool_ng_util_dir'], 'defconfig')
kwargs['crosstool_ng_executable'] = join(kwargs['crosstool_ng_util_dir'], 'ct-ng')
kwargs['crosstool_ng_build_dir'] = join(kwargs['crosstool_ng_buildid_dir'], 'build')
kwargs['crosstool_ng_download_dir'] = join(kwargs['crosstool_ng_out_dir'], 'download')
# run
kwargs['gem5_run_dir'] = join(kwargs['run_dir_base'], 'gem5', kwargs['arch'], str(kwargs['run_id']))
kwargs['m5out_dir'] = join(kwargs['gem5_run_dir'], 'm5out')
kwargs['stats_file'] = join(kwargs['m5out_dir'], 'stats.txt')
kwargs['gem5_trace_txt_file'] = join(kwargs['m5out_dir'], 'trace.txt')
kwargs['gem5_guest_terminal_file'] = join(kwargs['m5out_dir'], 'system.terminal')
kwargs['gem5_readfile'] = join(kwargs['gem5_run_dir'], 'readfile')
kwargs['gem5_termout_file'] = join(kwargs['gem5_run_dir'], 'termout.txt')
kwargs['qemu_run_dir'] = join(kwargs['run_dir_base'], 'qemu', kwargs['arch'], str(kwargs['run_id']))
kwargs['qemu_termout_file'] = join(kwargs['qemu_run_dir'], 'termout.txt')
kwargs['qemu_trace_basename'] = 'trace.bin'
kwargs['qemu_trace_file'] = join(kwargs['qemu_run_dir'], 'trace.bin')
kwargs['qemu_trace_txt_file'] = join(kwargs['qemu_run_dir'], 'trace.txt')
kwargs['qemu_rrfile'] = join(kwargs['qemu_run_dir'], 'rrfile')
kwargs['gem5_out_dir'] = join(kwargs['out_dir'], 'gem5')
# Ports
if kwargs['port_offset'] is None:
try:
kwargs['port_offset'] = int(kwargs['run_id'])
except ValueError:
kwargs['port_offset'] = 0
if kwargs['emulator'] == 'gem5':
kwargs['gem5_telnet_port'] = 3456 + kwargs['port_offset']
kwargs['gdb_port'] = 7000 + kwargs['port_offset']
else:
kwargs['qemu_base_port'] = 45454 + 10 * kwargs['port_offset']
kwargs['qemu_monitor_port'] = kwargs['qemu_base_port'] + 0
kwargs['qemu_hostfwd_generic_port'] = kwargs['qemu_base_port'] + 1
kwargs['qemu_hostfwd_ssh_port'] = kwargs['qemu_base_port'] + 2
kwargs['qemu_gdb_port'] = kwargs['qemu_base_port'] + 3
kwargs['extra_serial_port'] = kwargs['qemu_base_port'] + 4
kwargs['gdb_port'] = kwargs['qemu_gdb_port']
kwargs['qemu_background_serial_file'] = join(kwargs['qemu_run_dir'], 'background.log')
# gem5 QEMU polymorphism.
if kwargs['emulator'] == 'gem5':
kwargs['executable'] = kwargs['gem5_executable']
kwargs['run_dir'] = kwargs['gem5_run_dir']
kwargs['termout_file'] = kwargs['gem5_termout_file']
kwargs['guest_terminal_file'] = kwargs['gem5_guest_terminal_file']
kwargs['trace_txt_file'] = kwargs['gem5_trace_txt_file']
else:
kwargs['executable'] = kwargs['qemu_executable']
kwargs['run_dir'] = kwargs['qemu_run_dir']
kwargs['termout_file'] = kwargs['qemu_termout_file']
kwargs['trace_txt_file'] = kwargs['qemu_trace_txt_file']
kwargs['run_cmd_file'] = join(kwargs['run_dir'], 'run.sh')
# Linux kernl.
if 'linux_build_id' in kwargs:
kwargs['linux_build_dir'] = join(kwargs['out_dir'], 'linux', kwargs['linux_build_id'], kwargs['arch'])
kwargs['lkmc_vmlinux'] = join(kwargs['linux_build_dir'], "vmlinux")
if kwargs['arch'] == 'arm':
kwargs['linux_arch'] = 'arm'
kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'zImage')
elif kwargs['arch'] == 'aarch64':
kwargs['linux_arch'] = 'arm64'
kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'Image')
elif kwargs['arch'] == 'x86_64':
kwargs['linux_arch'] = 'x86'
kwargs['linux_image_prefix'] = join('arch', kwargs['linux_arch'], 'boot', 'bzImage')
kwargs['lkmc_linux_image'] = join(kwargs['linux_build_dir'], kwargs['linux_image_prefix'])
kwargs['buildroot_linux_image'] = join(kwargs['buildroot_linux_build_dir'], kwargs['linux_image_prefix'])
if kwargs['buildroot_linux']:
kwargs['vmlinux'] = kwargs['buildroot_vmlinux']
kwargs['linux_image'] = kwargs['buildroot_linux_image']
else:
kwargs['vmlinux'] = kwargs['lkmc_vmlinux']
kwargs['linux_image'] = kwargs['lkmc_linux_image']
# Kernel modules.
kwargs['kernel_modules_build_dir'] = join(kwargs['kernel_modules_build_base_dir'], kwargs['arch'])
kwargs['kernel_modules_build_subdir'] = join(kwargs['kernel_modules_build_dir'], kwargs['kernel_modules_subdir'])
kwargs['kernel_modules_build_host_dir'] = join(kwargs['kernel_modules_build_base_dir'], 'host')
kwargs['kernel_modules_build_host_subdir'] = join(kwargs['kernel_modules_build_host_dir'], kwargs['kernel_modules_subdir'])
kwargs['userland_build_dir'] = join(kwargs['out_dir'], 'userland', kwargs['userland_build_id'], kwargs['arch'])
kwargs['out_rootfs_overlay_dir'] = join(kwargs['out_dir'], 'rootfs_overlay', kwargs['arch'])
kwargs['out_rootfs_overlay_bin_dir'] = join(kwargs['out_rootfs_overlay_dir'], 'bin')
# Baremetal.
kwargs['baremetal_src_dir'] = join(kwargs['root_dir'], 'baremetal')
kwargs['baremetal_src_lib_dir'] = join(kwargs['baremetal_src_dir'], kwargs['baremetal_lib_basename'])
if kwargs['emulator'] == 'gem5':
kwargs['simulator_name'] = 'gem5'
else:
kwargs['simulator_name'] = 'qemu'
kwargs['baremetal_build_dir'] = join(kwargs['out_dir'], 'baremetal', kwargs['arch'], kwargs['simulator_name'], kwargs['machine'])
kwargs['baremetal_build_lib_dir'] = join(kwargs['baremetal_build_dir'], kwargs['baremetal_lib_basename'])
kwargs['baremetal_build_ext'] = '.elf'
# Docker
kwargs['docker_build_dir'] = join(kwargs['out_dir'], 'docker', kwargs['arch'])
kwargs['docker_tar_dir'] = join(kwargs['docker_build_dir'], 'export')
kwargs['docker_tar_file'] = join(kwargs['docker_build_dir'], 'export.tar')
kwargs['docker_rootfs_raw_file'] = join(kwargs['docker_build_dir'], 'export.ext2')
kwargs['docker_qcow2_file'] = join(kwargs['docker_rootfs_raw_file'] + '.qcow2')
if kwargs['docker']:
kwargs['rootfs_raw_file'] = kwargs['docker_rootfs_raw_file']
kwargs['qcow2_file'] = kwargs['docker_qcow2_file']
else:
kwargs['rootfs_raw_file'] = kwargs['buildroot_rootfs_raw_file']
kwargs['qcow2_file'] = kwargs['buildroot_qcow2_file']
# Image.
if kwargs['baremetal'] is None:
if kwargs['emulator'] == 'gem5':
kwargs['image'] = kwargs['vmlinux']
kwargs['disk_image'] = kwargs['rootfs_raw_file']
else:
kwargs['image'] = kwargs['linux_image']
kwargs['disk_image'] = kwargs['qcow2_file']
else:
kwargs['disk_image'] = kwargs['gem5_fake_iso']
if kwargs['baremetal'] == 'all':
path = kwargs['baremetal']
else:
path = kwargs.resolve_executable(
kwargs['baremetal'],
kwargs['baremetal_src_dir'],
kwargs['baremetal_build_dir'],
kwargs['baremetal_build_ext'],
)
source_path_noext = os.path.splitext(join(
kwargs['baremetal_src_dir'],
os.path.relpath(path, kwargs['baremetal_build_dir'])
))[0]
for ext in [c_ext, asm_ext]:
source_path = source_path_noext + ext
if os.path.exists(source_path):
kwargs['source_path'] = source_path
break
kwargs['image'] = path
return dict(kwargs)

4
copy-overlay
View File

@@ -9,8 +9,8 @@ import common
class CopyOverlayComponent(common.Component): class CopyOverlayComponent(common.Component):
def do_build(self, args): def do_build(self, args):
distutils.dir_util.copy_tree( distutils.dir_util.copy_tree(
common.rootfs_overlay_dir, kwargs['rootfs_overlay_dir'],
common.out_rootfs_overlay_dir, kwargs['out_rootfs_overlay_dir'],
update=1, update=1,
) )

4
gem5-shell
View File

@@ -10,7 +10,7 @@ parser = common.get_argparse(
) )
args = common.setup(parser) args = common.setup(parser)
sys.exit(self.sh.run_cmd([ sys.exit(self.sh.run_cmd([
common.gem5_m5term, LF, kwargs['gem5_m5term'], LF,
'localhost', LF, 'localhost', LF,
str(common.gem5_telnet_port), LF, str(kwargs['gem5_telnet_port']), LF,
])) ]))

4
getvar
View File

@@ -5,12 +5,12 @@ import types
import common import common
parser = common.get_argparse(argparse_args={ parser = common.get_argparse(argparse_args={
'description': '''Print the value of a common.py variable. 'description': '''Print the value of a kwargs['py'] variable.
This is useful to: This is useful to:
* give dry commands on the README that don't change when we refactor directory structure * give dry commands on the README that don't change when we refactor directory structure
* create simple bash scripts that call use common.py variables * create simple bash scripts that call use kwargs['py'] variables
For example, to get the Buildroot output directory for an ARM build, use: For example, to get the Buildroot output directory for an ARM build, use:

2
qemu-monitor
View File

@@ -27,7 +27,7 @@ def write_and_read(tn, cmd, prompt):
tn.write(cmd.encode('utf-8')) tn.write(cmd.encode('utf-8'))
return '\n'.join(tn.read_until(prompt).decode('utf-8').splitlines()[1:])[:-len(prompt)] return '\n'.join(tn.read_until(prompt).decode('utf-8').splitlines()[1:])[:-len(prompt)]
with telnetlib.Telnet('localhost', common.qemu_monitor_port) as tn: with telnetlib.Telnet('localhost', kwargs['qemu_monitor_port']) as tn:
# Couldn't disable server echo, so just removing the write for now. # Couldn't disable server echo, so just removing the write for now.
# https://stackoverflow.com/questions/12421799/how-to-disable-telnet-echo-in-python-telnetlib # https://stackoverflow.com/questions/12421799/how-to-disable-telnet-echo-in-python-telnetlib
# sock = tn.get_socket() # sock = tn.get_socket()

10
qemu-trace2txt
View File

@@ -9,12 +9,12 @@ import common
def main(): def main():
return self.sh.run_cmd( return self.sh.run_cmd(
[ [
os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), LF, os.path.join(kwargs['qemu_src_dir'], 'scripts/simpletrace.py'), LF,
os.path.join(common.qemu_build_dir, 'trace-events-all'), LF, os.path.join(kwargs['qemu_build_dir'], 'trace-events-all'), LF,
os.path.join(common.qemu_trace_file), LF, os.path.join(kwargs['qemu_trace_file']), LF,
], ],
cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'), cmd_file=os.path.join(kwargs['run_dir'], 'qemu-trace2txt'),
out_file=common.qemu_trace_txt_file, out_file=kwargs['qemu_trace_txt_file'],
show_stdout=False, show_stdout=False,
) )

12
release
View File

@@ -10,21 +10,21 @@ import subprocess
import time import time
import common import common
release_zip = imp.load_source('release_zip', os.path.join(common.root_dir, 'release-zip')) release_zip = imp.load_source('release_zip', os.path.join(kwargs['root_dir'], 'release-zip'))
release_upload = imp.load_source('release_upload', os.path.join(common.root_dir, 'release-upload')) release_upload = imp.load_source('release_upload', os.path.join(kwargs['root_dir'], 'release-upload'))
start_time = time.time() start_time = time.time()
# TODO factor those out so we don't redo the same thing multiple times. # TODO factor those out so we don't redo the same thing multiple times.
# subprocess.check_call([os.path.join(common.root_dir, 'test')]) # subprocess.check_call([os.path.join(kwargs['root_dir'], 'test')])
# subprocess.check_call([os.path.join(common.root_dir, ''bench-all', '-A', '-u']) # subprocess.check_call([os.path.join(kwargs['root_dir'], ''bench-all', '-A', '-u'])
# A clean release requires a full rebuild unless we hack it :-( # A clean release requires a full rebuild unless we hack it :-(
# We can't just use our current build as it contains packages we've # We can't just use our current build as it contains packages we've
# installed in random experiments. And with EXT2: we can't easily # installed in random experiments. And with EXT2: we can't easily
# know what the smallest root filesystem size is and use it either... # know what the smallest root filesystem size is and use it either...
# https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot # https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot
subprocess.check_call([os.path.join(common.root_dir, 'configure'), '--all']) subprocess.check_call([os.path.join(kwargs['root_dir'], 'configure'), '--all'])
subprocess.check_call([os.path.join(common.root_dir, 'build'), '--all-archs', 'release']) subprocess.check_call([os.path.join(kwargs['root_dir'], 'build'), '--all-archs', 'release'])
release_zip.main() release_zip.main()
subprocess.check_call(['git', 'push']) subprocess.check_call(['git', 'push'])
release_upload.main() release_upload.main()

6
release-upload
View File

@@ -18,9 +18,9 @@ import urllib.error
import common import common
def main(): def main():
repo = common.github_repo_id repo = kwargs['github_repo_id']
tag = 'sha-{}'.format(common.sha) tag = 'sha-{}'.format(kwargs['sha'])
upload_path = common.release_zip_file upload_path = kwargs['release_zip_file']
# Check the release already exists. # Check the release already exists.
try: try:

14
release-zip
View File

@@ -11,14 +11,14 @@ import zipfile
import common import common
def main(): def main():
os.makedirs(common.release_dir, exist_ok=True) os.makedirs(kwargs['release_dir'], exist_ok=True)
if os.path.exists(common.release_zip_file): if os.path.exists(kwargs['release_zip_file']):
os.unlink(common.release_zip_file) os.unlink(kwargs['release_zip_file'])
zipf = zipfile.ZipFile(common.release_zip_file, 'w', zipfile.ZIP_DEFLATED) zipf = zipfile.ZipFile(kwargs['release_zip_file'], 'w', zipfile.ZIP_DEFLATED)
for arch in common.all_archs: for arch in kwargs['all_archs']:
common.setup(common.get_argparse(default_args={'arch': arch})) common.setup(common.get_argparse(default_args={'arch': arch}))
zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir)) zipf.write(kwargs['qcow2_file'], arcname=os.path.relpath(kwargs['qcow2_file'], kwargs['root_dir']))
zipf.write(common.linux_image, arcname=os.path.relpath(common.linux_image, common.root_dir)) zipf.write(kwargs['linux_image'], arcname=os.path.relpath(kwargs['linux_image'], kwargs['root_dir']))
zipf.close() zipf.close()
if __name__ == '__main__': if __name__ == '__main__':

144
run
View File

@@ -85,7 +85,7 @@ def main(args, extra_args=None):
console_count = 0 console_count = 0
if kwargs['arch'] == 'x86_64': if kwargs['arch'] == 'x86_64':
console_type = 'ttyS' console_type = 'ttyS'
elif common.is_arm: elif kwargs['is_arm']:
console_type = 'ttyAMA' console_type = 'ttyAMA'
console = '{}{}'.format(console_type, console_count) console = '{}{}'.format(console_type, console_count)
console_count += 1 console_count += 1
@@ -118,36 +118,36 @@ def main(args, extra_args=None):
def raise_rootfs_not_found(): def raise_rootfs_not_found():
if not kwargs['dry_run']: if not kwargs['dry_run']:
raise Exception('Root filesystem not found. Did you build it?\n' \ raise Exception('Root filesystem not found. Did you build it?\n' \
'Tried to use: ' + common.disk_image) 'Tried to use: ' + kwargs['disk_image'])
def raise_image_not_found(): def raise_image_not_found():
if not kwargs['dry_run']: if not kwargs['dry_run']:
raise Exception('Executable image not found. Did you build it?\n' \ raise Exception('Executable image not found. Did you build it?\n' \
'Tried to use: ' + common.image) 'Tried to use: ' + kwargs['image'])
if common.image is None: if kwargs['image'] is None:
raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
cmd = debug_vm.copy() cmd = debug_vm.copy()
if common.emulator == 'gem5': if kwargs['emulator'] == 'gem5':
if common.baremetal is None: if kwargs['baremetal'] is None:
if not os.path.exists(common.rootfs_raw_file): if not os.path.exists(kwargs['rootfs_raw_file']):
if not os.path.exists(common.qcow2_file): if not os.path.exists(kwargs['qcow2_file']):
raise_rootfs_not_found() raise_rootfs_not_found()
common.raw_to_qcow2(prebuilt=kwargs['prebuilt'], reverse=True) common.raw_to_qcow2(prebuilt=kwargs['prebuilt'], reverse=True)
else: else:
if not os.path.exists(common.gem5_fake_iso): if not os.path.exists(kwargs['gem5_fake_iso']):
os.makedirs(os.path.dirname(common.gem5_fake_iso), exist_ok=True) os.makedirs(os.path.dirname(kwargs['gem5_fake_iso']), exist_ok=True)
common.write_string_to_file(common.gem5_fake_iso, 'a' * 512) common.write_string_to_file(kwargs['gem5_fake_iso'], 'a' * 512)
if not os.path.exists(common.image): if not os.path.exists(kwargs['image']):
# This is to run gem5 from a prebuilt download. # This is to run gem5 from a prebuilt download.
if (not common.baremetal is None) or (not os.path.exists(common.linux_image)): if (not kwargs['baremetal'] is None) or (not os.path.exists(kwargs['linux_image'])):
raise_image_not_found() raise_image_not_found()
self.sh.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) self.sh.run_cmd([os.path.join(kwargs['extract_vmlinux'], kwargs['linux_image'])])
os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True) os.makedirs(os.path.dirname(kwargs['gem5_readfile']), exist_ok=True)
common.write_string_to_file(common.gem5_readfile, kwargs['gem5_readfile']) common.write_string_to_file(kwargs['gem5_readfile'], kwargs['gem5_readfile'])
memory = '{}B'.format(kwargs['memory']) memory = '{}B'.format(kwargs['memory'])
gem5_exe_args = common.shlex_split(kwargs['gem5_exe_args']) gem5_exe_args = common.shlex_split(kwargs['gem5_exe_args'])
if do_trace: if do_trace:
gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), LF]) gem5_exe_args.extend(['--debug-flags={}'.format(trace_type), LF])
extra_env['M5_PATH'] = common.gem5_system_dir extra_env['M5_PATH'] = kwargs['gem5_system_dir']
# https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071 # https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071
extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true' extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true'
if kwargs['trace_stdout']: if kwargs['trace_stdout']:
@@ -156,16 +156,16 @@ def main(args, extra_args=None):
debug_file = 'trace.txt' debug_file = 'trace.txt'
cmd.extend( cmd.extend(
[ [
common.executable, LF, kwargs['executable'], LF,
'--debug-file', debug_file, LF, '--debug-file', debug_file, LF,
'--listener-mode', 'on', LF, '--listener-mode', 'on', LF,
'--outdir', common.m5out_dir, LF, '--outdir', kwargs['m5out_dir'], LF,
] + ] +
gem5_exe_args gem5_exe_args
) )
if kwargs['userland'] is not None: if kwargs['userland'] is not None:
cmd.extend([ cmd.extend([
common.gem5_se_file, LF, kwargs['gem5_se_file'], LF,
'-c', common.resolve_userland(kwargs['userland']), LF, '-c', common.resolve_userland(kwargs['userland']), LF,
]) ])
else: else:
@@ -176,28 +176,28 @@ def main(args, extra_args=None):
cpt_dir = cpt_dirs[-kwargs['gem5_restore']] cpt_dir = cpt_dirs[-kwargs['gem5_restore']]
extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)])
cmd.extend([ cmd.extend([
common.gem5_fs_file, LF, kwargs['gem5_fs_file'], LF,
'--disk-image', common.disk_image, LF, '--disk-image', kwargs['disk_image'], LF,
'--kernel', common.image, LF, '--kernel', kwargs['image'], LF,
'--mem-size', memory, LF, '--mem-size', memory, LF,
'--num-cpus', str(kwargs['cpus']), LF, '--num-cpus', str(kwargs['cpus']), LF,
'--script', common.gem5_readfile, LF, '--script', kwargs['gem5_readfile'], LF,
]) ])
if kwargs['arch'] == 'x86_64': if kwargs['arch'] == 'x86_64':
if kwargs['kvm']: if kwargs['kvm']:
cmd.extend(['--cpu-type', 'X86KvmCPU', LF]) cmd.extend(['--cpu-type', 'X86KvmCPU', LF])
cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), LF]) cmd.extend(['--command-line', 'earlyprintk={} lpj=7999923 root=/dev/sda {}'.format(console, kernel_cli), LF])
elif common.is_arm: elif kwargs['is_arm']:
if kwargs['kvm']: if kwargs['kvm']:
cmd.extend(['--cpu-type', 'ArmV8KvmCPU', LF]) cmd.extend(['--cpu-type', 'ArmV8KvmCPU', LF])
cmd.extend([ cmd.extend([
# TODO why is it mandatory to pass mem= here? Not true for QEMU. # TODO why is it mandatory to pass mem= here? Not true for QEMU.
# Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value?
'--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), LF, '--command-line', 'earlyprintk=pl011,0x1c090000 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), LF,
'--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, kwargs['cpus'])), LF, '--dtb-filename', os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(kwargs['armv'], kwargs['cpus'])), LF,
'--machine-type', common.machine, LF, '--machine-type', kwargs['machine'], LF,
]) ])
if common.baremetal is None: if kwargs['baremetal'] is None:
cmd.extend([ cmd.extend([
'--param', 'system.panic_on_panic = True', LF]) '--param', 'system.panic_on_panic = True', LF])
else: else:
@@ -215,14 +215,14 @@ def main(args, extra_args=None):
cpu_type = 'atomic' cpu_type = 'atomic'
if kwargs['gem5_restore'] is not None: if kwargs['gem5_restore'] is not None:
cpt_dir = common.gem_list_checkpoint_dirs()[-kwargs['gem5_restore']] cpt_dir = common.gem_list_checkpoint_dirs()[-kwargs['gem5_restore']]
extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)]) extra_emulator_args.extend(['--restore-from', os.path.join(kwargs['m5out_dir'], cpt_dir)])
cmd.extend([ cmd.extend([
os.path.join(common.gem5_source_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF, os.path.join(kwargs['gem5_source_dir'], 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF,
'--big-cpus', '2', LF, '--big-cpus', '2', LF,
'--cpu-type', cpu_type, LF, '--cpu-type', cpu_type, LF,
'--disk', common.disk_image, LF, '--disk', kwargs['disk_image'], LF,
'--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), LF, '--dtb', os.path.join(kwargs['gem5_system_dir'], 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), LF,
'--kernel', common.image, LF, '--kernel', kwargs['image'], LF,
'--little-cpus', '2', LF, '--little-cpus', '2', LF,
]) ])
if kwargs['wait_gdb']: if kwargs['wait_gdb']:
@@ -230,17 +230,17 @@ def main(args, extra_args=None):
cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', LF]) cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True', LF])
else: else:
qemu_user_and_system_options = [ qemu_user_and_system_options = [
'-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), LF, '-trace', 'enable={},file={}'.format(trace_type, kwargs['qemu_trace_file']), LF,
] ]
if kwargs['userland'] is not None: if kwargs['userland'] is not None:
if kwargs['wait_gdb']: if kwargs['wait_gdb']:
debug_args = ['-g', str(common.gdb_port), LF] debug_args = ['-g', str(kwargs['gdb_port']), LF]
else: else:
debug_args = [] debug_args = []
cmd.extend( cmd.extend(
[ [
os.path.join(common.qemu_build_dir, '{}-linux-user'.format(kwargs['arch']), 'qemu-{}'.format(kwargs['arch'])), LF, os.path.join(kwargs['qemu_build_dir'], '{}-linux-user'.format(kwargs['arch']), 'qemu-{}'.format(kwargs['arch'])), LF,
'-L', common.target_dir, LF '-L', kwargs['target_dir'], LF
] + ] +
qemu_user_and_system_options + qemu_user_and_system_options +
common.shlex_split(kwargs['userland_before']) + common.shlex_split(kwargs['userland_before']) +
@@ -250,15 +250,15 @@ def main(args, extra_args=None):
] ]
) )
else: else:
if not os.path.exists(common.image): if not os.path.exists(kwargs['image']):
raise_image_not_found() raise_image_not_found()
extra_emulator_args.extend(extra_qemu_args) extra_emulator_args.extend(extra_qemu_args)
common.make_run_dirs() common.make_run_dirs()
if kwargs['prebuilt'] or not os.path.exists(common.qemu_executable): if kwargs['prebuilt'] or not os.path.exists(kwargs['qemu_executable']):
qemu_executable = common.qemu_executable_basename qemu_executable = kwargs['qemu_executable_basename']
qemu_executable_prebuilt = True qemu_executable_prebuilt = True
else: else:
qemu_executable = common.qemu_executable qemu_executable = kwargs['qemu_executable']
qemu_executable_prebuilt = False qemu_executable_prebuilt = False
qemu_executable = shutil.which(qemu_executable) qemu_executable = shutil.which(qemu_executable)
if qemu_executable is None: if qemu_executable is None:
@@ -268,17 +268,17 @@ def main(args, extra_args=None):
serial_monitor = [] serial_monitor = []
else: else:
if kwargs['background']: if kwargs['background']:
serial_monitor = ['-serial', 'file:{}'.format(common.qemu_background_serial_file), LF] serial_monitor = ['-serial', 'file:{}'.format(kwargs['qemu_background_serial_file']), LF]
else: else:
serial_monitor = ['-serial', 'mon:stdio', LF] serial_monitor = ['-serial', 'mon:stdio', LF]
if kwargs['kvm']: if kwargs['kvm']:
extra_emulator_args.extend(['-enable-kvm', LF]) extra_emulator_args.extend(['-enable-kvm', LF])
extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.extra_serial_port), LF]) extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(kwargs['extra_serial_port']), LF])
virtfs_data = [ virtfs_data = [
(common.p9_dir, 'host_data'), (kwargs['p9_dir'], 'host_data'),
(common.out_dir, 'host_out'), (kwargs['out_dir'], 'host_out'),
(common.out_rootfs_overlay_dir, 'host_out_rootfs_overlay'), (kwargs['out_rootfs_overlay_dir'], 'host_out_rootfs_overlay'),
(common.rootfs_overlay_dir, 'host_rootfs_overlay'), (kwargs['rootfs_overlay_dir'], 'host_rootfs_overlay'),
] ]
virtfs_cmd = [] virtfs_cmd = []
for virtfs_dir, virtfs_tag in virtfs_data: for virtfs_dir, virtfs_tag in virtfs_data:
@@ -293,11 +293,11 @@ def main(args, extra_args=None):
[ [
qemu_executable, LF, qemu_executable, LF,
'-device', 'rtl8139,netdev=net0', LF, '-device', 'rtl8139,netdev=net0', LF,
'-gdb', 'tcp::{}'.format(common.gdb_port), LF, '-gdb', 'tcp::{}'.format(kwargs['gdb_port']), LF,
'-kernel', common.image, LF, '-kernel', kwargs['image'], LF,
'-m', kwargs['memory'], LF, '-m', kwargs['memory'], LF,
'-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), LF, '-monitor', 'telnet::{},server,nowait'.format(kwargs['qemu_monitor_port']), LF,
'-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), LF, '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(kwargs['qemu_hostfwd_generic_port'], kwargs['qemu_hostfwd_generic_port'], kwargs['qemu_hostfwd_ssh_port']), LF,
'-no-reboot', LF, '-no-reboot', LF,
'-smp', str(kwargs['cpus']), LF, '-smp', str(kwargs['cpus']), LF,
] + ] +
@@ -308,7 +308,7 @@ def main(args, extra_args=None):
if not qemu_executable_prebuilt: if not qemu_executable_prebuilt:
cmd.extend(qemu_user_and_system_options) cmd.extend(qemu_user_and_system_options)
if kwargs['initrd']: if kwargs['initrd']:
extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')]) extra_emulator_args.extend(['-initrd', os.path.join(kwargs['buildroot_images_dir'], 'rootfs.cpio')])
rr = kwargs['record'] or kwargs['replay'] rr = kwargs['record'] or kwargs['replay']
if ramfs: if ramfs:
# TODO why is this needed, and why any string works. # TODO why is this needed, and why any string works.
@@ -324,14 +324,14 @@ def main(args, extra_args=None):
root = 'root=/dev/vda' root = 'root=/dev/vda'
rrid = '' rrid = ''
snapshot = ',snapshot' snapshot = ',snapshot'
if common.baremetal is None: if kwargs['baremetal'] is None:
if not os.path.exists(common.qcow2_file): if not os.path.exists(kwargs['qcow2_file']):
if not os.path.exists(common.rootfs_raw_file): if not os.path.exists(kwargs['rootfs_raw_file']):
raise_rootfs_not_found() raise_rootfs_not_found()
common.raw_to_qcow2(prebuilt=kwargs['prebuilt']) common.raw_to_qcow2(prebuilt=kwargs['prebuilt'])
extra_emulator_args.extend([ extra_emulator_args.extend([
'-drive', '-drive',
'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid), 'file={},format=qcow2,if={}{}{}'.format(kwargs['disk_image'], driveif, snapshot, rrid),
LF, LF,
]) ])
if rr: if rr:
@@ -342,7 +342,7 @@ def main(args, extra_args=None):
if rr: if rr:
extra_emulator_args.extend([ extra_emulator_args.extend([
'-object', 'filter-replay,id=replay,netdev=net0', '-object', 'filter-replay,id=replay,netdev=net0',
'-icount', 'shift=7,rr={},rrfile={}'.format('record' if kwargs['record'] else 'replay', common.qemu_rrfile), '-icount', 'shift=7,rr={},rrfile={}'.format('record' if kwargs['record'] else 'replay', kwargs['qemu_rrfile']),
]) ])
virtio_gpu_pci = [] virtio_gpu_pci = []
else: else:
@@ -350,10 +350,10 @@ def main(args, extra_args=None):
if kwargs['arch'] == 'x86_64': if kwargs['arch'] == 'x86_64':
append = ['-append', '{} nopat {}'.format(root, kernel_cli), LF] append = ['-append', '{} nopat {}'.format(root, kernel_cli), LF]
cmd.extend([ cmd.extend([
'-M', common.machine, LF, '-M', kwargs['machine'], LF,
'-device', 'edu', LF, '-device', 'edu', LF,
]) ])
elif common.is_arm: elif kwargs['is_arm']:
extra_emulator_args.extend(['-semihosting', LF]) extra_emulator_args.extend(['-semihosting', LF])
if kwargs['arch'] == 'arm': if kwargs['arch'] == 'arm':
cpu = 'cortex-a15' cpu = 'cortex-a15'
@@ -364,16 +364,16 @@ def main(args, extra_args=None):
[ [
# highmem=off needed since v3.0.0 due to: # highmem=off needed since v3.0.0 due to:
# http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
'-M', '{},highmem=off'.format(common.machine), LF, '-M', '{},highmem=off'.format(kwargs['machine']), LF,
'-cpu', cpu, LF, '-cpu', cpu, LF,
] + ] +
virtio_gpu_pci virtio_gpu_pci
) )
if common.baremetal is None: if kwargs['baremetal'] is None:
cmd.extend(append) cmd.extend(append)
if kwargs['tmux'] is not None: if kwargs['tmux'] is not None:
tmux_args = '--run-id {}'.format(kwargs['run_id']) tmux_args = '--run-id {}'.format(kwargs['run_id'])
if common.emulator == 'gem5': if kwargs['emulator'] == 'gem5':
tmux_cmd = './gem5-shell' tmux_cmd = './gem5-shell'
elif kwargs['wait_gdb']: elif kwargs['wait_gdb']:
tmux_cmd = './run-gdb' tmux_cmd = './run-gdb'
@@ -386,13 +386,13 @@ def main(args, extra_args=None):
kwargs['linux_build_id'], kwargs['linux_build_id'],
kwargs['run_id'], kwargs['run_id'],
) )
if common.baremetal: if kwargs['baremetal']:
tmux_args += " --baremetal '{}'".format(common.baremetal) tmux_args += " --baremetal '{}'".format(kwargs['baremetal'])
if kwargs['userland']: if kwargs['userland']:
tmux_args += " --userland '{}'".format(kwargs['userland']) tmux_args += " --userland '{}'".format(kwargs['userland'])
tmux_args += ' {}'.format(kwargs['tmux']) tmux_args += ' {}'.format(kwargs['tmux'])
subprocess.Popen([ subprocess.Popen([
os.path.join(common.root_dir, 'tmu'), os.path.join(kwargs['root_dir'], 'tmu'),
"sleep 2;{} {}".format(tmux_cmd, tmux_args) "sleep 2;{} {}".format(tmux_cmd, tmux_args)
]) ])
cmd.extend(extra_emulator_args) cmd.extend(extra_emulator_args)
@@ -400,10 +400,10 @@ def main(args, extra_args=None):
if debug_vm or kwargs['terminal']: if debug_vm or kwargs['terminal']:
out_file = None out_file = None
else: else:
out_file = common.termout_file out_file = kwargs['termout_file']
self.sh.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) self.sh.run_cmd(cmd, cmd_file=kwargs['run_cmd_file'], out_file=out_file, extra_env=extra_env)
# Check if guest panicked. # Check if guest panicked.
if common.emulator == 'gem5': if kwargs['emulator'] == 'gem5':
# We have to do some parsing here because gem5 exits with status 0 even when panic happens. # We have to do some parsing here because gem5 exits with status 0 even when panic happens.
# Grepping for '^panic: ' does not work because some errors don't show that message. # Grepping for '^panic: ' does not work because some errors don't show that message.
panic_msg = b'--- BEGIN LIBC BACKTRACE ---$' panic_msg = b'--- BEGIN LIBC BACKTRACE ---$'
@@ -412,16 +412,16 @@ def main(args, extra_args=None):
panic_re = re.compile(panic_msg) panic_re = re.compile(panic_msg)
error_string_found = False error_string_found = False
if out_file is not None and not kwargs['dry_run']: if out_file is not None and not kwargs['dry_run']:
with open(common.termout_file, 'br') as logfile: with open(kwargs['termout_file'], 'br') as logfile:
for line in logfile: for line in logfile:
if panic_re.search(line): if panic_re.search(line):
error_string_found = True error_string_found = True
if os.path.exists(common.guest_terminal_file): if os.path.exists(kwargs['guest_terminal_file']):
with open(common.guest_terminal_file, 'br') as logfile: with open(kwargs['guest_terminal_file'], 'br') as logfile:
lines = logfile.readlines() lines = logfile.readlines()
if lines: if lines:
last_line = lines[-1] last_line = lines[-1]
if last_line.rstrip() == common.magic_fail_string: if last_line.rstrip() == kwargs['magic_fail_string']:
error_string_found = True error_string_found = True
if error_string_found: if error_string_found:
common.log_error('simulation error detected by parsing logs') common.log_error('simulation error detected by parsing logs')

8
run-docker
View File

@@ -5,10 +5,10 @@ import os
import common import common
container_name = common.repo_short_id container_name = kwargs['repo_short_id']
container_hostname = common.repo_short_id container_hostname = kwargs['repo_short_id']
image_name = common.repo_short_id image_name = kwargs['repo_short_id']
target_dir = '/root/{}'.format(common.repo_short_id) target_dir = '/root/{}'.format(kwargs['repo_short_id'])
docker = ['sudo', 'docker'] docker = ['sudo', 'docker']
def create(args): def create(args):
self.sh.run_cmd(docker + ['build', '-t', image_name, '.', LF]) self.sh.run_cmd(docker + ['build', '-t', image_name, '.', LF])

26
run-gdb
View File

@@ -120,15 +120,15 @@ def main(args, extra_args=None):
break_at = ['-ex', 'break {}'.format(kwargs['break_at']), LF] break_at = ['-ex', 'break {}'.format(kwargs['break_at']), LF]
else: else:
break_at = [] break_at = []
linux_full_system = (common.baremetal is None and kwargs['userland'] is None) linux_full_system = (kwargs['baremetal'] is None and kwargs['userland'] is None)
if kwargs['userland']: if kwargs['userland']:
image = common.resolve_userland(kwargs['userland']) image = common.resolve_userland(kwargs['userland'])
elif common.baremetal: elif kwargs['baremetal']:
image = common.image image = kwargs['image']
test_script_path = os.path.splitext(common.source_path)[0] + '.py' test_script_path = os.path.splitext(kwargs['source_path'])[0] + '.py'
else: else:
image = common.vmlinux image = kwargs['vmlinux']
if common.baremetal: if kwargs['baremetal']:
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
else: else:
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
@@ -138,14 +138,14 @@ def main(args, extra_args=None):
['-q', LF] ['-q', LF]
) )
if linux_full_system: if linux_full_system:
cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), LF]) cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(kwargs['linux_build_dir']), LF])
if kwargs['sim']: if kwargs['sim']:
target = 'sim' target = 'sim'
else: else:
if kwargs['kgdb']: if kwargs['kgdb']:
port = common.extra_serial_port port = kwargs['extra_serial_port']
else: else:
port = common.gdb_port port = kwargs['gdb_port']
target = 'remote localhost:{}'.format(port) target = 'remote localhost:{}'.format(port)
cmd.extend([ cmd.extend([
'-ex', 'file {}'.format(image), LF, '-ex', 'file {}'.format(image), LF,
@@ -175,11 +175,11 @@ def main(args, extra_args=None):
# which gets put on the kernel build root when python debugging scripts are enabled. # which gets put on the kernel build root when python debugging scripts are enabled.
cmd.extend(['-ex', 'continue', LF]) cmd.extend(['-ex', 'continue', LF])
if not kwargs['no_lxsymbols'] and linux_full_system: if not kwargs['no_lxsymbols'] and linux_full_system:
cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), LF]) cmd.extend(['-ex', 'lx-symbols {}'.format(kwargs['kernel_modules_build_subdir']), LF])
cmd.extend(after) cmd.extend(after)
if kwargs['test']: if kwargs['test']:
GdbTestcase( GdbTestcase(
common.source_path, kwargs['source_path'],
test_script_path, test_script_path,
cmd, cmd,
verbose=kwargs['verbose'], verbose=kwargs['verbose'],
@@ -190,8 +190,8 @@ def main(args, extra_args=None):
# vmlinux: No such file or directory. # vmlinux: No such file or directory.
return self.sh.run_cmd( return self.sh.run_cmd(
cmd, cmd,
cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'), cmd_file=os.path.join(kwargs['run_dir'], 'run-gdb.sh'),
cwd=common.linux_build_dir cwd=kwargs['linux_build_dir']
) )
if __name__ == '__main__': if __name__ == '__main__':

4
run-gdb-user
View File

@@ -5,7 +5,7 @@ import os
import sys import sys
import common import common
rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'run-gdb')) rungdb = imp.load_source('rungdb', os.path.join(kwargs['root_dir'], 'run-gdb'))
parser = common.get_argparse(argparse_args={ parser = common.get_argparse(argparse_args={
'description': '''GDB step debug guest userland processes without gdbserver. 'description': '''GDB step debug guest userland processes without gdbserver.
@@ -25,7 +25,7 @@ parser.add_argument(
) )
args = common.setup(parser) args = common.setup(parser)
executable = common.resolve_userland(kwargs['executable']) executable = common.resolve_userland(kwargs['executable'])
addr = common.get_elf_entry(os.path.join(common.buildroot_build_build_dir, executable)) addr = common.get_elf_entry(os.path.join(kwargs['buildroot_build_build_dir'], executable))
extra_args = {} extra_args = {}
extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(addr)) extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(addr))
# Or else lx-symbols throws for arm: # Or else lx-symbols throws for arm:

6
run-gdbserver
View File

@@ -20,9 +20,9 @@ args = common.setup(parser)
sys.exit(subprocess.Popen([ sys.exit(subprocess.Popen([
common.get_toolchain_tool('gdb'), common.get_toolchain_tool('gdb'),
'-q', '-q',
'-ex', 'set sysroot {}'.format(common.buildroot_staging_dir), '-ex', 'set sysroot {}'.format(kwargs['buildroot_staging_dir']),
'-ex', 'target remote localhost:{}'.format(common.qemu_hostfwd_generic_port), '-ex', 'target remote localhost:{}'.format(kwargs['qemu_hostfwd_generic_port']),
'-ex', 'tbreak {}'.format(kwargs['break_at']), '-ex', 'tbreak {}'.format(kwargs['break_at']),
'-ex', 'continue', '-ex', 'continue',
os.path.join(common.buildroot_build_build_dir, common.resolve_userland(kwargs['executable'])), os.path.join(kwargs['buildroot_build_build_dir'], common.resolve_userland(kwargs['executable'])),
]).wait()) ]).wait())

10
run-toolchain
View File

@@ -35,16 +35,16 @@ parser.add_argument(
nargs='*' nargs='*'
) )
args = common.setup(parser) args = common.setup(parser)
if common.baremetal is None: if kwargs['baremetal'] is None:
image = common.vmlinux image = kwargs['vmlinux']
else: else:
image = common.image image = kwargs['image']
tool= common.get_toolchain_tool(kwargs['tool']) tool= common.get_toolchain_tool(kwargs['tool'])
if kwargs['dry']: if kwargs['dry']:
print(tool) print(tool)
else: else:
sys.exit(self.sh.run_cmd( sys.exit(self.sh.run_cmd(
[tool, LF] [tool, LF]
+ common.add_newlines(kwargs['extra_args']), + self.sh.add_newlines(kwargs['extra_args']),
cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'), cmd_file=os.path.join(kwargs['run_dir'], 'run-toolchain.sh'),
)) ))

6
shell_helpers.py
View File

@@ -1,10 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import distutils.file_util
import itertools import itertools
import os import os
import shlex import shlex
import shutil import shutil
import signal import signal
import subprocess
class LF: class LF:
''' '''
@@ -227,8 +229,8 @@ class ShellHelpers:
def strip_newlines(self, cmd): def strip_newlines(self, cmd):
return [x for x in cmd if x != LF] return [x for x in cmd if x != LF]
def rmrf(path): def rmrf(self, path):
self.print_cmd(['rm', '-r', '-f', path]) self.print_cmd(['rm', '-r', '-f', path, LF])
if not self.dry_run and os.path.exists(path): if not self.dry_run and os.path.exists(path):
shutil.rmtree(path) shutil.rmtree(path)

10
trace-boot
View File

@@ -6,8 +6,8 @@ import subprocess
import re import re
import common import common
run = imp.load_source('run', os.path.join(common.root_dir, 'run')) run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run'))
qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(common.root_dir, 'qemu-trace2txt')) qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(kwargs['root_dir'], 'qemu-trace2txt'))
parser = common.get_argparse(argparse_args={ parser = common.get_argparse(argparse_args={
'description': '''Trace the PIC addresses executed on a Linux kernel boot. 'description': '''Trace the PIC addresses executed on a Linux kernel boot.
@@ -23,7 +23,7 @@ args = common.setup(parser)
extra_args = { extra_args = {
'extra_emulator_args': kwargs['extra_emulator_args'], 'extra_emulator_args': kwargs['extra_emulator_args'],
} }
if common.emulator == 'gem5': if kwargs['emulator'] == 'gem5':
extra_args.update({ extra_args.update({
'eval': 'm5 exit', 'eval': 'm5 exit',
'trace': 'Exec,-ExecSymbol,-ExecMicro', 'trace': 'Exec,-ExecSymbol,-ExecMicro',
@@ -39,10 +39,10 @@ else:
# Instruction count. # Instruction count.
# We could put this on a separate script, but it just adds more arch boilerplate to a new script. # We could put this on a separate script, but it just adds more arch boilerplate to a new script.
# So let's just leave it here for now since it did not add a significant processing time. # So let's just leave it here for now since it did not add a significant processing time.
kernel_entry_addr = hex(common.get_elf_entry(common.vmlinux)) kernel_entry_addr = hex(common.get_elf_entry(kwargs['vmlinux']))
nlines = 0 nlines = 0
nlines_firmware = 0 nlines_firmware = 0
with open(common.qemu_trace_txt_file, 'r') as trace_file: with open(kwargs['qemu_trace_txt_file'], 'r') as trace_file:
in_firmware = True in_firmware = True
for line in trace_file: for line in trace_file:
line = line.rstrip() line = line.rstrip()

20
trace2line
View File

@@ -19,17 +19,17 @@ parser = common.get_argparse(argparse_args={
}) })
args = common.setup(parser) args = common.setup(parser)
sys.exit(subprocess.Popen([ sys.exit(subprocess.Popen([
os.path.join(common.root_dir, 'trace2line.sh'), os.path.join(kwargs['root_dir'], 'trace2line.sh'),
'true' if common.emulator == 'gem5' else 'false', 'true' if kwargs['emulator'] == 'gem5' else 'false',
common.trace_txt_file, kwargs['trace_txt_file'],
common.get_toolchain_tool('addr2line'), common.get_toolchain_tool('addr2line'),
common.vmlinux, kwargs['vmlinux'],
common.run_dir, kwargs['run_dir'],
]).wait()) ]).wait())
# This was the full conversion attempt. # This was the full conversion attempt.
# if common.emulator == 'gem5': # if kwargs['emulator'] == 'gem5':
# def get_pc(line): # def get_pc(line):
# # TODO # # TODO
# # stdin = sed -r 's/^.* (0x[^. ]*)[. ].*/\1/' "$common_trace_txt_file") # # stdin = sed -r 's/^.* (0x[^. ]*)[. ].*/\1/' "$common_trace_txt_file")
@@ -42,15 +42,15 @@ sys.exit(subprocess.Popen([
# [ # [
# common.get_toolchain_tool('addr2line'), # common.get_toolchain_tool('addr2line'),
# '-e', # '-e',
# common.vmlinux, # kwargs['vmlinux'],
# '-f', # '-f',
# '-p', # '-p',
# ], # ],
# stdout=subprocess.PIPE, # stdout=subprocess.PIPE,
# stdin=subprocess.PIPE, # stdin=subprocess.PIPE,
# ) as proc, \ # ) as proc, \
# open(common.trace_txt_file, 'r') as infile, \ # open(kwargs['trace_txt_file'], 'r') as infile, \
# open(os.path.join(common.run_dir, 'trace-lines.txt'), 'w') as outfile \ # open(os.path.join(kwargs['run_dir'], 'trace-lines.txt'), 'w') as outfile \
# : # :
# for in_line in infile: # for in_line in infile:
# proc.stdin.write(get_pc(in_line).encode()) # proc.stdin.write(get_pc(in_line).encode())
@@ -58,5 +58,5 @@ sys.exit(subprocess.Popen([
# stdout = proc.stdout.read() # stdout = proc.stdout.read()
# outfile.write(stdout.decode()) # outfile.write(stdout.decode())
# # TODO # # TODO
# # sed -E "s|at ${common.linux_build_dir}/(\./\|)||" # # sed -E "s|at ${kwargs['linux_build_dir']}/(\./\|)||"
# # uniq -c # # uniq -c