mirror of
https://github.com/cirosantilli/linux-kernel-module-cheat.git
synced 2026-01-23 02:05:57 +01:00
dae60aa248
Then inside, split packages/lkmc into kernel_modules and userland, to keep userland out of the kernel_modules parent path, which makes no sense. Copy built modules and userland to the output rootfs overlay. Document Linux distro tradeoffs.
751 lines
29 KiB
Python
751 lines
29 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import base64
|
|
import copy
|
|
import datetime
|
|
import distutils.file_util
|
|
import glob
|
|
import imp
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib
|
|
import urllib.request
|
|
|
|
this = sys.modules[__name__]
|
|
root_dir = os.path.dirname(os.path.abspath(__file__))
|
|
data_dir = os.path.join(root_dir, 'data')
|
|
p9_dir = os.path.join(data_dir, '9p')
|
|
gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
|
|
out_dir = os.path.join(root_dir, 'out')
|
|
bench_boot = os.path.join(out_dir, 'bench-boot.txt')
|
|
packages_dir = os.path.join(root_dir, 'packages')
|
|
lkmc_package_src_dir = os.path.join(this.packages_dir, 'lkmc')
|
|
kernel_modules_subdir = 'kernel_modules'
|
|
kernel_modules_src_dir = os.path.join(this.lkmc_package_src_dir, this.kernel_modules_subdir)
|
|
userland_subdir = 'userland'
|
|
userland_src_dir = os.path.join(this.lkmc_package_src_dir, this.userland_subdir)
|
|
submodules_dir = os.path.join(root_dir, 'submodules')
|
|
buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
|
|
crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng')
|
|
linux_src_dir = os.path.join(submodules_dir, 'linux')
|
|
linux_config_dir = os.path.join(this.root_dir, 'kernel_config')
|
|
rootfs_overlay_dir = os.path.join(this.root_dir, 'rootfs_overlay')
|
|
extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux')
|
|
qemu_src_dir = os.path.join(submodules_dir, 'qemu')
|
|
parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark')
|
|
ccache_dir = os.path.join('/usr', 'lib', 'ccache')
|
|
arch_map = {
|
|
'a': 'arm',
|
|
'A': 'aarch64',
|
|
'x': 'x86_64',
|
|
}
|
|
arches = [arch_map[k] for k in arch_map]
|
|
gem5_cpt_prefix = '^cpt\.'
|
|
sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
|
|
release_dir = os.path.join(this.out_dir, 'release')
|
|
release_zip_file = os.path.join(this.release_dir, 'lkmc-{}.zip'.format(this.sha))
|
|
github_repo_id = 'cirosantilli/linux-kernel-module-cheat'
|
|
asm_ext = '.S'
|
|
c_ext = '.c'
|
|
header_ext = '.h'
|
|
kernel_module_ext = '.ko'
|
|
obj_ext = '.o'
|
|
executable_ext = '.out'
|
|
config_file = os.path.join(data_dir, 'config')
|
|
command_prefix = '+ '
|
|
if os.path.exists(config_file):
|
|
config = imp.load_source('config', config_file)
|
|
configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')}
|
|
|
|
def add_build_arguments(parser):
|
|
parser.add_argument(
|
|
'--clean',
|
|
help='Clean the build instead of building.',
|
|
action='store_true',
|
|
)
|
|
|
|
def base64_encode(string):
|
|
return base64.b64encode(string.encode()).decode()
|
|
|
|
def copy_dir_if_update_non_recursive(srcdir, destdir, filter_ext=None):
|
|
os.makedirs(destdir, exist_ok=True)
|
|
for basename in os.listdir(srcdir):
|
|
src = os.path.join(srcdir, basename)
|
|
if os.path.isfile(src):
|
|
noext, ext = os.path.splitext(basename)
|
|
if filter_ext is not None and ext == filter_ext:
|
|
distutils.file_util.copy_file(
|
|
src,
|
|
os.path.join(destdir, basename),
|
|
update=1,
|
|
)
|
|
|
|
def gem_list_checkpoint_dirs():
|
|
'''
|
|
List checkpoint directory, oldest first.
|
|
'''
|
|
global this
|
|
prefix_re = re.compile(this.gem5_cpt_prefix)
|
|
files = list(filter(lambda x: os.path.isdir(os.path.join(this.m5out_dir, x)) and prefix_re.search(x), os.listdir(this.m5out_dir)))
|
|
files.sort(key=lambda x: os.path.getmtime(os.path.join(this.m5out_dir, x)))
|
|
return files
|
|
|
|
def get_argparse(default_args=None, argparse_args=None):
|
|
'''
|
|
Return an argument parser with common arguments set.
|
|
|
|
:type default_args: Dict[str,str]
|
|
:type argparse_args: Dict
|
|
'''
|
|
global this
|
|
if default_args is None:
|
|
default_args = {}
|
|
if argparse_args is None:
|
|
argparse_args = {}
|
|
arch_choices = []
|
|
for key in this.arch_map:
|
|
arch_choices.append(key)
|
|
arch_choices.append(this.arch_map[key])
|
|
default_build_id = 'default'
|
|
parser = argparse.ArgumentParser(
|
|
formatter_class=argparse.RawTextHelpFormatter,
|
|
**argparse_args
|
|
)
|
|
parser.add_argument(
|
|
'-a', '--arch', choices=arch_choices, default='x86_64',
|
|
help='CPU architecture. Default: %(default)s'
|
|
)
|
|
parser.add_argument(
|
|
'--baremetal',
|
|
help='Use Baremetal examples instead of Linux kernel ones'
|
|
)
|
|
parser.add_argument(
|
|
'--buildroot-build-id',
|
|
default=default_build_id,
|
|
help='Buildroot build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
|
|
)
|
|
parser.add_argument(
|
|
'--crosstool-ng-build-id', default=default_build_id,
|
|
help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds. Default: %(default)s'
|
|
)
|
|
parser.add_argument(
|
|
'--docker', default=False, action='store_true',
|
|
help='''\
|
|
Use the docker download Ubuntu root filesystem instead of the default Buildroot one.
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'-g', '--gem5', default=False, action='store_true',
|
|
help='Use gem5 instead of QEMU'
|
|
)
|
|
parser.add_argument(
|
|
'--gem5-src',
|
|
help='''\
|
|
Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`.
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'-L', '--linux-build-id', default=default_build_id,
|
|
help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s'
|
|
)
|
|
parser.add_argument(
|
|
'--machine',
|
|
help='''Machine type.
|
|
QEMU default: virt
|
|
gem5 default: VExpress_GEM5_V1
|
|
See the documentation for other values known to work.
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'-M', '--gem5-build-id', default=default_build_id,
|
|
help='gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
|
|
)
|
|
parser.add_argument(
|
|
'-N', '--gem5-worktree',
|
|
help='''\
|
|
gem5 git worktree to use for build and Python scripts at runtime. Automatically
|
|
create a new git worktree with the given id if one does not exist. If not
|
|
given, just use the submodule source.
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'--gem5-build-dir',
|
|
help='''\
|
|
Use the given directory as the gem5 build directory.
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'--gem5-source-dir',
|
|
help='''\
|
|
Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`.
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'-n', '--run-id', default='0',
|
|
help='''\
|
|
ID for run outputs such as gem5's m5out. Allows you to do multiple runs,
|
|
and then inspect separate outputs later in different output directories.
|
|
Default: %(default)s
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'-P', '--prebuilt', default=False, action='store_true',
|
|
help='''\
|
|
Use prebuilt packaged host utilities as much as possible instead
|
|
of the ones we built ourselves. Saves build time, but decreases
|
|
the likelihood of compatibility.
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'--port-offset', type=int,
|
|
help='''\
|
|
Increase the ports to be used such as for GDB by an offset to run multiple
|
|
instances in parallel.
|
|
Default: the run ID (-n) if that is an integer, otherwise 0.
|
|
'''
|
|
)
|
|
parser.add_argument(
|
|
'-Q', '--qemu-build-id', default=default_build_id,
|
|
help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
|
|
)
|
|
parser.add_argument(
|
|
'-t', '--gem5-build-type', default='opt',
|
|
help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
|
|
)
|
|
parser.add_argument(
|
|
'-v', '--verbose', default=False, action='store_true',
|
|
help='Show full compilation commands when they are not shown by default.'
|
|
)
|
|
if hasattr(this, 'configs'):
|
|
defaults = this.configs.copy()
|
|
else:
|
|
defaults = {}
|
|
defaults.update(default_args)
|
|
# A bit ugly as it actually changes the defaults shown on --help, but we can't do any better
|
|
# because it is impossible to check if arguments were given or not...
|
|
# - https://stackoverflow.com/questions/30487767/check-if-argparse-optional-argument-is-set-or-not
|
|
# - https://stackoverflow.com/questions/3609852/which-is-the-best-way-to-allow-configuration-options-be-overridden-at-the-comman
|
|
parser.set_defaults(**defaults)
|
|
return parser
|
|
|
|
def get_elf_entry(elf_file_path):
|
|
global this
|
|
readelf_header = subprocess.check_output([
|
|
this.get_toolchain_tool('readelf'),
|
|
'-h',
|
|
elf_file_path
|
|
])
|
|
for line in readelf_header.decode().split('\n'):
|
|
split = line.split()
|
|
if line.startswith(' Entry point address:'):
|
|
addr = line.split()[-1]
|
|
break
|
|
return int(addr, 0)
|
|
|
|
def get_stats(stat_re=None, stats_file=None):
|
|
global this
|
|
if stat_re is None:
|
|
stat_re = '^system.cpu[0-9]*.numCycles$'
|
|
if stats_file is None:
|
|
stats_file = this.stats_file
|
|
stat_re = re.compile(stat_re)
|
|
ret = []
|
|
with open(stats_file, 'r') as statfile:
|
|
for line in statfile:
|
|
if line[0] != '-':
|
|
cols = line.split()
|
|
if len(cols) > 1 and stat_re.search(cols[0]):
|
|
ret.append(cols[1])
|
|
return ret
|
|
|
|
def get_toolchain_prefix(tool, allowed_toolchains=None):
|
|
buildroot_full_prefix = os.path.join(this.host_bin_dir, this.buildroot_toolchain_prefix)
|
|
buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
|
|
crosstool_ng_full_prefix = os.path.join(this.crosstool_ng_bin_dir, this.crosstool_ng_toolchain_prefix)
|
|
crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
|
|
host_tool = '{}-{}'.format(this.ubuntu_toolchain_prefix, tool)
|
|
host_path = shutil.which(host_tool)
|
|
if host_path is not None:
|
|
host_exists = True
|
|
host_full_prefix = host_path[:-(len(tool)+1)]
|
|
else:
|
|
host_exists = False
|
|
host_full_prefix = None
|
|
known_toolchains = {
|
|
'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix),
|
|
'buildroot': (buildroot_exists, buildroot_full_prefix),
|
|
'host': (host_exists, host_full_prefix),
|
|
}
|
|
if allowed_toolchains is None:
|
|
if this.baremetal is None:
|
|
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
|
|
else:
|
|
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
|
|
tried = []
|
|
for toolchain in allowed_toolchains:
|
|
exists, prefix = known_toolchains[toolchain]
|
|
tried.append('{}-{}'.format(prefix, tool))
|
|
if exists:
|
|
return prefix
|
|
raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))
|
|
|
|
def get_toolchain_tool(tool, allowed_toolchains=None):
|
|
return '{}-{}'.format(this.get_toolchain_prefix(tool, allowed_toolchains), tool)
|
|
|
|
def github_make_request(
|
|
authenticate=False,
|
|
data=None,
|
|
extra_headers=None,
|
|
path='',
|
|
subdomain='api',
|
|
url_params=None,
|
|
**extra_request_args
|
|
):
|
|
global this
|
|
if extra_headers is None:
|
|
extra_headers = {}
|
|
headers = {'Accept': 'application/vnd.github.v3+json'}
|
|
headers.update(extra_headers)
|
|
if authenticate:
|
|
headers['Authorization'] = 'token ' + os.environ['LKMC_GITHUB_TOKEN']
|
|
if url_params is not None:
|
|
path += '?' + urllib.parse.urlencode(url_params)
|
|
request = urllib.request.Request(
|
|
'https://' + subdomain + '.github.com/repos/' + github_repo_id + path,
|
|
headers=headers,
|
|
data=data,
|
|
**extra_request_args
|
|
)
|
|
response_body = urllib.request.urlopen(request).read().decode()
|
|
if response_body:
|
|
_json = json.loads(response_body)
|
|
else:
|
|
_json = {}
|
|
return _json
|
|
|
|
def log_error(msg):
|
|
print('error: {}'.format(msg), file=sys.stderr)
|
|
|
|
def mkdir():
|
|
global this
|
|
os.makedirs(this.build_dir, exist_ok=True)
|
|
os.makedirs(this.gem5_build_dir, exist_ok=True)
|
|
os.makedirs(this.gem5_run_dir, exist_ok=True)
|
|
os.makedirs(this.qemu_run_dir, exist_ok=True)
|
|
os.makedirs(this.p9_dir, exist_ok=True)
|
|
|
|
def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
|
|
'''
|
|
Format a command given as a list of strings so that it can
|
|
be viewed nicely and executed by bash directly and print it to stdout.
|
|
|
|
Optionally save the command to cmd_file file, and add extra_env
|
|
environment variables to the command generated.
|
|
'''
|
|
newline_separator = ' \\\n'
|
|
out = []
|
|
if extra_env is None:
|
|
extra_env = {}
|
|
if cwd is not None:
|
|
out.append('cd {} &&{}'.format(shlex.quote(cwd), newline_separator))
|
|
if extra_paths is not None:
|
|
out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)) + newline_separator)
|
|
for key in extra_env:
|
|
out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])) + newline_separator)
|
|
for arg in cmd:
|
|
out.append(shlex.quote(arg) + newline_separator)
|
|
out = ' '.join(out) + ';\n'
|
|
print(this.command_prefix + out, end='')
|
|
if cmd_file is not None:
|
|
with open(cmd_file, 'w') as f:
|
|
f.write('#!/usr/bin/env bash\n')
|
|
f.write(out)
|
|
st = os.stat(cmd_file)
|
|
os.chmod(cmd_file, st.st_mode | stat.S_IXUSR)
|
|
|
|
def print_time(ellapsed_seconds):
|
|
hours, rem = divmod(ellapsed_seconds, 3600)
|
|
minutes, seconds = divmod(rem, 60)
|
|
print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))
|
|
|
|
def raw_to_qcow2(prebuilt=False, reverse=False):
|
|
global this
|
|
if prebuilt:
|
|
qemu_img_executable = this.qemu_img_basename
|
|
else:
|
|
qemu_img_executable = this.qemu_img_executable
|
|
infmt = 'raw'
|
|
outfmt = 'qcow2'
|
|
infile = this.rootfs_raw_file
|
|
outfile = this.qcow2_file
|
|
if reverse:
|
|
tmp = infmt
|
|
infmt = outfmt
|
|
outfmt = tmp
|
|
tmp = infile
|
|
infile = outfile
|
|
outfile = tmp
|
|
assert this.run_cmd([
|
|
qemu_img_executable,
|
|
# Prevent qemu-img from generating trace files like QEMU. Disgusting.
|
|
'-T', 'pr_manager_run,file=/dev/null',
|
|
'convert',
|
|
'-f', infmt,
|
|
'-O', outfmt,
|
|
infile,
|
|
outfile,
|
|
]) == 0
|
|
|
|
def raise_no_x86(arch):
|
|
if (arch == 'x86_64'):
|
|
raise Exception('x86_64 not yet supported')
|
|
|
|
def resolve_args(defaults, args, extra_args):
|
|
if extra_args is None:
|
|
extra_args = {}
|
|
argcopy = copy.copy(args)
|
|
argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items()))
|
|
return argcopy
|
|
|
|
def rmrf(path):
|
|
if os.path.exists(path):
|
|
shutil.rmtree(path)
|
|
|
|
def run_cmd(
|
|
cmd,
|
|
cmd_file=None,
|
|
out_file=None,
|
|
show_stdout=True,
|
|
show_cmd=True,
|
|
extra_env=None,
|
|
extra_paths=None,
|
|
delete_env=None,
|
|
**kwargs
|
|
):
|
|
'''
|
|
Run a command. Write the command to stdout before running it.
|
|
|
|
Wait until the command finishes execution.
|
|
|
|
:param cmd: command to run
|
|
:type cmd: List[str]
|
|
|
|
:param cmd_file: if not None, write the command to be run to that file
|
|
:type cmd_file: str
|
|
|
|
:param out_file: if not None, write the stdout and stderr of the command the file
|
|
:type out_file: str
|
|
|
|
:param show_stdout: wether to show stdout and stderr on the terminal or not
|
|
:type show_stdout: bool
|
|
|
|
:param extra_env: extra environment variables to add when running the command
|
|
:type extra_env: Dict[str,str]
|
|
'''
|
|
if out_file is not None:
|
|
stdout = subprocess.PIPE
|
|
stderr = subprocess.STDOUT
|
|
else:
|
|
if show_stdout:
|
|
stdout = None
|
|
stderr = None
|
|
else:
|
|
stdout = subprocess.DEVNULL
|
|
stderr = subprocess.DEVNULL
|
|
if extra_env is None:
|
|
extra_env = {}
|
|
if delete_env is None:
|
|
delete_env = []
|
|
if 'cwd' in kwargs:
|
|
cwd = kwargs['cwd']
|
|
else:
|
|
cwd = None
|
|
env = os.environ.copy()
|
|
env.update(extra_env)
|
|
if extra_paths is not None:
|
|
path = ':'.join(extra_paths)
|
|
if 'PATH' in os.environ:
|
|
path += ':' + os.environ['PATH']
|
|
env['PATH'] = path
|
|
for key in delete_env:
|
|
if key in env:
|
|
del env[key]
|
|
if show_cmd:
|
|
print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths)
|
|
|
|
# Otherwise Ctrl + C gives:
|
|
# - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
|
|
# - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
|
|
sigint_old = signal.getsignal(signal.SIGINT)
|
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
|
|
# Otherwise BrokenPipeError when piping through | grep
|
|
# But if I do this, my terminal gets broken at the end. Why, why, why.
|
|
# https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python
|
|
# Ignoring the exception is not enough as it prints a warning anyways.
|
|
#sigpipe_old = signal.getsignal(signal.SIGPIPE)
|
|
#signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
|
|
|
# https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
|
|
with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
|
|
if out_file is not None:
|
|
os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True)
|
|
with open(out_file, 'bw') as logfile:
|
|
while True:
|
|
byte = proc.stdout.read(1)
|
|
if byte:
|
|
if show_stdout:
|
|
sys.stdout.buffer.write(byte)
|
|
try:
|
|
sys.stdout.flush()
|
|
except BlockingIOError:
|
|
# TODO understand. Why, Python, why.
|
|
pass
|
|
logfile.write(byte)
|
|
else:
|
|
break
|
|
signal.signal(signal.SIGINT, sigint_old)
|
|
#signal.signal(signal.SIGPIPE, sigpipe_old)
|
|
return proc.returncode
|
|
|
|
def setup(parser):
|
|
'''
|
|
Parse the command line arguments, and setup several variables based on them.
|
|
Typically done after getting inputs from the command line arguments.
|
|
'''
|
|
global this
|
|
args = parser.parse_args()
|
|
if args.arch in this.arch_map:
|
|
args.arch = this.arch_map[args.arch]
|
|
this.machine = args.machine
|
|
if args.arch == 'arm':
|
|
this.armv = 7
|
|
this.gem5_arch = 'ARM'
|
|
this.mcpu = 'cortex-a15'
|
|
this.buildroot_toolchain_prefix = 'arm-buildroot-linux-uclibcgnueabihf'
|
|
this.crosstool_ng_toolchain_prefix = 'arm-unknown-eabi'
|
|
this.ubuntu_toolchain_prefix = 'arm-linux-gnueabihf'
|
|
if args.gem5:
|
|
if this.machine is None:
|
|
this.machine = 'VExpress_GEM5_V1'
|
|
else:
|
|
if this.machine is None:
|
|
this.machine = 'virt'
|
|
elif args.arch == 'aarch64':
|
|
this.armv = 8
|
|
this.gem5_arch = 'ARM'
|
|
this.mcpu = 'cortex-a57'
|
|
this.buildroot_toolchain_prefix = 'aarch64-buildroot-linux-uclibc'
|
|
this.crosstool_ng_toolchain_prefix = 'aarch64-unknown-elf'
|
|
this.ubuntu_toolchain_prefix = 'aarch64-linux-gnu'
|
|
if args.gem5:
|
|
if this.machine is None:
|
|
this.machine = 'VExpress_GEM5_V1'
|
|
else:
|
|
if this.machine is None:
|
|
this.machine = 'virt'
|
|
elif args.arch == 'x86_64':
|
|
this.crosstool_ng_toolchain_prefix = 'x86_64-unknown-elf'
|
|
this.gem5_arch = 'X86'
|
|
this.buildroot_toolchain_prefix = 'x86_64-buildroot-linux-uclibc'
|
|
this.ubuntu_toolchain_prefix = 'x86_64-linux-gnu'
|
|
if args.gem5:
|
|
if this.machine is None:
|
|
this.machine = 'TODO'
|
|
else:
|
|
if this.machine is None:
|
|
this.machine = 'pc'
|
|
this.buildroot_out_dir = os.path.join(this.out_dir, 'buildroot')
|
|
this.buildroot_build_dir = os.path.join(this.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch)
|
|
this.buildroot_download_dir = os.path.join(this.buildroot_out_dir, 'download')
|
|
this.buildroot_config_file = os.path.join(this.buildroot_build_dir, '.config')
|
|
this.build_dir = os.path.join(this.buildroot_build_dir, 'build')
|
|
this.qemu_build_dir = os.path.join(this.out_dir, 'qemu', args.qemu_build_id)
|
|
this.qemu_executable_basename = 'qemu-system-{}'.format(args.arch)
|
|
this.qemu_executable = os.path.join(this.qemu_build_dir, '{}-softmmu'.format(args.arch), this.qemu_executable_basename)
|
|
this.qemu_img_basename = 'qemu-img'
|
|
this.qemu_img_executable = os.path.join(this.qemu_build_dir, this.qemu_img_basename)
|
|
this.qemu_guest_build_dir = os.path.join(this.build_dir, 'qemu-custom')
|
|
this.host_dir = os.path.join(this.buildroot_build_dir, 'host')
|
|
this.host_bin_dir = os.path.join(this.host_dir, 'usr', 'bin')
|
|
this.buildroot_pkg_config = os.path.join(this.host_bin_dir, 'pkg-config')
|
|
this.buildroot_images_dir = os.path.join(this.buildroot_build_dir, 'images')
|
|
this.buildroot_rootfs_raw_file = os.path.join(this.buildroot_images_dir, 'rootfs.ext2')
|
|
this.buildroot_qcow2_file = this.buildroot_rootfs_raw_file + '.qcow2'
|
|
this.staging_dir = os.path.join(this.out_dir, 'staging', args.arch)
|
|
this.buildroot_staging_dir = os.path.join(this.buildroot_build_dir, 'staging')
|
|
this.target_dir = os.path.join(this.buildroot_build_dir, 'target')
|
|
this.run_dir_base = os.path.join(this.out_dir, 'run')
|
|
this.gem5_run_dir = os.path.join(this.run_dir_base, 'gem5', args.arch, str(args.run_id))
|
|
this.m5out_dir = os.path.join(this.gem5_run_dir, 'm5out')
|
|
this.stats_file = os.path.join(this.m5out_dir, 'stats.txt')
|
|
this.trace_txt_file = os.path.join(this.m5out_dir, 'trace.txt')
|
|
this.gem5_readfile = os.path.join(this.gem5_run_dir, 'readfile')
|
|
this.gem5_termout_file = os.path.join(this.gem5_run_dir, 'termout.txt')
|
|
this.qemu_run_dir = os.path.join(this.run_dir_base, 'qemu', args.arch, str(args.run_id))
|
|
this.qemu_trace_basename = 'trace.bin'
|
|
this.qemu_trace_file = os.path.join(this.qemu_run_dir, 'trace.bin')
|
|
this.qemu_trace_txt_file = os.path.join(this.qemu_run_dir, 'trace.txt')
|
|
this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt')
|
|
this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile')
|
|
this.gem5_out_dir = os.path.join(this.out_dir, 'gem5')
|
|
if args.gem5_build_dir is None:
|
|
this.gem5_build_dir = os.path.join(this.gem5_out_dir, args.gem5_build_id, args.gem5_build_type)
|
|
else:
|
|
this.gem5_build_dir = args.gem5_build_dir
|
|
this.gem5_fake_iso = os.path.join(this.gem5_out_dir, 'fake.iso')
|
|
this.gem5_m5term = os.path.join(this.gem5_build_dir, 'm5term')
|
|
this.gem5_build_build_dir = os.path.join(this.gem5_build_dir, 'build')
|
|
this.gem5_executable = os.path.join(this.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type))
|
|
this.gem5_system_dir = os.path.join(this.gem5_build_dir, 'system')
|
|
this.crosstool_ng_out_dir = os.path.join(this.out_dir, 'crosstool-ng')
|
|
this.crosstool_ng_buildid_dir = os.path.join(this.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id)
|
|
this.crosstool_ng_install_dir = os.path.join(this.crosstool_ng_buildid_dir, 'install', args.arch)
|
|
this.crosstool_ng_bin_dir = os.path.join(this.crosstool_ng_install_dir, 'bin')
|
|
this.crosstool_ng_util_dir = os.path.join(this.crosstool_ng_buildid_dir, 'util')
|
|
this.crosstool_ng_config = os.path.join(this.crosstool_ng_util_dir, '.config')
|
|
this.crosstool_ng_defconfig = os.path.join(this.crosstool_ng_util_dir, 'defconfig')
|
|
this.crosstool_ng_executable = os.path.join(this.crosstool_ng_util_dir, 'ct-ng')
|
|
this.crosstool_ng_build_dir = os.path.join(this.crosstool_ng_buildid_dir, 'build')
|
|
this.crosstool_ng_download_dir = os.path.join(this.crosstool_ng_out_dir, 'download')
|
|
this.gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
|
|
if args.gem5_source_dir is not None:
|
|
this.gem5_src_dir = args.gem5_source_dir
|
|
assert(os.path.exists(args.gem5_source_dir))
|
|
else:
|
|
if args.gem5_worktree is not None:
|
|
this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree)
|
|
else:
|
|
this.gem5_src_dir = this.gem5_default_src_dir
|
|
if args.gem5:
|
|
this.executable = this.gem5_executable
|
|
this.run_dir = this.gem5_run_dir
|
|
this.termout_file = this.gem5_termout_file
|
|
else:
|
|
this.executable = this.qemu_executable
|
|
this.run_dir = this.qemu_run_dir
|
|
this.termout_file = this.qemu_termout_file
|
|
this.gem5_config_dir = os.path.join(this.gem5_src_dir, 'configs')
|
|
this.gem5_se_file = os.path.join(this.gem5_config_dir, 'example', 'se.py')
|
|
this.gem5_fs_file = os.path.join(this.gem5_config_dir, 'example', 'fs.py')
|
|
this.run_cmd_file = os.path.join(this.run_dir, 'run.sh')
|
|
|
|
# Linux
|
|
this.linux_buildroot_build_dir = os.path.join(this.build_dir, 'linux-custom')
|
|
this.linux_build_dir = os.path.join(this.out_dir, 'linux', args.linux_build_id, args.arch)
|
|
this.vmlinux = os.path.join(this.linux_build_dir, "vmlinux")
|
|
if args.arch == 'arm':
|
|
this.linux_arch = 'arm'
|
|
this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'zImage')
|
|
elif args.arch == 'aarch64':
|
|
this.linux_arch = 'arm64'
|
|
this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'Image')
|
|
elif args.arch == 'x86_64':
|
|
this.linux_arch = 'x86'
|
|
this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'bzImage')
|
|
this.linux_image = os.path.join(this.linux_build_dir, linux_image)
|
|
|
|
# Kernel modules.
|
|
this.kernel_modules_build_base_dir = os.path.join(this.out_dir, 'kernel_modules')
|
|
this.kernel_modules_build_dir = os.path.join(this.kernel_modules_build_base_dir, args.arch)
|
|
this.kernel_modules_build_host_dir = os.path.join(this.kernel_modules_build_base_dir, 'host')
|
|
this.userland_build_dir = os.path.join(this.out_dir, 'userland', args.arch)
|
|
this.out_rootfs_overlay_dir = os.path.join(this.out_dir, 'rootfs_overlay', args.arch)
|
|
|
|
# Ports
|
|
if args.port_offset is None:
|
|
try:
|
|
args.port_offset = int(args.run_id)
|
|
except ValueError:
|
|
args.port_offset = 0
|
|
if args.gem5:
|
|
this.gem5_telnet_port = 3456 + args.port_offset
|
|
this.gdb_port = 7000 + args.port_offset
|
|
else:
|
|
this.qemu_base_port = 45454 + 10 * args.port_offset
|
|
this.qemu_monitor_port = this.qemu_base_port + 0
|
|
this.qemu_hostfwd_generic_port = this.qemu_base_port + 1
|
|
this.qemu_hostfwd_ssh_port = this.qemu_base_port + 2
|
|
this.qemu_gdb_port = this.qemu_base_port + 3
|
|
this.gdb_port = this.qemu_gdb_port
|
|
|
|
# Baremetal.
|
|
this.baremetal = args.baremetal
|
|
this.baremetal_lib_basename = 'lib'
|
|
this.baremetal_src_dir = os.path.join(this.root_dir, 'baremetal')
|
|
this.baremetal_src_lib_dir = os.path.join(this.baremetal_src_dir, this.baremetal_lib_basename)
|
|
if args.gem5:
|
|
this.simulator_name = 'gem5'
|
|
else:
|
|
this.simulator_name = 'qemu'
|
|
this.baremetal_out_dir = os.path.join(out_dir, 'baremetal', args.arch, this.simulator_name, this.machine)
|
|
this.baremetal_out_lib_dir = os.path.join(this.baremetal_out_dir, this.baremetal_lib_basename)
|
|
this.baremetal_out_ext = '.elf'
|
|
|
|
|
|
# Docker
|
|
this.docker_build_dir = os.path.join(this.out_dir, 'docker', args.arch)
|
|
this.docker_tar_dir = os.path.join(this.docker_build_dir, 'export')
|
|
this.docker_tar_file = os.path.join(this.docker_build_dir, 'export.tar')
|
|
this.docker_rootfs_raw_file = os.path.join(this.docker_build_dir, 'export.ext2')
|
|
this.docker_qcow2_file = os.path.join(this.docker_rootfs_raw_file + '.qcow2')
|
|
if args.docker:
|
|
this.rootfs_raw_file = this.docker_rootfs_raw_file
|
|
this.qcow2_file = this.docker_qcow2_file
|
|
else:
|
|
this.rootfs_raw_file = this.buildroot_rootfs_raw_file
|
|
this.qcow2_file = this.buildroot_qcow2_file
|
|
|
|
# Image.
|
|
if args.baremetal is None:
|
|
if args.gem5:
|
|
this.image = this.vmlinux
|
|
this.disk_image = this.rootfs_raw_file
|
|
else:
|
|
this.image = this.linux_image
|
|
this.disk_image = this.qcow2_file
|
|
else:
|
|
this.disk_image = this.gem5_fake_iso
|
|
paths = [
|
|
os.path.join(this.baremetal_out_dir, this.baremetal),
|
|
os.path.join(
|
|
this.baremetal_out_dir,
|
|
os.path.relpath(this.baremetal, this.baremetal_src_dir),
|
|
)
|
|
]
|
|
paths[:] = [os.path.splitext(path)[0] + this.baremetal_out_ext for path in paths]
|
|
found = False
|
|
for path in paths:
|
|
if os.path.exists(path):
|
|
found = True
|
|
break
|
|
if not found and this.baremetal != 'all':
|
|
raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
|
|
this.image = path
|
|
return args
|
|
|
|
def write_configs(config_path, configs, config_fragments=None):
|
|
"""
|
|
Write extra configs into the Buildroot config file.
|
|
TODO Can't get rid of these for now with nice fragments:
|
|
http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
|
|
"""
|
|
if config_fragments is None:
|
|
config_fragments = []
|
|
with open(config_path, 'a') as config_file:
|
|
for config_fragment in config_fragments:
|
|
with open(config_fragment, 'r') as config_fragment:
|
|
for line in config_fragment:
|
|
config_file.write(line)
|
|
for config in configs:
|
|
config_file.write(config + '\n')
|