3rd-party/linux-kernel-module-cheat
1
0
Fork 0
mirror of https://github.com/cirosantilli/linux-kernel-module-cheat.git synced 2026-01-29 21:14:27 +01:00
Code Issues Releases Wiki Activity

common: add --qemu option to override configs

Browse Source
This commit is contained in:
Ciro Santilli 六四事件 法轮功
2018-11-13 00:00:01 +00:00
parent e54635c3ab
commit 238f1ad1c0
8 changed files with 305 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build
View File

@@ -84,7 +84,7 @@ buildroot_component = Component(
name_to_component_map = { name_to_component_map = {
# Leaves without dependencies. # Leaves without dependencies.
'baremetal-qemu': Component( 'baremetal-qemu': Component(
lambda arch: run_cmd(['build-baremetal'], arch), lambda arch: run_cmd(['build-baremetal', '--qemu'], arch),
supported_archs=common.crosstool_ng_supported_archs, supported_archs=common.crosstool_ng_supported_archs,
), ),
'baremetal-gem5': Component( 'baremetal-gem5': Component(

2
build-baremetal
View File

@@ -22,7 +22,7 @@ class BaremetalComponent(common.Component):
else: else:
os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH'] os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH']
gcc = common.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng']) gcc = common.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng'])
if args.gem5: if common.emulator == 'gem5':
if common.machine == 'VExpress_GEM5_V1': if common.machine == 'VExpress_GEM5_V1':
entry_address = 0x80000000 entry_address = 0x80000000
uart_address = 0x1c090000 uart_address = 0x1c090000

2
build-buildroot
View File

@@ -71,7 +71,7 @@ usually extra Buildroot targets.
extra_make_args = common.add_newlines(args.extra_make_args) extra_make_args = common.add_newlines(args.extra_make_args)
if args.build_linux: if args.build_linux:
extra_make_args.extend(['linux-reconfigure', common.Newline]) extra_make_args.extend(['linux-reconfigure', common.Newline])
if args.gem5: if common.emulator == 'gem5':
extra_make_args.extend(['gem5-reconfigure', common.Newline]) extra_make_args.extend(['gem5-reconfigure', common.Newline])
if args.arch == 'x86_64': if args.arch == 'x86_64':
defconfig = 'qemu_x86_64_defconfig' defconfig = 'qemu_x86_64_defconfig'

565
common.py
View File

@@ -23,7 +23,7 @@ import time
import urllib import urllib
import urllib.request import urllib.request
this_module = sys.modules[__name__] common = sys.modules[__name__]
repo_short_id = 'lkmc' repo_short_id = 'lkmc'
# https://stackoverflow.com/questions/20010199/how-to-determine-if-a-process-runs-inside-lxc-docker # https://stackoverflow.com/questions/20010199/how-to-determine-if-a-process-runs-inside-lxc-docker
in_docker = os.path.exists('/.dockerenv') in_docker = os.path.exists('/.dockerenv')
@@ -38,19 +38,19 @@ else:
bench_boot = os.path.join(out_dir, 'bench-boot.txt') bench_boot = os.path.join(out_dir, 'bench-boot.txt')
packages_dir = os.path.join(root_dir, 'buildroot_packages') packages_dir = os.path.join(root_dir, 'buildroot_packages')
kernel_modules_subdir = 'kernel_modules' kernel_modules_subdir = 'kernel_modules'
kernel_modules_src_dir = os.path.join(this_module.root_dir, this_module.kernel_modules_subdir) kernel_modules_src_dir = os.path.join(common.root_dir, common.kernel_modules_subdir)
userland_subdir = 'userland' userland_subdir = 'userland'
userland_src_dir = os.path.join(this_module.root_dir, this_module.userland_subdir) userland_src_dir = os.path.join(common.root_dir, common.userland_subdir)
userland_build_ext = '.out' userland_build_ext = '.out'
include_subdir = 'include' include_subdir = 'include'
include_src_dir = os.path.join(this_module.root_dir, this_module.include_subdir) include_src_dir = os.path.join(common.root_dir, common.include_subdir)
submodules_dir = os.path.join(root_dir, 'submodules') submodules_dir = os.path.join(root_dir, 'submodules')
buildroot_src_dir = os.path.join(submodules_dir, 'buildroot') buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng') crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng')
crosstool_ng_supported_archs = set(['arm', 'aarch64']) crosstool_ng_supported_archs = set(['arm', 'aarch64'])
linux_src_dir = os.path.join(submodules_dir, 'linux') linux_src_dir = os.path.join(submodules_dir, 'linux')
linux_config_dir = os.path.join(this_module.root_dir, 'linux_config') linux_config_dir = os.path.join(common.root_dir, 'linux_config')
rootfs_overlay_dir = os.path.join(this_module.root_dir, 'rootfs_overlay') rootfs_overlay_dir = os.path.join(common.root_dir, 'rootfs_overlay')
extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux') extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux')
qemu_src_dir = os.path.join(submodules_dir, 'qemu') qemu_src_dir = os.path.join(submodules_dir, 'qemu')
parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark') parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark')
@@ -63,14 +63,14 @@ arch_short_to_long_dict = collections.OrderedDict([
]) ])
all_archs = [arch_short_to_long_dict[k] for k in arch_short_to_long_dict] all_archs = [arch_short_to_long_dict[k] for k in arch_short_to_long_dict]
arch_choices = [] arch_choices = []
for key in this_module.arch_short_to_long_dict: for key in common.arch_short_to_long_dict:
arch_choices.append(key) arch_choices.append(key)
arch_choices.append(this_module.arch_short_to_long_dict[key]) arch_choices.append(common.arch_short_to_long_dict[key])
default_arch = 'x86_64' default_arch = 'x86_64'
gem5_cpt_prefix = '^cpt\.' gem5_cpt_prefix = '^cpt\.'
sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip() sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
release_dir = os.path.join(this_module.out_dir, 'release') release_dir = os.path.join(common.out_dir, 'release')
release_zip_file = os.path.join(this_module.release_dir, 'lkmc-{}.zip'.format(this_module.sha)) release_zip_file = os.path.join(common.release_dir, 'lkmc-{}.zip'.format(common.sha))
github_repo_id = 'cirosantilli/linux-kernel-module-cheat' github_repo_id = 'cirosantilli/linux-kernel-module-cheat'
asm_ext = '.S' asm_ext = '.S'
c_ext = '.c' c_ext = '.c'
@@ -94,7 +94,7 @@ class Component:
The actual build work is done by do_build in implementing classes. The actual build work is done by do_build in implementing classes.
''' '''
parser = this_module.get_argparse( parser = common.get_argparse(
argparse_args=self.get_argparse_args(), argparse_args=self.get_argparse_args(),
default_args=self.get_default_args(), default_args=self.get_default_args(),
) )
@@ -110,16 +110,16 @@ class Component:
type=int, type=int,
default=multiprocessing.cpu_count(), default=multiprocessing.cpu_count(),
) )
args = this_module.setup(parser) args = common.setup(parser)
if not this_module.dry_run: if not common.dry_run:
start_time = time.time() start_time = time.time()
if args.clean: if args.clean:
self.clean(args) self.clean(args)
else: else:
self.do_build(args) self.do_build(args)
if not this_module.dry_run: if not common.dry_run:
end_time = time.time() end_time = time.time()
this_module.print_time(end_time - start_time) common.print_time(end_time - start_time)
def add_parser_arguments(self, parser): def add_parser_arguments(self, parser):
pass pass
@@ -127,7 +127,7 @@ class Component:
def clean(self, args): def clean(self, args):
build_dir = self.get_build_dir(args) build_dir = self.get_build_dir(args)
if build_dir is not None: if build_dir is not None:
this_module.rmrf(build_dir) common.rmrf(build_dir)
def do_build(self, args): def do_build(self, args):
''' '''
@@ -173,24 +173,23 @@ mkdir are generally omitted since those are obvious.
def add_newlines(cmd): def add_newlines(cmd):
out = [] out = []
for arg in cmd: for arg in cmd:
out.extend([arg, this_module.Newline]) out.extend([arg, common.Newline])
return out return out
def assert_crosstool_ng_supports_arch(arch): def assert_crosstool_ng_supports_arch(arch):
if arch not in this_module.crosstool_ng_supported_archs: if arch not in common.crosstool_ng_supported_archs:
raise Exception('arch not yet supported: ' + arch) raise Exception('arch not yet supported: ' + arch)
def base64_encode(string): def base64_encode(string):
return base64.b64encode(string.encode()).decode() return base64.b64encode(string.encode()).decode()
def write_string_to_file(path, string, mode='w'): def write_string_to_file(path, string, mode='w'):
global this_module
if mode == 'a': if mode == 'a':
redirect = '>>' redirect = '>>'
else: else:
redirect = '>' redirect = '>'
print_cmd("cat << 'EOF' {} {}\n{}\nEOF".format(redirect, path, string)) print_cmd("cat << 'EOF' {} {}\n{}\nEOF".format(redirect, path, string))
if not this_module.dry_run: if not common.dry_run:
with open(path, 'a') as f: with open(path, 'a') as f:
f.write(string) f.write(string)
@@ -212,10 +211,9 @@ def gem_list_checkpoint_dirs():
''' '''
List checkpoint directory, oldest first. List checkpoint directory, oldest first.
''' '''
global this_module prefix_re = re.compile(common.gem5_cpt_prefix)
prefix_re = re.compile(this_module.gem5_cpt_prefix) files = list(filter(lambda x: os.path.isdir(os.path.join(common.m5out_dir, x)) and prefix_re.search(x), os.listdir(common.m5out_dir)))
files = list(filter(lambda x: os.path.isdir(os.path.join(this_module.m5out_dir, x)) and prefix_re.search(x), os.listdir(this_module.m5out_dir))) files.sort(key=lambda x: os.path.getmtime(os.path.join(common.m5out_dir, x)))
files.sort(key=lambda x: os.path.getmtime(os.path.join(this_module.m5out_dir, x)))
return files return files
def get_argparse(default_args=None, argparse_args=None): def get_argparse(default_args=None, argparse_args=None):
@@ -225,7 +223,6 @@ def get_argparse(default_args=None, argparse_args=None):
:type default_args: Dict[str,str] :type default_args: Dict[str,str]
:type argparse_args: Dict :type argparse_args: Dict
''' '''
global this_module
if default_args is None: if default_args is None:
default_args = {} default_args = {}
if argparse_args is None: if argparse_args is None:
@@ -234,9 +231,10 @@ def get_argparse(default_args=None, argparse_args=None):
formatter_class=argparse.RawTextHelpFormatter, formatter_class=argparse.RawTextHelpFormatter,
**argparse_args **argparse_args
) )
this_module.add_dry_run_argument(parser) common.add_dry_run_argument(parser)
emulator_group = parser.add_mutually_exclusive_group(required=False)
parser.add_argument( parser.add_argument(
'-a', '--arch', choices=this_module.arch_choices, default=this_module.default_arch, '-a', '--arch', choices=common.arch_choices, default=common.default_arch,
help='CPU architecture. Default: %(default)s' help='CPU architecture. Default: %(default)s'
) )
parser.add_argument( parser.add_argument(
@@ -267,16 +265,6 @@ inside baremetal/ and then try to use corresponding executable.
'--docker', default=False, action='store_true', '--docker', default=False, action='store_true',
help='''\ help='''\
Use the docker download Ubuntu root filesystem instead of the default Buildroot one. Use the docker download Ubuntu root filesystem instead of the default Buildroot one.
'''
)
parser.add_argument(
'-g', '--gem5', default=False, action='store_true',
help='Use gem5 instead of QEMU'
)
parser.add_argument(
'--gem5-src',
help='''\
Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`.
''' '''
) )
parser.add_argument( parser.add_argument(
@@ -291,18 +279,9 @@ gem5 default: VExpress_GEM5_V1
See the documentation for other values known to work. See the documentation for other values known to work.
''' '''
) )
parser.add_argument( emulator_group.add_argument(
'-M', '--gem5-build-id', '-g', '--gem5', default=False, action='store_true',
help='''\ help='Use gem5 instead of QEMU. Default: False'
gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: {}
'''.format(default_build_id)
)
parser.add_argument(
'-N', '--gem5-worktree',
help='''\
Create and use a git worktree of the gem5 submodule.
See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree
'''
) )
parser.add_argument( parser.add_argument(
'--gem5-build-dir', '--gem5-build-dir',
@@ -310,10 +289,27 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree
Use the given directory as the gem5 build directory. Use the given directory as the gem5 build directory.
''' '''
) )
parser.add_argument(
'-M', '--gem5-build-id',
help='''\
gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: {}
'''.format(default_build_id)
)
parser.add_argument(
'--gem5-build-type', default='opt',
help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
)
parser.add_argument( parser.add_argument(
'--gem5-source-dir', '--gem5-source-dir',
help='''\ help='''\
Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`. Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`.
'''
)
parser.add_argument(
'-N', '--gem5-worktree',
help='''\
Create and use a git worktree of the gem5 submodule.
See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree
''' '''
) )
parser.add_argument( parser.add_argument(
@@ -338,16 +334,19 @@ the likelihood of incompatibilities.
Increase the ports to be used such as for GDB by an offset to run multiple Increase the ports to be used such as for GDB by an offset to run multiple
instances in parallel. instances in parallel.
Default: the run ID (-n) if that is an integer, otherwise 0. Default: the run ID (-n) if that is an integer, otherwise 0.
'''
)
emulator_group.add_argument(
'--qemu', default=False, action='store_true',
help='''\
Use QEMU as the emulator. This option exists in addition to --gem5
to allow overriding configs from the CLI.
''' '''
) )
parser.add_argument( parser.add_argument(
'-Q', '--qemu-build-id', default=default_build_id, '-Q', '--qemu-build-id', default=default_build_id,
help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s' help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
) )
parser.add_argument(
'--gem5-build-type', default='opt',
help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
)
parser.add_argument( parser.add_argument(
'--userland-build-id', default=None '--userland-build-id', default=None
) )
@@ -355,8 +354,8 @@ Default: the run ID (-n) if that is an integer, otherwise 0.
'-v', '--verbose', default=False, action='store_true', '-v', '--verbose', default=False, action='store_true',
help='Show full compilation commands when they are not shown by default.' help='Show full compilation commands when they are not shown by default.'
) )
if hasattr(this_module, 'configs'): if hasattr(common, 'configs'):
defaults = this_module.configs.copy() defaults = common.configs.copy()
else: else:
defaults = {} defaults = {}
defaults.update(default_args) defaults.update(default_args)
@@ -368,9 +367,8 @@ Default: the run ID (-n) if that is an integer, otherwise 0.
return parser return parser
def get_elf_entry(elf_file_path): def get_elf_entry(elf_file_path):
global this_module
readelf_header = subprocess.check_output([ readelf_header = subprocess.check_output([
this_module.get_toolchain_tool('readelf'), common.get_toolchain_tool('readelf'),
'-h', '-h',
elf_file_path elf_file_path
]) ])
@@ -382,11 +380,10 @@ def get_elf_entry(elf_file_path):
return int(addr, 0) return int(addr, 0)
def get_stats(stat_re=None, stats_file=None): def get_stats(stat_re=None, stats_file=None):
global this_module
if stat_re is None: if stat_re is None:
stat_re = '^system.cpu[0-9]*.numCycles$' stat_re = '^system.cpu[0-9]*.numCycles$'
if stats_file is None: if stats_file is None:
stats_file = this_module.stats_file stats_file = common.stats_file
stat_re = re.compile(stat_re) stat_re = re.compile(stat_re)
ret = [] ret = []
with open(stats_file, 'r') as statfile: with open(stats_file, 'r') as statfile:
@@ -398,11 +395,11 @@ def get_stats(stat_re=None, stats_file=None):
return ret return ret
def get_toolchain_prefix(tool, allowed_toolchains=None): def get_toolchain_prefix(tool, allowed_toolchains=None):
buildroot_full_prefix = os.path.join(this_module.host_bin_dir, this_module.buildroot_toolchain_prefix) buildroot_full_prefix = os.path.join(common.host_bin_dir, common.buildroot_toolchain_prefix)
buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool)) buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
crosstool_ng_full_prefix = os.path.join(this_module.crosstool_ng_bin_dir, this_module.crosstool_ng_toolchain_prefix) crosstool_ng_full_prefix = os.path.join(common.crosstool_ng_bin_dir, common.crosstool_ng_toolchain_prefix)
crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool)) crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
host_tool = '{}-{}'.format(this_module.ubuntu_toolchain_prefix, tool) host_tool = '{}-{}'.format(common.ubuntu_toolchain_prefix, tool)
host_path = shutil.which(host_tool) host_path = shutil.which(host_tool)
if host_path is not None: if host_path is not None:
host_exists = True host_exists = True
@@ -416,7 +413,7 @@ def get_toolchain_prefix(tool, allowed_toolchains=None):
'host': (host_exists, host_full_prefix), 'host': (host_exists, host_full_prefix),
} }
if allowed_toolchains is None: if allowed_toolchains is None:
if this_module.baremetal is None: if common.baremetal is None:
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
else: else:
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
@@ -429,7 +426,7 @@ def get_toolchain_prefix(tool, allowed_toolchains=None):
raise Exception('Tool not found. Tried:\n' + '\n'.join(tried)) raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))
def get_toolchain_tool(tool, allowed_toolchains=None): def get_toolchain_tool(tool, allowed_toolchains=None):
return '{}-{}'.format(this_module.get_toolchain_prefix(tool, allowed_toolchains), tool) return '{}-{}'.format(common.get_toolchain_prefix(tool, allowed_toolchains), tool)
def github_make_request( def github_make_request(
authenticate=False, authenticate=False,
@@ -440,7 +437,6 @@ def github_make_request(
url_params=None, url_params=None,
**extra_request_args **extra_request_args
): ):
global this_module
if extra_headers is None: if extra_headers is None:
extra_headers = {} extra_headers = {}
headers = {'Accept': 'application/vnd.github.v3+json'} headers = {'Accept': 'application/vnd.github.v3+json'}
@@ -466,20 +462,18 @@ def log_error(msg):
print('error: {}'.format(msg), file=sys.stderr) print('error: {}'.format(msg), file=sys.stderr)
def make_build_dirs(): def make_build_dirs():
global this_module os.makedirs(common.buildroot_build_build_dir, exist_ok=True)
os.makedirs(this_module.buildroot_build_build_dir, exist_ok=True) os.makedirs(common.gem5_build_dir, exist_ok=True)
os.makedirs(this_module.gem5_build_dir, exist_ok=True) os.makedirs(common.out_rootfs_overlay_dir, exist_ok=True)
os.makedirs(this_module.out_rootfs_overlay_dir, exist_ok=True)
def make_run_dirs(): def make_run_dirs():
''' '''
Make directories required for the run. Make directories required for the run.
The user could nuke those anytime between runs to try and clean things up. The user could nuke those anytime between runs to try and clean things up.
''' '''
global this_module os.makedirs(common.gem5_run_dir, exist_ok=True)
os.makedirs(this_module.gem5_run_dir, exist_ok=True) os.makedirs(common.p9_dir, exist_ok=True)
os.makedirs(this_module.p9_dir, exist_ok=True) os.makedirs(common.qemu_run_dir, exist_ok=True)
os.makedirs(this_module.qemu_run_dir, exist_ok=True)
def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None): def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None):
''' '''
@@ -500,15 +494,15 @@ def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None):
cmd_quote = [] cmd_quote = []
newline_count = 0 newline_count = 0
for arg in cmd: for arg in cmd:
if arg == this_module.Newline: if arg == common.Newline:
cmd_quote.append(arg) cmd_quote.append(arg)
newline_count += 1 newline_count += 1
else: else:
cmd_quote.append(shlex.quote(arg)) cmd_quote.append(shlex.quote(arg))
if newline_count > 0: if newline_count > 0:
cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == this_module.Newline) if not x] cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == common.Newline) if not x]
out.extend(cmd_quote) out.extend(cmd_quote)
if newline_count == 1 and cmd[-1] == this_module.Newline: if newline_count == 1 and cmd[-1] == common.Newline:
ending = '' ending = ''
else: else:
ending = last_newline + ';' ending = last_newline + ';'
@@ -529,7 +523,7 @@ def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
cmd_string = cmd cmd_string = cmd
else: else:
cmd_string = cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths) cmd_string = cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths)
print(this_module.command_prefix + cmd_string) print(common.command_prefix + cmd_string)
if cmd_file is not None: if cmd_file is not None:
with open(cmd_file, 'w') as f: with open(cmd_file, 'w') as f:
f.write('#!/usr/bin/env bash\n') f.write('#!/usr/bin/env bash\n')
@@ -543,15 +537,14 @@ def print_time(ellapsed_seconds):
print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))
def raw_to_qcow2(prebuilt=False, reverse=False): def raw_to_qcow2(prebuilt=False, reverse=False):
global this_module
if prebuilt: if prebuilt:
qemu_img_executable = this_module.qemu_img_basename qemu_img_executable = common.qemu_img_basename
else: else:
qemu_img_executable = this_module.qemu_img_executable qemu_img_executable = common.qemu_img_executable
infmt = 'raw' infmt = 'raw'
outfmt = 'qcow2' outfmt = 'qcow2'
infile = this_module.rootfs_raw_file infile = common.rootfs_raw_file
outfile = this_module.qcow2_file outfile = common.qcow2_file
if reverse: if reverse:
tmp = infmt tmp = infmt
infmt = outfmt infmt = outfmt
@@ -559,15 +552,15 @@ def raw_to_qcow2(prebuilt=False, reverse=False):
tmp = infile tmp = infile
infile = outfile infile = outfile
outfile = tmp outfile = tmp
this_module.run_cmd([ common.run_cmd([
qemu_img_executable, this_module.Newline, qemu_img_executable, common.Newline,
# Prevent qemu-img from generating trace files like QEMU. Disgusting. # Prevent qemu-img from generating trace files like QEMU. Disgusting.
'-T', 'pr_manager_run,file=/dev/null', this_module.Newline, '-T', 'pr_manager_run,file=/dev/null', common.Newline,
'convert', this_module.Newline, 'convert', common.Newline,
'-f', infmt, this_module.Newline, '-f', infmt, common.Newline,
'-O', outfmt, this_module.Newline, '-O', outfmt, common.Newline,
infile, this_module.Newline, infile, common.Newline,
outfile, this_module.Newline, outfile, common.Newline,
]) ])
def resolve_args(defaults, args, extra_args): def resolve_args(defaults, args, extra_args):
@@ -578,14 +571,13 @@ def resolve_args(defaults, args, extra_args):
return argcopy return argcopy
def cp(src, dest): def cp(src, dest):
global this_module
print_cmd(['cp', src, dest]) print_cmd(['cp', src, dest])
if not this_module.dry_run: if not common.dry_run:
shutil.copy2(src, dest) shutil.copy2(src, dest)
def rmrf(path): def rmrf(path):
print_cmd(['rm', '-r', '-f', path]) print_cmd(['rm', '-r', '-f', path])
if not this_module.dry_run and os.path.exists(path): if not common.dry_run and os.path.exists(path):
shutil.rmtree(path) shutil.rmtree(path)
def run_cmd( def run_cmd(
@@ -668,8 +660,8 @@ def run_cmd(
#sigpipe_old = signal.getsignal(signal.SIGPIPE) #sigpipe_old = signal.getsignal(signal.SIGPIPE)
#signal.signal(signal.SIGPIPE, signal.SIG_DFL) #signal.signal(signal.SIGPIPE, signal.SIG_DFL)
cmd = this_module.strip_newlines(cmd) cmd = common.strip_newlines(cmd)
if not dry_run and not this_module.dry_run: if not dry_run and not common.dry_run:
# https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
if out_file is not None: if out_file is not None:
@@ -702,10 +694,13 @@ def setup(parser):
Parse the command line arguments, and setup several variables based on them. Parse the command line arguments, and setup several variables based on them.
Typically done after getting inputs from the command line arguments. Typically done after getting inputs from the command line arguments.
''' '''
global this_module
args = parser.parse_args() args = parser.parse_args()
if args.arch in this_module.arch_short_to_long_dict: if args.qemu or not args.gem5:
args.arch = this_module.arch_short_to_long_dict[args.arch] common.emulator = 'qemu'
else:
common.emulator = 'gem5'
if args.arch in common.arch_short_to_long_dict:
args.arch = common.arch_short_to_long_dict[args.arch]
if args.gem5_build_id is None: if args.gem5_build_id is None:
args.gem5_build_id = default_build_id args.gem5_build_id = default_build_id
gem5_build_id_given = False gem5_build_id_given = False
@@ -713,166 +708,166 @@ def setup(parser):
gem5_build_id_given = True gem5_build_id_given = True
if args.userland_build_id is None: if args.userland_build_id is None:
args.userland_build_id = default_build_id args.userland_build_id = default_build_id
this_module.userland_build_id_given = False common.userland_build_id_given = False
else: else:
this_module.userland_build_id_given = True common.userland_build_id_given = True
if args.gem5_worktree is not None and not gem5_build_id_given: if args.gem5_worktree is not None and not gem5_build_id_given:
args.gem5_build_id = args.gem5_worktree args.gem5_build_id = args.gem5_worktree
this_module.machine = args.machine common.machine = args.machine
this_module.setup_dry_run_arguments(args) common.setup_dry_run_arguments(args)
this_module.is_arm = False common.is_arm = False
if args.arch == 'arm': if args.arch == 'arm':
this_module.armv = 7 common.armv = 7
this_module.gem5_arch = 'ARM' common.gem5_arch = 'ARM'
this_module.mcpu = 'cortex-a15' common.mcpu = 'cortex-a15'
this_module.buildroot_toolchain_prefix = 'arm-buildroot-linux-uclibcgnueabihf' common.buildroot_toolchain_prefix = 'arm-buildroot-linux-uclibcgnueabihf'
this_module.crosstool_ng_toolchain_prefix = 'arm-unknown-eabi' common.crosstool_ng_toolchain_prefix = 'arm-unknown-eabi'
this_module.ubuntu_toolchain_prefix = 'arm-linux-gnueabihf' common.ubuntu_toolchain_prefix = 'arm-linux-gnueabihf'
if args.gem5: if common.emulator == 'gem5':
if this_module.machine is None: if common.machine is None:
this_module.machine = 'VExpress_GEM5_V1' common.machine = 'VExpress_GEM5_V1'
else: else:
if this_module.machine is None: if common.machine is None:
this_module.machine = 'virt' common.machine = 'virt'
this_module.is_arm = True common.is_arm = True
elif args.arch == 'aarch64': elif args.arch == 'aarch64':
this_module.armv = 8 common.armv = 8
this_module.gem5_arch = 'ARM' common.gem5_arch = 'ARM'
this_module.mcpu = 'cortex-a57' common.mcpu = 'cortex-a57'
this_module.buildroot_toolchain_prefix = 'aarch64-buildroot-linux-uclibc' common.buildroot_toolchain_prefix = 'aarch64-buildroot-linux-uclibc'
this_module.crosstool_ng_toolchain_prefix = 'aarch64-unknown-elf' common.crosstool_ng_toolchain_prefix = 'aarch64-unknown-elf'
this_module.ubuntu_toolchain_prefix = 'aarch64-linux-gnu' common.ubuntu_toolchain_prefix = 'aarch64-linux-gnu'
if args.gem5: if common.emulator == 'gem5':
if this_module.machine is None: if common.machine is None:
this_module.machine = 'VExpress_GEM5_V1' common.machine = 'VExpress_GEM5_V1'
else: else:
if this_module.machine is None: if common.machine is None:
this_module.machine = 'virt' common.machine = 'virt'
this_module.is_arm = True common.is_arm = True
elif args.arch == 'x86_64': elif args.arch == 'x86_64':
this_module.crosstool_ng_toolchain_prefix = 'x86_64-unknown-elf' common.crosstool_ng_toolchain_prefix = 'x86_64-unknown-elf'
this_module.gem5_arch = 'X86' common.gem5_arch = 'X86'
this_module.buildroot_toolchain_prefix = 'x86_64-buildroot-linux-uclibc' common.buildroot_toolchain_prefix = 'x86_64-buildroot-linux-uclibc'
this_module.ubuntu_toolchain_prefix = 'x86_64-linux-gnu' common.ubuntu_toolchain_prefix = 'x86_64-linux-gnu'
if args.gem5: if common.emulator == 'gem5':
if this_module.machine is None: if common.machine is None:
this_module.machine = 'TODO' common.machine = 'TODO'
else: else:
if this_module.machine is None: if common.machine is None:
this_module.machine = 'pc' common.machine = 'pc'
this_module.buildroot_out_dir = os.path.join(this_module.out_dir, 'buildroot') common.buildroot_out_dir = os.path.join(common.out_dir, 'buildroot')
this_module.buildroot_build_dir = os.path.join(this_module.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch) common.buildroot_build_dir = os.path.join(common.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch)
this_module.buildroot_download_dir = os.path.join(this_module.buildroot_out_dir, 'download') common.buildroot_download_dir = os.path.join(common.buildroot_out_dir, 'download')
this_module.buildroot_config_file = os.path.join(this_module.buildroot_build_dir, '.config') common.buildroot_config_file = os.path.join(common.buildroot_build_dir, '.config')
this_module.buildroot_build_build_dir = os.path.join(this_module.buildroot_build_dir, 'build') common.buildroot_build_build_dir = os.path.join(common.buildroot_build_dir, 'build')
this_module.buildroot_linux_build_dir = os.path.join(this_module.buildroot_build_build_dir, 'linux-custom') common.buildroot_linux_build_dir = os.path.join(common.buildroot_build_build_dir, 'linux-custom')
this_module.buildroot_vmlinux = os.path.join(this_module.buildroot_linux_build_dir, "vmlinux") common.buildroot_vmlinux = os.path.join(common.buildroot_linux_build_dir, "vmlinux")
this_module.qemu_build_dir = os.path.join(this_module.out_dir, 'qemu', args.qemu_build_id) common.qemu_build_dir = os.path.join(common.out_dir, 'qemu', args.qemu_build_id)
this_module.qemu_executable_basename = 'qemu-system-{}'.format(args.arch) common.qemu_executable_basename = 'qemu-system-{}'.format(args.arch)
this_module.qemu_executable = os.path.join(this_module.qemu_build_dir, '{}-softmmu'.format(args.arch), this_module.qemu_executable_basename) common.qemu_executable = os.path.join(common.qemu_build_dir, '{}-softmmu'.format(args.arch), common.qemu_executable_basename)
this_module.qemu_img_basename = 'qemu-img' common.qemu_img_basename = 'qemu-img'
this_module.qemu_img_executable = os.path.join(this_module.qemu_build_dir, this_module.qemu_img_basename) common.qemu_img_executable = os.path.join(common.qemu_build_dir, common.qemu_img_basename)
this_module.host_dir = os.path.join(this_module.buildroot_build_dir, 'host') common.host_dir = os.path.join(common.buildroot_build_dir, 'host')
this_module.host_bin_dir = os.path.join(this_module.host_dir, 'usr', 'bin') common.host_bin_dir = os.path.join(common.host_dir, 'usr', 'bin')
this_module.buildroot_pkg_config = os.path.join(this_module.host_bin_dir, 'pkg-config') common.buildroot_pkg_config = os.path.join(common.host_bin_dir, 'pkg-config')
this_module.buildroot_images_dir = os.path.join(this_module.buildroot_build_dir, 'images') common.buildroot_images_dir = os.path.join(common.buildroot_build_dir, 'images')
this_module.buildroot_rootfs_raw_file = os.path.join(this_module.buildroot_images_dir, 'rootfs.ext2') common.buildroot_rootfs_raw_file = os.path.join(common.buildroot_images_dir, 'rootfs.ext2')
this_module.buildroot_qcow2_file = this_module.buildroot_rootfs_raw_file + '.qcow2' common.buildroot_qcow2_file = common.buildroot_rootfs_raw_file + '.qcow2'
this_module.staging_dir = os.path.join(this_module.out_dir, 'staging', args.arch) common.staging_dir = os.path.join(common.out_dir, 'staging', args.arch)
this_module.buildroot_staging_dir = os.path.join(this_module.buildroot_build_dir, 'staging') common.buildroot_staging_dir = os.path.join(common.buildroot_build_dir, 'staging')
this_module.target_dir = os.path.join(this_module.buildroot_build_dir, 'target') common.target_dir = os.path.join(common.buildroot_build_dir, 'target')
this_module.run_dir_base = os.path.join(this_module.out_dir, 'run') common.run_dir_base = os.path.join(common.out_dir, 'run')
this_module.gem5_run_dir = os.path.join(this_module.run_dir_base, 'gem5', args.arch, str(args.run_id)) common.gem5_run_dir = os.path.join(common.run_dir_base, 'gem5', args.arch, str(args.run_id))
this_module.m5out_dir = os.path.join(this_module.gem5_run_dir, 'm5out') common.m5out_dir = os.path.join(common.gem5_run_dir, 'm5out')
this_module.stats_file = os.path.join(this_module.m5out_dir, 'stats.txt') common.stats_file = os.path.join(common.m5out_dir, 'stats.txt')
this_module.trace_txt_file = os.path.join(this_module.m5out_dir, 'trace.txt') common.trace_txt_file = os.path.join(common.m5out_dir, 'trace.txt')
this_module.gem5_guest_terminal_file = os.path.join(this_module.m5out_dir, 'system.terminal') common.gem5_guest_terminal_file = os.path.join(common.m5out_dir, 'system.terminal')
this_module.gem5_readfile = os.path.join(this_module.gem5_run_dir, 'readfile') common.gem5_readfile = os.path.join(common.gem5_run_dir, 'readfile')
this_module.gem5_termout_file = os.path.join(this_module.gem5_run_dir, 'termout.txt') common.gem5_termout_file = os.path.join(common.gem5_run_dir, 'termout.txt')
this_module.qemu_run_dir = os.path.join(this_module.run_dir_base, 'qemu', args.arch, str(args.run_id)) common.qemu_run_dir = os.path.join(common.run_dir_base, 'qemu', args.arch, str(args.run_id))
this_module.qemu_trace_basename = 'trace.bin' common.qemu_trace_basename = 'trace.bin'
this_module.qemu_trace_file = os.path.join(this_module.qemu_run_dir, 'trace.bin') common.qemu_trace_file = os.path.join(common.qemu_run_dir, 'trace.bin')
this_module.qemu_trace_txt_file = os.path.join(this_module.qemu_run_dir, 'trace.txt') common.qemu_trace_txt_file = os.path.join(common.qemu_run_dir, 'trace.txt')
this_module.qemu_termout_file = os.path.join(this_module.qemu_run_dir, 'termout.txt') common.qemu_termout_file = os.path.join(common.qemu_run_dir, 'termout.txt')
this_module.qemu_rrfile = os.path.join(this_module.qemu_run_dir, 'rrfile') common.qemu_rrfile = os.path.join(common.qemu_run_dir, 'rrfile')
this_module.qemu_guest_terminal_file = os.path.join(this_module.m5out_dir, qemu_termout_file) common.qemu_guest_terminal_file = os.path.join(common.m5out_dir, qemu_termout_file)
this_module.gem5_out_dir = os.path.join(this_module.out_dir, 'gem5') common.gem5_out_dir = os.path.join(common.out_dir, 'gem5')
if args.gem5_build_dir is None: if args.gem5_build_dir is None:
this_module.gem5_build_dir = os.path.join(this_module.gem5_out_dir, args.gem5_build_id, args.gem5_build_type) common.gem5_build_dir = os.path.join(common.gem5_out_dir, args.gem5_build_id, args.gem5_build_type)
else: else:
this_module.gem5_build_dir = args.gem5_build_dir common.gem5_build_dir = args.gem5_build_dir
this_module.gem5_fake_iso = os.path.join(this_module.gem5_out_dir, 'fake.iso') common.gem5_fake_iso = os.path.join(common.gem5_out_dir, 'fake.iso')
this_module.gem5_m5term = os.path.join(this_module.gem5_build_dir, 'm5term') common.gem5_m5term = os.path.join(common.gem5_build_dir, 'm5term')
this_module.gem5_build_build_dir = os.path.join(this_module.gem5_build_dir, 'build') common.gem5_build_build_dir = os.path.join(common.gem5_build_dir, 'build')
this_module.gem5_executable = os.path.join(this_module.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type)) common.gem5_executable = os.path.join(common.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type))
this_module.gem5_system_dir = os.path.join(this_module.gem5_build_dir, 'system') common.gem5_system_dir = os.path.join(common.gem5_build_dir, 'system')
this_module.crosstool_ng_out_dir = os.path.join(this_module.out_dir, 'crosstool-ng') common.crosstool_ng_out_dir = os.path.join(common.out_dir, 'crosstool-ng')
this_module.crosstool_ng_buildid_dir = os.path.join(this_module.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id) common.crosstool_ng_buildid_dir = os.path.join(common.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id)
this_module.crosstool_ng_install_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'install', args.arch) common.crosstool_ng_install_dir = os.path.join(common.crosstool_ng_buildid_dir, 'install', args.arch)
this_module.crosstool_ng_bin_dir = os.path.join(this_module.crosstool_ng_install_dir, 'bin') common.crosstool_ng_bin_dir = os.path.join(common.crosstool_ng_install_dir, 'bin')
this_module.crosstool_ng_util_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'util') common.crosstool_ng_util_dir = os.path.join(common.crosstool_ng_buildid_dir, 'util')
this_module.crosstool_ng_config = os.path.join(this_module.crosstool_ng_util_dir, '.config') common.crosstool_ng_config = os.path.join(common.crosstool_ng_util_dir, '.config')
this_module.crosstool_ng_defconfig = os.path.join(this_module.crosstool_ng_util_dir, 'defconfig') common.crosstool_ng_defconfig = os.path.join(common.crosstool_ng_util_dir, 'defconfig')
this_module.crosstool_ng_executable = os.path.join(this_module.crosstool_ng_util_dir, 'ct-ng') common.crosstool_ng_executable = os.path.join(common.crosstool_ng_util_dir, 'ct-ng')
this_module.crosstool_ng_build_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'build') common.crosstool_ng_build_dir = os.path.join(common.crosstool_ng_buildid_dir, 'build')
this_module.crosstool_ng_download_dir = os.path.join(this_module.crosstool_ng_out_dir, 'download') common.crosstool_ng_download_dir = os.path.join(common.crosstool_ng_out_dir, 'download')
this_module.gem5_default_src_dir = os.path.join(submodules_dir, 'gem5') common.gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
if args.gem5_source_dir is not None: if args.gem5_source_dir is not None:
this_module.gem5_src_dir = args.gem5_source_dir common.gem5_src_dir = args.gem5_source_dir
assert(os.path.exists(args.gem5_source_dir)) assert(os.path.exists(args.gem5_source_dir))
else: else:
if args.gem5_worktree is not None: if args.gem5_worktree is not None:
this_module.gem5_src_dir = os.path.join(this_module.gem5_non_default_src_root_dir, args.gem5_worktree) common.gem5_src_dir = os.path.join(common.gem5_non_default_src_root_dir, args.gem5_worktree)
else: else:
this_module.gem5_src_dir = this_module.gem5_default_src_dir common.gem5_src_dir = common.gem5_default_src_dir
this_module.gem5_m5_src_dir = os.path.join(this_module.gem5_src_dir, 'util', 'm5') common.gem5_m5_src_dir = os.path.join(common.gem5_src_dir, 'util', 'm5')
this_module.gem5_m5_build_dir = os.path.join(this_module.out_dir, 'util', 'm5') common.gem5_m5_build_dir = os.path.join(common.out_dir, 'util', 'm5')
if args.gem5: if common.emulator == 'gem5':
this_module.executable = this_module.gem5_executable common.executable = common.gem5_executable
this_module.run_dir = this_module.gem5_run_dir common.run_dir = common.gem5_run_dir
this_module.termout_file = this_module.gem5_termout_file common.termout_file = common.gem5_termout_file
this_module.guest_terminal_file = gem5_guest_terminal_file common.guest_terminal_file = gem5_guest_terminal_file
else: else:
this_module.executable = this_module.qemu_executable common.executable = common.qemu_executable
this_module.run_dir = this_module.qemu_run_dir common.run_dir = common.qemu_run_dir
this_module.termout_file = this_module.qemu_termout_file common.termout_file = common.qemu_termout_file
this_module.guest_terminal_file = qemu_guest_terminal_file common.guest_terminal_file = qemu_guest_terminal_file
this_module.gem5_config_dir = os.path.join(this_module.gem5_src_dir, 'configs') common.gem5_config_dir = os.path.join(common.gem5_src_dir, 'configs')
this_module.gem5_se_file = os.path.join(this_module.gem5_config_dir, 'example', 'se.py') common.gem5_se_file = os.path.join(common.gem5_config_dir, 'example', 'se.py')
this_module.gem5_fs_file = os.path.join(this_module.gem5_config_dir, 'example', 'fs.py') common.gem5_fs_file = os.path.join(common.gem5_config_dir, 'example', 'fs.py')
this_module.run_cmd_file = os.path.join(this_module.run_dir, 'run.sh') common.run_cmd_file = os.path.join(common.run_dir, 'run.sh')
# Linux # Linux
this_module.linux_buildroot_build_dir = os.path.join(this_module.buildroot_build_build_dir, 'linux-custom') common.linux_buildroot_build_dir = os.path.join(common.buildroot_build_build_dir, 'linux-custom')
this_module.linux_build_dir = os.path.join(this_module.out_dir, 'linux', args.linux_build_id, args.arch) common.linux_build_dir = os.path.join(common.out_dir, 'linux', args.linux_build_id, args.arch)
this_module.lkmc_vmlinux = os.path.join(this_module.linux_build_dir, "vmlinux") common.lkmc_vmlinux = os.path.join(common.linux_build_dir, "vmlinux")
if args.arch == 'arm': if args.arch == 'arm':
this_module.linux_arch = 'arm' common.linux_arch = 'arm'
this_module.linux_image_prefix = os.path.join('arch', this_module.linux_arch, 'boot', 'zImage') common.linux_image_prefix = os.path.join('arch', common.linux_arch, 'boot', 'zImage')
elif args.arch == 'aarch64': elif args.arch == 'aarch64':
this_module.linux_arch = 'arm64' common.linux_arch = 'arm64'
this_module.linux_image_prefix = os.path.join('arch', this_module.linux_arch, 'boot', 'Image') common.linux_image_prefix = os.path.join('arch', common.linux_arch, 'boot', 'Image')
elif args.arch == 'x86_64': elif args.arch == 'x86_64':
this_module.linux_arch = 'x86' common.linux_arch = 'x86'
this_module.linux_image_prefix = os.path.join('arch', this_module.linux_arch, 'boot', 'bzImage') common.linux_image_prefix = os.path.join('arch', common.linux_arch, 'boot', 'bzImage')
this_module.lkmc_linux_image = os.path.join(this_module.linux_build_dir, this_module.linux_image_prefix) common.lkmc_linux_image = os.path.join(common.linux_build_dir, common.linux_image_prefix)
this_module.buildroot_linux_image = os.path.join(this_module.buildroot_linux_build_dir, linux_image_prefix) common.buildroot_linux_image = os.path.join(common.buildroot_linux_build_dir, linux_image_prefix)
if args.buildroot_linux: if args.buildroot_linux:
this_module.vmlinux = this_module.buildroot_vmlinux common.vmlinux = common.buildroot_vmlinux
this_module.linux_image = this_module.buildroot_linux_image common.linux_image = common.buildroot_linux_image
else: else:
this_module.vmlinux = this_module.lkmc_vmlinux common.vmlinux = common.lkmc_vmlinux
this_module.linux_image = this_module.lkmc_linux_image common.linux_image = common.lkmc_linux_image
# Kernel modules. # Kernel modules.
this_module.kernel_modules_build_base_dir = os.path.join(this_module.out_dir, 'kernel_modules') common.kernel_modules_build_base_dir = os.path.join(common.out_dir, 'kernel_modules')
this_module.kernel_modules_build_dir = os.path.join(this_module.kernel_modules_build_base_dir, args.arch) common.kernel_modules_build_dir = os.path.join(common.kernel_modules_build_base_dir, args.arch)
this_module.kernel_modules_build_subdir = os.path.join(this_module.kernel_modules_build_dir, kernel_modules_subdir) common.kernel_modules_build_subdir = os.path.join(common.kernel_modules_build_dir, kernel_modules_subdir)
this_module.kernel_modules_build_host_dir = os.path.join(this_module.kernel_modules_build_base_dir, 'host') common.kernel_modules_build_host_dir = os.path.join(common.kernel_modules_build_base_dir, 'host')
this_module.kernel_modules_build_host_subdir = os.path.join(this_module.kernel_modules_build_host_dir, kernel_modules_subdir) common.kernel_modules_build_host_subdir = os.path.join(common.kernel_modules_build_host_dir, kernel_modules_subdir)
this_module.userland_build_dir = os.path.join(this_module.out_dir, 'userland', args.userland_build_id, args.arch) common.userland_build_dir = os.path.join(common.out_dir, 'userland', args.userland_build_id, args.arch)
this_module.out_rootfs_overlay_dir = os.path.join(this_module.out_dir, 'rootfs_overlay', args.arch) common.out_rootfs_overlay_dir = os.path.join(common.out_dir, 'rootfs_overlay', args.arch)
this_module.out_rootfs_overlay_bin_dir = os.path.join(this_module.out_rootfs_overlay_dir, 'bin') common.out_rootfs_overlay_bin_dir = os.path.join(common.out_rootfs_overlay_dir, 'bin')
# Ports # Ports
if args.port_offset is None: if args.port_offset is None:
@@ -880,74 +875,74 @@ def setup(parser):
args.port_offset = int(args.run_id) args.port_offset = int(args.run_id)
except ValueError: except ValueError:
args.port_offset = 0 args.port_offset = 0
if args.gem5: if common.emulator == 'gem5':
this_module.gem5_telnet_port = 3456 + args.port_offset common.gem5_telnet_port = 3456 + args.port_offset
this_module.gdb_port = 7000 + args.port_offset common.gdb_port = 7000 + args.port_offset
else: else:
this_module.qemu_base_port = 45454 + 10 * args.port_offset common.qemu_base_port = 45454 + 10 * args.port_offset
this_module.qemu_monitor_port = this_module.qemu_base_port + 0 common.qemu_monitor_port = common.qemu_base_port + 0
this_module.qemu_hostfwd_generic_port = this_module.qemu_base_port + 1 common.qemu_hostfwd_generic_port = common.qemu_base_port + 1
this_module.qemu_hostfwd_ssh_port = this_module.qemu_base_port + 2 common.qemu_hostfwd_ssh_port = common.qemu_base_port + 2
this_module.qemu_gdb_port = this_module.qemu_base_port + 3 common.qemu_gdb_port = common.qemu_base_port + 3
this_module.extra_serial_port = this_module.qemu_base_port + 4 common.extra_serial_port = common.qemu_base_port + 4
this_module.gdb_port = this_module.qemu_gdb_port common.gdb_port = common.qemu_gdb_port
this_module.qemu_background_serial_file = os.path.join(this_module.qemu_run_dir, 'background.log') common.qemu_background_serial_file = os.path.join(common.qemu_run_dir, 'background.log')
# Baremetal. # Baremetal.
this_module.baremetal = args.baremetal common.baremetal = args.baremetal
this_module.baremetal_lib_basename = 'lib' common.baremetal_lib_basename = 'lib'
this_module.baremetal_src_dir = os.path.join(this_module.root_dir, 'baremetal') common.baremetal_src_dir = os.path.join(common.root_dir, 'baremetal')
this_module.baremetal_src_lib_dir = os.path.join(this_module.baremetal_src_dir, this_module.baremetal_lib_basename) common.baremetal_src_lib_dir = os.path.join(common.baremetal_src_dir, common.baremetal_lib_basename)
if args.gem5: if common.emulator == 'gem5':
this_module.simulator_name = 'gem5' common.simulator_name = 'gem5'
else: else:
this_module.simulator_name = 'qemu' common.simulator_name = 'qemu'
this_module.baremetal_build_dir = os.path.join(out_dir, 'baremetal', args.arch, this_module.simulator_name, this_module.machine) common.baremetal_build_dir = os.path.join(out_dir, 'baremetal', args.arch, common.simulator_name, common.machine)
this_module.baremetal_build_lib_dir = os.path.join(this_module.baremetal_build_dir, this_module.baremetal_lib_basename) common.baremetal_build_lib_dir = os.path.join(common.baremetal_build_dir, common.baremetal_lib_basename)
this_module.baremetal_build_ext = '.elf' common.baremetal_build_ext = '.elf'
# Docker # Docker
this_module.docker_build_dir = os.path.join(this_module.out_dir, 'docker', args.arch) common.docker_build_dir = os.path.join(common.out_dir, 'docker', args.arch)
this_module.docker_tar_dir = os.path.join(this_module.docker_build_dir, 'export') common.docker_tar_dir = os.path.join(common.docker_build_dir, 'export')
this_module.docker_tar_file = os.path.join(this_module.docker_build_dir, 'export.tar') common.docker_tar_file = os.path.join(common.docker_build_dir, 'export.tar')
this_module.docker_rootfs_raw_file = os.path.join(this_module.docker_build_dir, 'export.ext2') common.docker_rootfs_raw_file = os.path.join(common.docker_build_dir, 'export.ext2')
this_module.docker_qcow2_file = os.path.join(this_module.docker_rootfs_raw_file + '.qcow2') common.docker_qcow2_file = os.path.join(common.docker_rootfs_raw_file + '.qcow2')
if args.docker: if args.docker:
this_module.rootfs_raw_file = this_module.docker_rootfs_raw_file common.rootfs_raw_file = common.docker_rootfs_raw_file
this_module.qcow2_file = this_module.docker_qcow2_file common.qcow2_file = common.docker_qcow2_file
else: else:
this_module.rootfs_raw_file = this_module.buildroot_rootfs_raw_file common.rootfs_raw_file = common.buildroot_rootfs_raw_file
this_module.qcow2_file = this_module.buildroot_qcow2_file common.qcow2_file = common.buildroot_qcow2_file
# Image. # Image.
if args.baremetal is None: if args.baremetal is None:
if args.gem5: if common.emulator == 'gem5':
this_module.image = this_module.vmlinux common.image = common.vmlinux
this_module.disk_image = this_module.rootfs_raw_file common.disk_image = common.rootfs_raw_file
else: else:
this_module.image = this_module.linux_image common.image = common.linux_image
this_module.disk_image = this_module.qcow2_file common.disk_image = common.qcow2_file
else: else:
this_module.disk_image = this_module.gem5_fake_iso common.disk_image = common.gem5_fake_iso
if args.baremetal == 'all': if args.baremetal == 'all':
path = args.baremetal path = args.baremetal
else: else:
path = this_module.resolve_executable( path = common.resolve_executable(
args.baremetal, args.baremetal,
this_module.baremetal_src_dir, common.baremetal_src_dir,
this_module.baremetal_build_dir, common.baremetal_build_dir,
this_module.baremetal_build_ext, common.baremetal_build_ext,
) )
source_path_noext = os.path.splitext(os.path.join( source_path_noext = os.path.splitext(os.path.join(
this_module.baremetal_src_dir, common.baremetal_src_dir,
os.path.relpath(path, this_module.baremetal_build_dir) os.path.relpath(path, common.baremetal_build_dir)
))[0] ))[0]
for ext in [c_ext, asm_ext]: for ext in [c_ext, asm_ext]:
source_path = source_path_noext + ext source_path = source_path_noext + ext
if os.path.exists(source_path): if os.path.exists(source_path):
this_module.source_path = source_path common.source_path = source_path
break break
this_module.image = path common.image = path
return args return args
def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext): def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext):
@@ -968,16 +963,15 @@ def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext):
raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths)) raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths))
def resolve_userland(path): def resolve_userland(path):
global this_module return common.resolve_executable(
return this_module.resolve_executable(
path, path,
this_module.userland_src_dir, common.userland_src_dir,
this_module.userland_build_dir, common.userland_build_dir,
this_module.userland_build_ext, common.userland_build_ext,
) )
def setup_dry_run_arguments(args): def setup_dry_run_arguments(args):
this_module.dry_run = args.dry_run common.dry_run = args.dry_run
def shlex_split(string): def shlex_split(string):
''' '''
@@ -985,10 +979,10 @@ def shlex_split(string):
Not perfect since it does not group arguments, but I don't see a solution. Not perfect since it does not group arguments, but I don't see a solution.
''' '''
return this_module.add_newlines(shlex.split(string)) return common.add_newlines(shlex.split(string))
def strip_newlines(cmd): def strip_newlines(cmd):
return [x for x in cmd if x != this_module.Newline] return [x for x in cmd if x != common.Newline]
def write_configs(config_path, configs, config_fragments=None): def write_configs(config_path, configs, config_fragments=None):
""" """
@@ -996,14 +990,13 @@ def write_configs(config_path, configs, config_fragments=None):
TODO Can't get rid of these for now with nice fragments: TODO Can't get rid of these for now with nice fragments:
http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
""" """
global this_module
if config_fragments is None: if config_fragments is None:
config_fragments = [] config_fragments = []
with open(config_path, 'a') as config_file: with open(config_path, 'a') as config_file:
for config_fragment in config_fragments: for config_fragment in config_fragments:
with open(config_fragment, 'r') as config_fragment_file: with open(config_fragment, 'r') as config_fragment_file:
print_cmd(['cat', config_fragment, '>>', config_path]) print_cmd(['cat', config_fragment, '>>', config_path])
if not this_module.dry_run: if not common.dry_run:
for line in config_fragment_file: for line in config_fragment_file:
config_file.write(line) config_file.write(line)
write_string_to_file(config_path, '\n'.join(configs), mode='a') write_string_to_file(config_path, '\n'.join(configs), mode='a')

6
run
View File

@@ -127,7 +127,7 @@ def main(args, extra_args=None):
if common.image is None: if common.image is None:
raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
cmd = debug_vm.copy() cmd = debug_vm.copy()
if args.gem5: if common.emulator == 'gem5':
if args.baremetal is None: if args.baremetal is None:
if not os.path.exists(common.rootfs_raw_file): if not os.path.exists(common.rootfs_raw_file):
if not os.path.exists(common.qcow2_file): if not os.path.exists(common.qcow2_file):
@@ -359,7 +359,7 @@ def main(args, extra_args=None):
if args.baremetal is None: if args.baremetal is None:
cmd.extend(append) cmd.extend(append)
if args.tmux is not None: if args.tmux is not None:
if args.gem5: if common.emulator == 'gem5':
subprocess.Popen([os.path.join(common.root_dir, 'tmu'), subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
'sleep 2;./gem5-shell -n {} {}' \ 'sleep 2;./gem5-shell -n {} {}' \
.format(args.run_id, args.tmux) .format(args.run_id, args.tmux)
@@ -386,7 +386,7 @@ def main(args, extra_args=None):
out_file = common.termout_file out_file = common.termout_file
common.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) common.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env)
# Check if guest panicked. # Check if guest panicked.
if args.gem5: if common.emulator == 'gem5':
# We have to do some parsing here because gem5 exits with status 0 even when panic happens. # We have to do some parsing here because gem5 exits with status 0 even when panic happens.
# Grepping for '^panic: ' does not work because some errors don't show that message. # Grepping for '^panic: ' does not work because some errors don't show that message.
panic_msg = b'--- BEGIN LIBC BACKTRACE ---$' panic_msg = b'--- BEGIN LIBC BACKTRACE ---$'

34
test-gdb
View File

@@ -1,28 +1,28 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -eux set -eux
for gem5 in '' --gem5; do for emulator in --qemu --gem5; do
./run --arch arm --background --baremetal add $gem5 --wait-gdb & ./run --arch arm --background --baremetal add "$emulator" --wait-gdb &
./run-gdb --arch arm --baremetal add $gem5 --test "$@" ./run-gdb --arch arm --baremetal add "$emulator" --test "$@"
wait wait
./run --arch arm --background --baremetal arch/arm/add $gem5 --wait-gdb & ./run --arch arm --background --baremetal arch/arm/add "$emulator" --wait-gdb &
./run-gdb --arch arm --baremetal arch/arm/add $gem5 --test "$@" ./run-gdb --arch arm --baremetal arch/arm/add "$emulator" --test "$@"
wait wait
./run --arch arm --background --baremetal arch/arm/regs $gem5 --wait-gdb & ./run --arch arm --background --baremetal arch/arm/regs "$emulator" --wait-gdb &
./run-gdb --arch arm --baremetal arch/arm/regs $gem5 --test "$@" ./run-gdb --arch arm --baremetal arch/arm/regs "$emulator" --test "$@"
wait wait
./run --arch aarch64 --background --baremetal add $gem5 --wait-gdb & ./run --arch aarch64 --background --baremetal add "$emulator" --wait-gdb &
./run-gdb --arch aarch64 --baremetal add $gem5 --test "$@" ./run-gdb --arch aarch64 --baremetal add "$emulator" --test "$@"
wait wait
./run --arch aarch64 --background --baremetal arch/aarch64/add $gem5 --wait-gdb & ./run --arch aarch64 --background --baremetal arch/aarch64/add "$emulator" --wait-gdb &
./run-gdb --arch aarch64 --baremetal arch/aarch64/add $gem5 --test "$@" ./run-gdb --arch aarch64 --baremetal arch/aarch64/add "$emulator" --test "$@"
wait wait
./run --arch aarch64 --background --baremetal arch/aarch64/regs $gem5 --wait-gdb & ./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb &
./run-gdb --arch aarch64 --baremetal arch/aarch64/regs $gem5 --test "$@" ./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@"
wait wait
./run --arch aarch64 --background --baremetal arch/aarch64/fadd $gem5 --wait-gdb & ./run --arch aarch64 --background --baremetal arch/aarch64/fadd "$emulator" --wait-gdb &
./run-gdb --arch aarch64 --baremetal arch/aarch64/fadd $gem5 --test "$@" ./run-gdb --arch aarch64 --baremetal arch/aarch64/fadd "$emulator" --test "$@"
wait wait
./run --arch aarch64 --background --baremetal arch/aarch64/regs $gem5 --wait-gdb & ./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb &
./run-gdb --arch aarch64 --baremetal arch/aarch64/regs $gem5 --test "$@" ./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@"
wait wait
done done

2
trace-boot
View File

@@ -23,7 +23,7 @@ args = common.setup(parser)
extra_args = { extra_args = {
'extra_emulator_args': args.extra_emulator_args, 'extra_emulator_args': args.extra_emulator_args,
} }
if args.gem5: if common.emulator == 'gem5':
extra_args.update({ extra_args.update({
'eval': 'm5 exit', 'eval': 'm5 exit',
'trace': 'Exec,-ExecSymbol,-ExecMicro', 'trace': 'Exec,-ExecSymbol,-ExecMicro',

4
trace2line
View File

@@ -20,7 +20,7 @@ parser = common.get_argparse(argparse_args={
args = common.setup(parser) args = common.setup(parser)
sys.exit(subprocess.Popen([ sys.exit(subprocess.Popen([
os.path.join(common.root_dir, 'trace2line.sh'), os.path.join(common.root_dir, 'trace2line.sh'),
'true' if args.gem5 else 'false', 'true' if common.emulator == 'gem5' else 'false',
common.trace_txt_file, common.trace_txt_file,
common.get_toolchain_tool('addr2line'), common.get_toolchain_tool('addr2line'),
common.vmlinux, common.vmlinux,
@@ -29,7 +29,7 @@ sys.exit(subprocess.Popen([
# This was the full conversion attempt. # This was the full conversion attempt.
# if args.gem5: # if common.emulator == 'gem5':
# def get_pc(line): # def get_pc(line):
# # TODO # # TODO
# # stdin = sed -r 's/^.* (0x[^. ]*)[. ].*/\1/' "$common_trace_txt_file") # # stdin = sed -r 's/^.* (0x[^. ]*)[. ].*/\1/' "$common_trace_txt_file")