From 1ff6a95ab3d4cfa0b8d7d61a215367ee7690c0c9 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Tue, 4 Sep 2018 09:22:46 +0100 Subject: [PATCH] run: generalize with main(), start porting trace-boot and qemu-trace2txt --- README.adoc | 2 +- build | 6 +- common.py | 102 ++++--- getvar | 2 +- qemu-trace2txt | 41 ++- run | 748 ++++++++++++++++++++++++++----------------------- rungdb | 21 +- rungdb-user | 4 +- runtc | 3 +- trace-boot | 71 +++-- 10 files changed, 547 insertions(+), 453 deletions(-) diff --git a/README.adoc b/README.adoc index 0151992..1d46cb9 100644 --- a/README.adoc +++ b/README.adoc @@ -3078,7 +3078,7 @@ CONFIG_IKCONFIG_PROC=y From host: .... -cat "$(./getvar linux_custom_dir)/.config" +cat "$(./getvar linux_build_dir)/.config" .... Just for fun link:https://stackoverflow.com/a/14958263/895245[]: diff --git a/build b/build index 69c78d2..6814011 100755 --- a/build +++ b/build @@ -185,11 +185,11 @@ symlink_buildroot_variant() ( mkdir -p "$variant_dir" ln -s "$variant_dir" "$custom_dir" ) -symlink_buildroot_variant "$common_linux_custom_dir" "$common_linux_variant_dir" -symlink_buildroot_variant "$common_qemu_custom_dir" "$common_qemu_variant_dir" +symlink_buildroot_variant "$common_linux_build_dir" "$common_linux_variant_dir" +symlink_buildroot_variant "$common_qemu_build_dir" "$common_qemu_variant_dir" # TODO: this breaks the build. But then I noticed that it wouldn't make sense, # because this is a guest tool, and we don't have image variants yet. Some other day maybe. -#symlink_buildroot_variant "$common_qemu_guest_custom_dir" "$common_qemu_guest_variant_dir" +#symlink_buildroot_variant "$common_qemu_guest_build_dir" "$common_qemu_guest_variant_dir" # Manage gem5 variants. if "$common_gem5"; then diff --git a/common.py b/common.py index f9d23cf..5f73856 100644 --- a/common.py +++ b/common.py @@ -2,6 +2,7 @@ import argparse import base64 +import copy import glob import imp import os @@ -36,14 +37,10 @@ this = sys.modules[__name__] def base64_encode(string): return base64.b64encode(string.encode()).decode() -def error(msg): - print('error: {}'.format(msg), file=sys.stderr) - sys.exit(1) - def gem_list_checkpoint_dirs(): - """ + ''' List checkpoint directory, oldest first. - """ + ''' global this prefix_re = re.compile(this.gem5_cpt_prefix) files = list(filter(lambda x: os.path.isdir(os.path.join(this.m5out_dir, x)) and prefix_re.search(x), os.listdir(this.m5out_dir))) @@ -51,9 +48,9 @@ def gem_list_checkpoint_dirs(): return files def get_argparse(default_args=None, argparse_args=None): - """ + ''' Return an argument parser with common arguments set. - """ + ''' global this if default_args is None: default_args = {} @@ -86,27 +83,27 @@ def get_argparse(default_args=None, argparse_args=None): ) parser.add_argument( '-N', '--gem5-worktree', - help="""\ + help='''\ gem5 git worktree to use for build and Python scripts at runtime. Automatically create a new git worktree with the given id if one does not exist. If not given, just use the submodule source. -""" +''' ) parser.add_argument( '-n', '--run-id', default='0', - help="""\ + help='''\ ID for run outputs such as gem5's m5out. Allows you to do multiple runs, and then inspect separate outputs later in different output directories. Default: %(default)s -""" +''' ) parser.add_argument( '--port-offset', type=int, - help="""\ + help='''\ Increase the ports to be used such as for GDB by an offset to run multiple instances in parallel. Default: the run ID (-n) if that is an integer, otherwise 0. -""" +''' ) parser.add_argument( '-Q', '--qemu-build-id', default=default_build_id, @@ -114,11 +111,11 @@ Default: the run ID (-n) if that is an integer, otherwise 0. ) parser.add_argument( '-s', '--suffix', - help="""\ + help='''\ Add a custom suffix to the build. E.g., doing `./build -s mysuf` puts all the build output into `out/x86_64-mysuf`. This allows keep multiple builds around when you checkout between branches. -""" +''' ) parser.add_argument( '-t', '--gem5-build-type', default='opt', @@ -156,14 +153,17 @@ def get_toolchain_tool(tool): global this return glob.glob(os.path.join(this.host_bin_dir, '*-buildroot-*-{}'.format(tool)))[0] +def log_error(msg): + print('error: {}'.format(msg), file=sys.stderr) + def print_cmd(cmd, cmd_file=None, extra_env=None): - """ + ''' Format a command given as a list of strings so that it can be viewed nicely and executed by bash directly and print it to stdout. Optionally save the command to cmd_file file, and add extra_env environment variables to the command generated. - """ + ''' newline_separator = ' \\\n' out = [] for key in extra_env: @@ -179,23 +179,44 @@ def print_cmd(cmd, cmd_file=None, extra_env=None): st = os.stat(cmd_file) os.chmod(cmd_file, st.st_mode | stat.S_IXUSR) -def run_cmd(cmd, cmd_file=None, out_file=None, extra_env=None, **kwargs): - """ +def resolve_args(defaults, args, extra_args): + if extra_args is None: + extra_args = {} + argcopy = copy.copy(args) + argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items())) + return argcopy + +def run_cmd(cmd, cmd_file=None, out_file=None, show_stdout=True, extra_env=None, **kwargs): + ''' Run a command. Write the command to stdout before running it. Wait until the command finishes execution. - If: + :param cmd: command to run + :type cmd: List[str] - - cmd_file is not None, write the command to the given file - - out_file is not None, write the stdout and stderr of the command to the given file - """ + :param cmd_file: if not None, write the command to be run to that file + :type cmd_file: str + + :param out_file: if not None, write the stdout and stderr of the command the file + :type out_file: str + + :param show_stdout: wether to show stdout and stderr on the terminal or not + :type show_stdout: bool + + :param extra_env: extra environment variables to add when running the command + :type extra_env: Dict[str,str] + ''' if out_file is not None: - stdout=subprocess.PIPE - stderr=subprocess.STDOUT + stdout = subprocess.PIPE + stderr = subprocess.STDOUT else: - stdout=None - stderr=None + if show_stdout: + stdout = None + stderr = None + else: + stdout = subprocess.DEVNULL + stderr = subprocess.DEVNULL if extra_env is None: extra_env = {} env = os.environ.copy() @@ -212,8 +233,9 @@ def run_cmd(cmd, cmd_file=None, out_file=None, extra_env=None, **kwargs): while True: byte = proc.stdout.read(1) if byte: - sys.stdout.buffer.write(byte) - sys.stdout.flush() + if show_stdout: + sys.stdout.buffer.write(byte) + sys.stdout.flush() logfile.write(byte) else: break @@ -221,10 +243,10 @@ def run_cmd(cmd, cmd_file=None, out_file=None, extra_env=None, **kwargs): return proc.returncode def setup(parser, **extra_args): - """ + ''' Parse the command line arguments, and setup several variables based on them. Typically done after getting inputs from the command line arguments. - """ + ''' global this args = parser.parse_args() if args.arch in this.arch_map: @@ -245,14 +267,14 @@ def setup(parser, **extra_args): this.out_arch_dir = os.path.join(this.out_dir, this.arch_dir) this.buildroot_out_dir = os.path.join(this.out_arch_dir, 'buildroot') this.build_dir = os.path.join(this.buildroot_out_dir, 'build') - this.linux_custom_dir = os.path.join(this.build_dir, 'linux-custom') - this.linux_variant_dir = '{}.{}'.format(this.linux_custom_dir, args.linux_build_id) + this.linux_build_dir = os.path.join(this.build_dir, 'linux-custom') + this.linux_variant_dir = '{}.{}'.format(this.linux_build_dir, args.linux_build_id) this.vmlinux = os.path.join(this.linux_variant_dir, "vmlinux") - this.qemu_custom_dir = os.path.join(this.build_dir, 'host-qemu-custom') - this.qemu_guest_variant_dir = os.path.join(this.qemu_custom_dir, args.qemu_build_id) - this.qemu_variant_dir = '{}.{}'.format(this.qemu_custom_dir, args.qemu_build_id) + this.qemu_build_dir = os.path.join(this.build_dir, 'host-qemu-custom') + this.qemu_guest_variant_dir = os.path.join(this.qemu_build_dir, args.qemu_build_id) + this.qemu_variant_dir = '{}.{}'.format(this.qemu_build_dir, args.qemu_build_id) this.qemu_executable = os.path.join(this.qemu_variant_dir, '{}-softmmu'.format(args.arch), 'qemu-system-{}'.format(args.arch)) - this.qemu_guest_custom_dir = os.path.join(this.build_dir, 'qemu-custom') + this.qemu_guest_build_dir = os.path.join(this.build_dir, 'qemu-custom') this.host_dir = os.path.join(this.buildroot_out_dir, 'host') this.host_bin_dir = os.path.join(this.host_dir, 'usr', 'bin') this.images_dir = os.path.join(this.buildroot_out_dir, 'images') @@ -266,6 +288,9 @@ def setup(parser, **extra_args): this.trace_txt_file = os.path.join(this.m5out_dir, 'trace.txt') this.gem5_termout_file = os.path.join(this.gem5_run_dir, 'termout.txt') this.qemu_run_dir = os.path.join(this.out_arch_dir, 'qemu', str(args.run_id)) + this.qemu_trace_basename = 'trace.bin' + this.qemu_trace_file = os.path.join(this.qemu_run_dir, 'trace.bin') + this.qemu_trace_txt_file = os.path.join(this.qemu_run_dir, 'trace.txt') this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt') this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile') this.gem5_out_dir = os.path.join(this.common_dir, 'gem5', args.gem5_build_id) @@ -327,6 +352,7 @@ p9_dir = os.path.join(data_dir, '9p') gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5') gem5_readfile_file = os.path.join(data_dir, 'readfile') gem5_default_src_dir = os.path.join(root_dir, 'gem5', 'gem5') +qemu_src_dir = os.path.join(root_dir, 'qemu') out_dir = os.path.join(root_dir, 'out') bench_boot = os.path.join(out_dir, 'bench-boot.txt') common_dir = os.path.join(out_dir, 'common') diff --git a/getvar b/getvar index 60df09f..26ac6bf 100755 --- a/getvar +++ b/getvar @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import common parser = common.get_argparse(argparse_args={ - 'description':'''Print the value of a common.py variable. + 'description': '''Print the value of a common.py variable. This is useful to: diff --git a/qemu-trace2txt b/qemu-trace2txt index 2a23e8e..ff58d8a 100755 --- a/qemu-trace2txt +++ b/qemu-trace2txt @@ -1,14 +1,27 @@ -#!/usr/bin/env bash -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common" -while getopts "${common_getopts_flags}" OPT; do - case "$OPT" in - ?) - common_getopts_case "$OPT" - ;; - esac -done -common_setup -./qemu/scripts/simpletrace.py \ - "${common_build_dir}/host-qemu-custom/trace-events-all" \ - "${common_qemu_run_dir}/trace.bin" \ -> "${common_trace_txt_file}" +#!/usr/bin/env python3 + +import os +import subprocess +import sys + +import common + +def main(): + cmd = [ + os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), + os.path.join(common.qemu_build_dir, 'trace-events-all'), + os.path.join(common.qemu_trace_file), + ] + return common.run_cmd( + cmd, + cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'), + out_file=common.qemu_trace_txt_file, + show_stdout=False, + ) + +if __name__ == '__main__': + parser = common.get_argparse(argparse_args={ + 'description': 'Convert a QEMU `-trace exec_tb` to text form.' + }) + args = common.setup(parser) + sys.exit(main()) diff --git a/run b/run index a805af2..6dcf8b6 100755 --- a/run +++ b/run @@ -8,396 +8,432 @@ import re import common -# Argparse. -parser = common.get_argparse(argparse_args={'description':'Run Linux on an emulator'}) -init_group = parser.add_mutually_exclusive_group() -kvm_group = parser.add_mutually_exclusive_group() -parser.add_argument( - '-c', '--cpus', default=1, type=int, - help='Number of guest CPUs to emulate. Default: %(default)s' -) -parser.add_argument( - '-D', '--debug-vm', default=False, action='store_true', - help='Run GDB on the emulator itself.' -) -kvm_group.add_argument( - '-d', '--debug-guest', default=False, action='store_true', - help='Wait for GDB to connect before starting execution' -) -parser.add_argument( - '-E', '--eval', - help="""\ +defaults = { + 'cpus': 1, + 'debug_vm': False, + 'debug_guest': False, + 'eval': None, + 'kernel_cli_extra': None, + 'kernel_cli_extra_after_dash_base64': None, + 'kernel_cli_extra_after_dash': None, + 'gem5_exe_args':'', + 'gem5_biglittle': False, + 'initramfs': False, + 'initrd': False, + 'kvm': False, + 'kgdb': False, + 'gem5_restore_last_checkpoint': None, + 'memory': '256M', + 'prebuilt': False, + 'qemu_replay': False, + 'qemu_record': False, + 'trace': None, + 'terminal': False, + 'tmux_args': '', + 'tmux': False, + 'graphic': False, + 'vnc': False, + 'extra_emulator_args': None, +} + +def main(args, extra_args=None): + global defaults + args = common.resolve_args(defaults, args, extra_args) + # Common qemu / gem5 logic. + # nokaslr: + # * https://unix.stackexchange.com/questions/397939/turning-off-kaslr-to-debug-linux-kernel-using-qemu-and-gdb + # * https://stackoverflow.com/questions/44612822/unable-to-debug-kernel-with-qemu-gdb/49840927#49840927 + # Turned on by default since v4.12 + kernel_cli_extra = 'console_msg_format=syslog nokaslr norandmaps panic=-1 printk.devkmsg=on printk.time=y' + if args.kernel_cli_extra is not None: + kernel_cli_extra += ' {}'.format(args.kernel_cli_extra) + kernel_cli_extra_after_dash = '' + extra_emulator_args = args.extra_emulator_args.copy() + extra_qemu_args = [] + if args.debug_vm: + debug_vm = ['gdb', '-q', '-ex', 'start', '--args'] + else: + debug_vm = [] + if args.debug_guest: + extra_qemu_args.append('-S') + if args.kernel_cli_extra_after_dash_base64 is not None: + kernel_cli_extra_after_dash += ' lkmc_eval_base64="{}"'.format(common.base64_encode(args.kernel_cli_extra_after_dash_base64)) + if args.kernel_cli_extra_after_dash is not None: + kernel_cli_extra_after_dash += ' {}'.format(args.kernel_cli_extra_after_dash) + if args.kgdb: + kernel_cli_extra += ' kgdbwait' + if args.vnc: + vnc = ['-vnc', ':0'] + else: + vnc = [] + if args.initrd or args.initramfs: + ramfs = True + else: + ramfs = False + if args.eval is not None: + if ramfs: + initarg = 'rdinit' + else: + initarg = 'init' + kernel_cli_extra += ' {}=/eval_base64.sh'.format(initarg) + kernel_cli_extra_after_dash += ' lkmc_eval="{}"'.format(common.base64_encode(args.eval)) + if not args.graphic: + if args.arch == 'x86_64': + kernel_cli_extra += ' console=ttyS0' + extra_qemu_args.append('-nographic') + if kernel_cli_extra_after_dash: + kernel_cli_extra += " -{}".format(kernel_cli_extra_after_dash) + extra_env = {} + + # A dummy value that is already turned on by default and does not produce large output, + # just to prevent QEMU from emitting a warning that '' is not valid. + trace_type = 'pr_manager_run' + + if args.gem5: + memory = '{}B'.format(args.memory) + gem5_exe_args = shlex.split(args.gem5_exe_args) + if args.trace is not None: + gem5_exe_args.append('--debug-flags={}'.format(args.trace)) + extra_env['M5_PATH'] = common.gem5_system_dir + cmd = ( + debug_vm + + [ + common.executable, + '--debug-file=trace.txt', + ] + + gem5_exe_args + + [ + '-d', common.m5out_dir + ] + ) + if args.gem5_biglittle: + if args.gem5_restore_last_checkpoint is not None: + cpt_dir = common.gem_list_checkpoint_dirs()[-args.gem5_restore_last_checkpoint] + extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)]) + cmd += [ + os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), + '--big-cpus', '2', + '--cpu-type', 'atomic', + '--disk', common.ext2_file, + '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), + '--kernel', common.vmlinux, + '--little-cpus', '2' + ] + else: + # TODO port + if args.gem5_restore_last_checkpoint is not None: + cpt_dirs = common.gem_list_checkpoint_dirs() + cpt_dir = cpt_dirs[-args.gem5_restore_last_checkpoint] + extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) + cmd += [ + os.path.join(common.gem5_src_dir, 'configs', 'example', 'fs.py'), + '--disk-image', common.ext2_file, + '--kernel', common.vmlinux, + '--mem-size', memory, + '--num-cpus', str(args.cpus), + '--script', common.gem5_readfile_file, + ] + if args.arch == 'x86_64': + if args.kvm: + cmd += ['--cpu-type', 'X86KvmCPU'] + cmd += ['--command-line', 'earlyprintk=ttyS0 console=ttyS0 lpj=7999923 root=/dev/sda {}'.format(kernel_cli_extra)] + elif args.arch == 'arm' or args.arch == 'aarch64': + # TODO why is it mandatory to pass mem= here? Not true for QEMU. + # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? + cmd += [ + '--command-line', 'earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli_extra), + '--dtb-file', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)), + '--machine-type', 'VExpress_GEM5_V1', + ] + else: + os.makedirs(common.run_dir, exist_ok=True) + if args.debug_vm: + serial_monitor = [] + else: + serial_monitor = ['-serial', 'mon:stdio'] + if args.kvm: + extra_emulator_args.append('-enable-kvm') + if args.kgdb: + extra_qemu_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.gdb_port)]) + if args.prebuilt: + common.mkdir() + qemu_executable = "qemu-system-{}".format(args.arch) + else: + qemu_executable = common.qemu_executable + extra_emulator_args = extra_qemu_args + extra_emulator_args + cmd = ( + debug_vm + + [ + qemu_executable, + '-device', 'rtl8139,netdev=net0', + '-gdb', 'tcp::{}'.format(common.gdb_port), + '-kernel', common.linux_image, + '-m', args.memory, + '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), + '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), + '-no-reboot', + '-smp', str(args.cpus), + '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file), + '-virtfs', 'local,path={},mount_tag=host_scratch,security_model=mapped,id=host_scratch'.format(common.p9_dir), + '-virtfs', 'local,path={},mount_tag=host_out,security_model=mapped,id=host_out'.format(common.build_dir), + ] + + serial_monitor + + vnc + ) + if args.initrd: + extra_emulator_args.extend(['-initrd', os.path.join(common.images_dir, 'rootfs.cpio')]) + rr = args.qemu_record or args.qemu_replay + if ramfs: + # TODO why is this needed, and why any string works. + root = 'root=/dev/anything' + else: + if rr: + driveif = 'none' + rrid = ',id=img-direct' + root = 'root=/dev/sda' + snapshot = '' + else: + driveif = 'virtio' + root = 'root=/dev/vda' + rrid = '' + snapshot = ',snapshot' + extra_emulator_args.extend([ + '-drive', + 'file={},format=qcow2,if={}{}{}'.format(common.qcow2_file, driveif, snapshot, rrid) + ]) + if rr: + extra_emulator_args.extend([ + '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', + '-device', 'ide-hd,drive=img-blkreplay' + ]) + if rr: + extra_emulator_args.extend([ + '-object', 'filter-replay,id=replay,netdev=net0', + '-icount', 'shift=7,rr={},rrfile={}'.format('record' if args.qemu_record else 'replay', common.qemu_rrfile), + ]) + virtio_gpu_pci = [] + else: + virtio_gpu_pci = ['-device', 'virtio-gpu-pci'] + if args.arch == 'x86_64': + if args.kgdb: + kernel_cli_extra += ' kgdboc=ttyS0,115200' + cmd.extend([ + '-M', 'pc', + '-append', '{} nopat {}'.format(root, kernel_cli_extra), + '-device', 'edu', + ]) + elif args.arch == 'arm' or args.arch == 'aarch64': + if args.kgdb: + kernel_cli_extra += ' kgdboc=ttyAMA0,115200' + if args.arch == 'arm': + cpu = 'cortex-a15' + else: + cpu = 'cortex-a57' + # highmem=off needed since v3.0.0 due to: + # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html + cmd = ( + cmd + + [ + '-M', 'virt,highmem=off', + '-append', '{} {}'.format(root, kernel_cli_extra), + '-cpu', cpu, + ] + + virtio_gpu_pci + ) + + if args.tmux: + if args.gem5: + subprocess.Popen([os.path.join(common.root_dir, 'tmu'), + 'sleep 2;./gem5-shell -n {} {}' \ + .format(args.run_id, args.tmux_args) + ]) + elif args.debug_guest: + # TODO find a nicer way to forward all those args automatically. + # Part of me wants to: https://github.com/jonathanslenders/pymux + # but it cannot be used as a library properly it seems, and it is + # slower than tmux. + subprocess.Popen([os.path.join(common.root_dir, 'tmu'), + "sleep 2;./rungdb -a '{}' -L '{}' -n '{}' {}" \ + .format(args.arch, args.linux_build_id, args.run_id, args.tmux_args) + ]) + + cmd += extra_emulator_args + if debug_vm or args.terminal: + out_file = None + else: + out_file = common.termout_file + returncode = common.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) + if returncode != 0: + common.log_error('simulator exited with status != 0') + return returncode + # Check if guest panicked. + if args.gem5: + # We have to do some parsing here because gem5 exits with status 0 even when panic happens. + # Grepping for '^panic: ' does not work because some errors don't show that message. + panic_msg = '--- BEGIN LIBC BACKTRACE ---$' + else: + panic_msg = 'Kernel panic - not syncing' + panic_re = re.compile(panic_msg) + with open(common.termout_file, 'r') as logfile: + for line in logfile: + if panic_re.search(line): + common.log_error('simulation error detected by parsing logs') + return 1 + return 0 + +if __name__ == '__main__': + # Argparse. + parser = common.get_argparse(argparse_args={'description':'Run Linux on an emulator'}) + init_group = parser.add_mutually_exclusive_group() + kvm_group = parser.add_mutually_exclusive_group() + parser.add_argument( + '-c', '--cpus', default=defaults['cpus'], type=int, + help='Number of guest CPUs to emulate. Default: %(default)s' + ) + parser.add_argument( + '-D', '--debug-vm', default=defaults['debug_vm'], action='store_true', + help='Run GDB on the emulator itself.' + ) + kvm_group.add_argument( + '-d', '--debug-guest', default=defaults['debug_guest'], action='store_true', + help='Wait for GDB to connect before starting execution' + ) + parser.add_argument( + '-E', '--eval', + help='''\ Replace the normal init with a minimal init that just evals with given `CMDSTR` bash command string. Example: `-E 'insmod /hello.ko;'` -""" -) -parser.add_argument( - '-e', '--kernel-cli-extra', - help="""\ +''' + ) + parser.add_argument( + '-e', '--kernel-cli-extra', + help='''\ Pass an extra Linux kernel command line options, and place them before the dash separator `-`. Only options that come before the `-`, i.e. "standard" options, should be passed with this option. Example: `./run -a arm -e 'init=/poweroff.out'` -""" -) -parser.add_argument( - '-F', '--kernel-cli-extra-after-dash-base64', - help="""\ -Much like `-f`, but base64 encodes the string. Mnemonic: -`-F` is to `-f` what `-E` is to `-e`.) -""" -) -parser.add_argument( - '-f', '--kernel-cli-extra-after-dash', - help="""\ +''' + ) + parser.add_argument( + '-F', '--kernel-cli-extra-after-dash-base64', + help='''\ + Much like `-f`, but base64 encodes the string. Mnemonic: + `-F` is to `-f` what `-E` is to `-e`.) + ''' + ) + parser.add_argument( + '-f', '--kernel-cli-extra-after-dash', + help='''\ Pass an extra Linux kernel command line options, add a dash `-` separator, and place the options after the dash. Intended for custom options understood by our `init` scripts, most of which are prefixed by `lkmc_`, e.g.: `./run -f 'lkmc_eval="wget google.com" lkmc_lala=y'` Mnenomic: comes after `-e`. -""" -) -parser.add_argument( - '-G', '--gem5-exe-args', default='', - help="""\ +''' + ) + parser.add_argument( + '-G', '--gem5-exe-args', default=defaults['gem5_exe_args'], + help='''\ Pass extra options to the gem5 executable. Do not confuse with the arguments passed to config scripts, like `fs.py`. Example: `./run -G '--debug-flags=Exec --debug' -g`. -""" -) -parser.add_argument( - '--gem5-biglittle', default=False, action='store_true', - help='Use fs_bigLITTLE.py instead of fs.py' -) -init_group.add_argument( - '-I', '--initramfs', default=False, action='store_true', - help='Use initramfs instead of a root filesystem' -) -init_group.add_argument( - '-i', '--initrd', default=False, action='store_true', - help='Use initrd instead of a root filesystem' -) -kvm_group.add_argument( - '-K', '--kvm', default=False, action='store_true', - help='Use KVM. Only works if guest arch == host arch' -) -parser.add_argument( - '-k', '--kgdb', default=False, action='store_true' -) -parser.add_argument( - '-l', '--gem5-restore-last-checkpoint', type=int, - help="""\ -Restore the nth most recently taken gem5 checkpoint according to directory -timestamps. -""" -) -parser.add_argument( - '-m', '--memory', default='256M', - help="""\ +''' + ) + parser.add_argument( + '--gem5-biglittle', default=defaults['gem5_biglittle'], action='store_true', + help='Use fs_bigLITTLE.py instead of fs.py' + ) + init_group.add_argument( + '-I', '--initramfs', default=defaults['initramfs'], action='store_true', + help='Use initramfs instead of a root filesystem' + ) + init_group.add_argument( + '-i', '--initrd', default=defaults['initrd'], action='store_true', + help='Use initrd instead of a root filesystem' + ) + kvm_group.add_argument( + '-K', '--kvm', default=defaults['kvm'], action='store_true', + help='Use KVM. Only works if guest arch == host arch' + ) + parser.add_argument( + '-k', '--kgdb', default=defaults['kgdb'], action='store_true' + ) + parser.add_argument( + '-l', '--gem5-restore-last-checkpoint', type=int, + help='''\ + Restore the nth most recently taken gem5 checkpoint according to directory + timestamps. + ''' + ) + parser.add_argument( + '-m', '--memory', default=defaults['memory'], + help='''\ Set the memory size of the guest. E.g.: `-m 512M`. We try to keep the default at the minimal ammount amount that boots all archs. Anything lower could lead some arch to fail to boot. Default: %(default)s -""" -) -parser.add_argument( - '-P', '--prebuilt', default=False, action='store_true', - help='Run the downloaded prebuilt images.' -) -group = parser.add_mutually_exclusive_group() -group.add_argument( - '-R', '--qemu-replay', default=False, action='store_true', - help='Replay a QEMU run record deterministically' -) -group.add_argument( - '-r', '--qemu-record', default=False, action='store_true', - help='Record a QEMU run record for later replay with `-R`' -) -parser.add_argument( - '-T', '--trace', - help="""\ +''' + ) + parser.add_argument( + '-P', '--prebuilt', default=defaults['prebuilt'], action='store_true', + help='Run the downloaded prebuilt images.' + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + '-R', '--qemu-replay', default=defaults['qemu_replay'], action='store_true', + help='Replay a QEMU run record deterministically' + ) + group.add_argument( + '-r', '--qemu-record', default=defaults['qemu_record'], action='store_true', + help='Record a QEMU run record for later replay with `-R`' + ) + parser.add_argument( + '-T', '--trace', + help='''\ Set trace events to be enabled. If not given, gem5 tracing is completely disabled, while QEMU tracing is enabled but uses default traces that are very rare and don't affect performance, because `./configure --enable-trace-backends=simple` seems to enable some traces by default, e.g. `pr_manager_run`, and I don't know how to get rid of them. -""" -) -init_group.add_argument( - '--terminal', default=False, action='store_true', - help='''Output to the terminal, don't pipe to tee as the default. +''' + ) + init_group.add_argument( + '--terminal', default=defaults['terminal'], action='store_true', + help='''Output to the terminal, don't pipe to tee as the default. Does not save the output to a file, but allows you to use debuggers. Set automatically by --debug-vm, but you still need this option to debug gem5 Python scripts. ''' -) -parser.add_argument( - '-U', '--tmux-args', default='', - help='Pass extra parameters to the program running on the `-u` tmux split' -) -parser.add_argument( - '-u', '--tmux', default=False, action='store_true', - help="""\ + ) + parser.add_argument( + '-U', '--tmux-args', default=defaults['tmux_args'], + help='Pass extra parameters to the program running on the `-u` tmux split' + ) + parser.add_argument( + '-u', '--tmux', default=defaults['tmux'], action='store_true', + help='''\ Create a tmUx split the window. You must already be inside of a `tmux` session to use this option: * on the main window, run the emulator as usual * on the split: ** if on QEMU and `-d` is given, GDB ** if on gem5, the gem5 terminal -""" -) -parser.add_argument( - '-x', '--graphic', default=False, action='store_true', - help='Run in graphic mode. Mnemonic: X11' -) -parser.add_argument( - '-V', '--vnc', default=False, action='store_true', - help="""\ +''' + ) + parser.add_argument( + '-x', '--graphic', default=defaults['graphic'], action='store_true', + help='Run in graphic mode. Mnemonic: X11' + ) + parser.add_argument( + '-V', '--vnc', default=defaults['vnc'], action='store_true', + help='''\ Run QEMU with VNC instead of the default SDL. Connect to it with: `vinagre localhost:5900`. -""" -) -parser.add_argument( - 'extra_emulator_args', nargs='*', - help='Extra options to append at the end of the emulator command line' -) -args = common.setup(parser) - -# Common qemu / gem5 logic. -# nokaslr: -# * https://unix.stackexchange.com/questions/397939/turning-off-kaslr-to-debug-linux-kernel-using-qemu-and-gdb -# * https://stackoverflow.com/questions/44612822/unable-to-debug-kernel-with-qemu-gdb/49840927#49840927 -# Turned on by default since v4.12 -kernel_cli_extra = 'console_msg_format=syslog nokaslr norandmaps panic=-1 printk.devkmsg=on printk.time=y' -if args.kernel_cli_extra is not None: - kernel_cli_extra += ' {}'.format(args.kernel_cli_extra) -kernel_cli_extra_after_dash = '' -extra_emulator_args = args.extra_emulator_args.copy() -extra_qemu_args = [] -if args.debug_vm: - debug_vm = ['gdb', '-q', '-ex', 'start', '--args'] -else: - debug_vm = [] -if args.debug_guest: - extra_qemu_args.append('-S') -if args.kernel_cli_extra_after_dash_base64 is not None: - kernel_cli_extra_after_dash += ' lkmc_eval_base64="{}"'.format(common.base64_encode(args.kernel_cli_extra_after_dash_base64)) -if args.kernel_cli_extra_after_dash is not None: - kernel_cli_extra_after_dash += ' {}'.format(args.kernel_cli_extra_after_dash) -if args.kgdb: - kernel_cli_extra += ' kgdbwait' -if args.vnc: - vnc = ['-vnc', ':0'] -else: - vnc = [] -if args.initrd or args.initramfs: - ramfs = True -else: - ramfs = False -if args.eval is not None: - if ramfs: - initarg = 'rdinit' - else: - initarg = 'init' - kernel_cli_extra += ' {}=/eval_base64.sh'.format(initarg) - kernel_cli_extra_after_dash += ' lkmc_eval="{}"'.format(common.base64_encode(args.eval)) -if not args.graphic: - if args.arch == 'x86_64': - kernel_cli_extra += ' console=ttyS0' - extra_qemu_args.append('-nographic') -if kernel_cli_extra_after_dash: - kernel_cli_extra += " -{}".format(kernel_cli_extra_after_dash) -extra_env = {} - -# A dummy value that is already turned on by default and does not produce large output, -# just to prevent QEMU from emitting a warning that '' is not valid. -trace_type = 'pr_manager_run' - -if args.gem5: - memory = '{}B'.format(args.memory) - gem5_exe_args = shlex.split(args.gem5_exe_args) - if args.trace is not None: - gem5_exe_args.append('--debug-flags={}'.format(args.trace)) - extra_env['M5_PATH'] = common.gem5_system_dir - cmd = ( - debug_vm + - [ - common.executable, - '--debug-file=trace.txt', - ] + - gem5_exe_args + - [ - '-d', common.m5out_dir - ] +''' ) - if args.gem5_biglittle: - if args.gem5_restore_last_checkpoint is not None: - cpt_dir = common.gem_list_checkpoint_dirs()[-args.gem5_restore_last_checkpoint] - extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)]) - cmd += [ - os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), - '--big-cpus', '2', - '--cpu-type', 'atomic', - '--disk', common.ext2_file, - '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), - '--kernel', common.vmlinux, - '--little-cpus', '2' - ] - else: - # TODO port - if args.gem5_restore_last_checkpoint is not None: - cpt_dirs = common.gem_list_checkpoint_dirs() - cpt_dir = cpt_dirs[-args.gem5_restore_last_checkpoint] - extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) - cmd += [ - os.path.join(common.gem5_src_dir, 'configs', 'example', 'fs.py'), - '--disk-image', common.ext2_file, - '--kernel', common.vmlinux, - '--mem-size', memory, - '--num-cpus', str(args.cpus), - '--script', common.gem5_readfile_file, - ] - if args.arch == 'x86_64': - if args.kvm: - cmd += ['--cpu-type', 'X86KvmCPU'] - cmd += ['--command-line', 'earlyprintk=ttyS0 console=ttyS0 lpj=7999923 root=/dev/sda {}'.format(kernel_cli_extra)] - elif args.arch == 'arm' or args.arch == 'aarch64': - # TODO why is it mandatory to pass mem= here? Not true for QEMU. - # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? - cmd += [ - '--command-line', 'earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli_extra), - '--dtb-file', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)), - '--machine-type', 'VExpress_GEM5_V1', - ] -else: - os.makedirs(common.run_dir, exist_ok=True) - if args.debug_vm: - serial_monitor = [] - else: - serial_monitor = ['-serial', 'mon:stdio'] - if args.kvm: - extra_emulator_args.append('-enable-kvm') - if args.kgdb: - extra_qemu_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.gdb_port)]) - if args.prebuilt: - common.mkdir() - qemu_executable = "qemu-system-{}".format(args.arch) - else: - qemu_executable = common.qemu_executable - extra_emulator_args = extra_qemu_args + extra_emulator_args - cmd = ( - debug_vm + - [ - qemu_executable, - '-device', 'rtl8139,netdev=net0', - '-gdb', 'tcp::{}'.format(common.gdb_port), - '-kernel', common.linux_image, - '-m', args.memory, - '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), - '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), - '-no-reboot', - '-smp', str(args.cpus), - '-trace', 'enable={},file={}'.format(trace_type, os.path.join(common.run_dir, 'trace.bin')), - '-virtfs', 'local,path={},mount_tag=host_scratch,security_model=mapped,id=host_scratch'.format(common.p9_dir), - '-virtfs', 'local,path={},mount_tag=host_out,security_model=mapped,id=host_out'.format(common.build_dir), - ] + - serial_monitor + - vnc + parser.add_argument( + 'extra_emulator_args', nargs='*', + help='Extra options to append at the end of the emulator command line' ) - if args.initrd: - extra_emulator_args.extend(['-initrd', os.path.join(common.images_dir, 'rootfs.cpio')]) - rr = args.qemu_record or args.qemu_replay - if ramfs: - # TODO why is this needed, and why any string works. - root = 'root=/dev/anything' - else: - if rr: - driveif = 'none' - rrid = ',id=img-direct' - root = 'root=/dev/sda' - snapshot = '' - else: - driveif = 'virtio' - root = 'root=/dev/vda' - rrid = '' - snapshot = ',snapshot' - extra_emulator_args.extend([ - '-drive', - 'file={},format=qcow2,if={}{}{}'.format(common.qcow2_file, driveif, snapshot, rrid) - ]) - if rr: - extra_emulator_args.extend([ - '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', - '-device', 'ide-hd,drive=img-blkreplay' - ]) - if rr: - extra_emulator_args.extend([ - '-object', 'filter-replay,id=replay,netdev=net0', - '-icount', 'shift=7,rr={},rrfile={}'.format('record' if args.qemu_record else 'replay', common.qemu_rrfile), - ]) - virtio_gpu_pci = [] - else: - virtio_gpu_pci = ['-device', 'virtio-gpu-pci'] - if args.arch == 'x86_64': - if args.kgdb: - kernel_cli_extra += ' kgdboc=ttyS0,115200' - cmd.extend([ - '-M', 'pc', - '-append', '{} nopat {}'.format(root, kernel_cli_extra), - '-device', 'edu', - ]) - elif args.arch == 'arm' or args.arch == 'aarch64': - if args.kgdb: - kernel_cli_extra += ' kgdboc=ttyAMA0,115200' - if args.arch == 'arm': - cpu = 'cortex-a15' - else: - cpu = 'cortex-a57' - # highmem=off needed since v3.0.0 due to: - # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html - cmd = ( - cmd + - [ - '-M', 'virt,highmem=off', - '-append', '{} {}'.format(root, kernel_cli_extra), - '-cpu', cpu, - ] + - virtio_gpu_pci - ) - -if args.tmux: - if args.gem5: - subprocess.Popen([os.path.join(common.root_dir, 'tmu'), - 'sleep 2;./gem5-shell -n {} {}' \ - .format(args.run_id, args.tmux_args) - ]) - elif args.debug_guest: - # TODO find a nicer way to forward all those args automatically. - # Part of me wants to: https://github.com/jonathanslenders/pymux - # but it cannot be used as a library properly it seems, and it is - # slower than tmux. - subprocess.Popen([os.path.join(common.root_dir, 'tmu'), - "sleep 2;./rungdb -a '{}' -L '{}' -n '{}' {}" \ - .format(args.arch, args.linux_build_id, args.run_id, args.tmux_args) - ]) - -cmd += extra_emulator_args -if debug_vm or args.terminal: - out_file = None -else: - out_file = common.termout_file -returncode = common.run_cmd(cmd, cmd_file=common.run_cmd_file, out_file=out_file, extra_env=extra_env) -if returncode != 0: - common.error('simulator exited with status != 0') -# Check if guest panicked. -if args.gem5: - # We have to do some parsing here because gem5 exits with status 0 even when panic happens. - # Grepping for '^panic: ' does not work because some errors don't show that message. - panic_msg = '--- BEGIN LIBC BACKTRACE ---$' -else: - panic_msg = 'Kernel panic - not syncing' -panic_re = re.compile(panic_msg) -with open(common.termout_file, 'r') as logfile: - for line in logfile: - if panic_re.search(line): - common.error('simulation error detected by parsing logs') + args = common.setup(parser) + sys.exit(main(args)) diff --git a/rungdb b/rungdb index 02786fb..270f86a 100755 --- a/rungdb +++ b/rungdb @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import copy import os import shlex import sys @@ -19,19 +18,19 @@ defaults = { } def main(args, extra_args=None): - """ + ''' :param args: argparse parse_argument() output. Must contain all the common options, but does not need GDB specific ones. :type args: argparse.Namespace :param extra_args: extra arguments to be added to args :type extra_args: Dict[str,Any] - """ + + :return: GDB exit status + :rtype: int + ''' global defaults - if extra_args is None: - extra_args = {} - args = copy.copy(args) - args.__dict__ = dict(list(defaults.items()) + list(args.__dict__.items()) + list(extra_args.items())) + args = common.resolve_args(defaults, args, extra_args) after = shlex.split(args.after) before = shlex.split(args.before) if args.no_lxsymbols: @@ -82,7 +81,7 @@ def main(args, extra_args=None): return common.run_cmd(cmd, cmd_file=os.path.join(common.run_dir, 'rungdb.sh'), cwd=common.linux_variant_dir) if __name__ == '__main__': - parser = common.get_argparse(argparse_args={'description':'Connect with GDB to an emulator to debug Linux itself'}) + parser = common.get_argparse(argparse_args={'description': 'Connect with GDB to an emulator to debug Linux itself'}) parser.add_argument( '-A', '--after', default=defaults['after'], help='Pass extra arguments to GDB, to be appended after all other arguments' @@ -92,14 +91,14 @@ if __name__ == '__main__': help='Pass extra arguments to GDB to be prepended before any of the arguments passed by this script' ) parser.add_argument( - '-C', '--no-continue', default=False, action='store_true', + '-C', '--no-continue', default=defaults['no_continue'], action='store_true', help="Don't run continue after connecting" ) parser.add_argument( - '-k', '--kgdb', default=False, action='store_true' + '-k', '--kgdb', default=defaults['kgdb'], action='store_true' ) parser.add_argument( - '-X', '--no-lxsymbols', default=False, action='store_true' + '-X', '--no-lxsymbols', default=defaults['no_lxsymbols'], action='store_true' ) parser.add_argument( 'break_at', nargs='?', diff --git a/rungdb-user b/rungdb-user index d230fb9..a6a96b2 100755 --- a/rungdb-user +++ b/rungdb-user @@ -6,10 +6,10 @@ import subprocess import re import common -rungdb = imp.load_source('config', 'rungdb') +rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'rungdb')) parser = common.get_argparse(argparse_args={ - 'description':'''GDB step debug guest userland processes without gdbserver. + 'description': '''GDB step debug guest userland processes without gdbserver. More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-step-debug-userland-processes ''' diff --git a/runtc b/runtc index c03aa55..b7e3fd9 100755 --- a/runtc +++ b/runtc @@ -1,13 +1,12 @@ #!/usr/bin/env python3 -import os import subprocess import sys import common parser = common.get_argparse(argparse_args={ - 'description':'''Run a Buildroot ToolChain tool like readelf or objdump. + 'description': '''Run a Buildroot ToolChain tool like readelf or objdump. For example, to get some information about the arm vmlinux: diff --git a/trace-boot b/trace-boot index e3ae276..bc5fbbc 100755 --- a/trace-boot +++ b/trace-boot @@ -1,25 +1,46 @@ -#!/usr/bin/env bash -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common" -while getopts "${common_getopts_flags}" OPT; do - case "$OPT" in - ?) - common_getopts_case "$OPT" - ;; - esac -done -shift "$(($OPTIND - 1))" -common_setup -if "$common_gem5"; then - time ./run -a "$common_arch" -E 'm5 exit' -g -T 'Exec,-ExecSymbol,-ExecMicro' "$@" -else - time ./run -a "$common_arch" -e 'init=/poweroff.out' -T exec_tb "$@" - time ./qemu-trace2txt -a "$common_arch" - # Instruction count. - # We could put this on a separate script, but it just adds more arch boilerplate to a new script. - # So let's just leave it here for now since it did not add a significant processing time. - echo "instructions $(wc -l "${common_trace_txt_file}" | cut -d' ' -f1)" - entry_addr=$("${common_root_dir}/runtc" readelf -h "${common_build_dir}/linux-custom/vmlinux" | grep 'Entry point address' | sed -E 's/.*: *//') - echo "entry_address ${entry_addr}" - sed "/${entry_addr}/q" "${common_trace_txt_file}" >"${common_qemu_run_dir}/trace-boot.txt" - echo "instructions_firmware $(wc -l "${common_qemu_run_dir}/trace-boot.txt" | cut -d' ' -f1)" -fi +#!/usr/bin/env python3 + +import imp +import os +import subprocess +import re + +import common +run = imp.load_source('run', os.path.join(common.root_dir, 'run')) +qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(common.root_dir, 'qemu-trace2txt')) + +parser = common.get_argparse(argparse_args={ + 'description': '''Trace the PIC addresses executed on a Linux kernel boot. + +More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#tracing +''' +}) +parser.add_argument( + 'extra_emulator_args', nargs='*', + help='Extra options to append at the end of the emulator command line' +) +args = common.setup(parser) +extra_args = { + 'extra_emulator_args': args.extra_emulator_args, +} +if args.gem5: + extra_args.update({ + 'eval': 'm5 exit', + 'trace': 'Exec,-ExecSymbol,-ExecMicro', + }) + run.main(args, extra_args) +else: + extra_args.update({ + 'kernel_cli_extra': 'init=/poweroff.out', + 'trace': 'exec_tb', + }) + run.main(args, extra_args) + qemu_trace2txt.main() + ## Instruction count. + ## We could put this on a separate script, but it just adds more arch boilerplate to a new script. + ## So let's just leave it here for now since it did not add a significant processing time. + #echo "instructions $(wc -l "${common_trace_txt_file}" | cut -d' ' -f1)" + #entry_addr=$("${common_root_dir}/runtc" readelf -h "${common_build_dir}/linux-custom/vmlinux" | grep 'Entry point address' | sed -E 's/.*: *//') + #echo "entry_address ${entry_addr}" + #sed "/${entry_addr}/q" "${common_trace_txt_file}" >"${common_qemu_run_dir}/trace-boot.txt" + #echo "instructions_firmware $(wc -l "${common_qemu_run_dir}/trace-boot.txt" | cut -d' ' -f1)"