linux-kernel-module-cheat/run

#!/usr/bin/env python3

import os
import shlex
import signal
import subprocess
import sys

import common

# Argparse.
parser = common.get_argparse(argparse_args={'description':'Run Linux on an emulator'})
init_group = parser.add_mutually_exclusive_group()
kvm_group = parser.add_mutually_exclusive_group()
parser.add_argument(
    '-c', '--cpus', default=1, type=int,
    help='Number of guest CPUs to emulate. Default: %(default)s'
)
parser.add_argument(
    '-D', '--debug-vm', default=False, action='store_true',
    help='Run GDB on the emulator itself.'
)
kvm_group.add_argument(
    '-d', '--debug', default=False, action='store_true',
    help='Wait for GDB to connect before starting execution'
)
parser.add_argument(
    '-E', '--eval',
    help="""\
Replace the normal init with a minimal init that just evals with given
`CMDSTR` bash command string. Example: `-E 'insmod /hello.ko;'`
"""
)
parser.add_argument(
    '-e', '--kernel-cli-extra',
    help="""\
Pass an extra Linux kernel command line options, and place them before
the dash separator `-`. Only options that come before the `-`, i.e.
"standard" options, should be passed with this option.
Example: `./run -a arm -e 'init=/poweroff.out'`
"""
)
parser.add_argument(
    '-F', '--kernel-cli-extra-after-dash-base64',
    help="""\
Much like `-f`, but base64 encodes the string. Mnemonic:
`-F` is to `-f` what `-E` is to `-e`.)
"""
)
parser.add_argument(
    '-f', '--kernel-cli-extra-after-dash',
    help="""\
Pass an extra Linux kernel command line options, add a dash `-`
separator, and place the options after the dash. Intended for custom
options understood by our `init` scripts, most of which are prefixed
by `lkmc_`, e.g.: `./run -f 'lkmc_eval="wget google.com" lkmc_lala=y'`
Mnenomic: comes after `-e`.
"""
)
parser.add_argument(
    '-G', '--gem5-exe-args', default='',
    help="""\
Pass extra options to the gem5 executable.
Do not confuse with the arguments passed to config scripts,
like `fs.py`. Example: `./run -G '--debug-flags=Exec --debug' -g`.
"""
)
parser.add_argument(
    '--gem5-biglittle', default=False, action='store_true',
    help='Use fs_bigLITTLE.py instead of fs.py'
)
init_group.add_argument(
    '-I', '--initramfs', default=False, action='store_true',
    help='Use initramfs instead of a root filesystem'
)
init_group.add_argument(
    '-i', '--initrd', default=False, action='store_true',
    help='Use initrd instead of a root filesystem'
)
kvm_group.add_argument(
    '-K', '--kvm', default=False, action='store_true',
    help='Use KVM. Only works if guest arch == host arch'
)
parser.add_argument(
    '-k', '--kgdb', default=False, action='store_true'
)
parser.add_argument(
    '-l', '--gem5-restore-last-checkpoint', type=int,
    help="""\
Restore the nth most recently taken gem5 checkpoint according to directory
timestamps.
"""
)
parser.add_argument(
    '-m', '--memory', default='256M',
    help="""\
Set the memory size of the guest. E.g.: `-m 512M`. We try to keep the default
at the minimal ammount amount that boots all archs. Anything lower could lead
some arch to fail to boot.
Default: %(default)s
"""
)
parser.add_argument(
    '-P', '--prebuilt', default=False, action='store_true',
    help='Run the downloaded prebuilt images.'
)
group = parser.add_mutually_exclusive_group()
group.add_argument(
    '-R', '--qemu-replay', default=False, action='store_true',
    help='Replay a QEMU run record deterministically'
)
group.add_argument(
    '-r', '--qemu-record', default=False, action='store_true',
    help='Record a QEMU run record for later replay with `-R`'
)
parser.add_argument(
    '-T', '--trace',
    help="""\
Set trace events to be enabled. If not given, gem5 tracing is completely
disabled, while QEMU tracing is enabled but uses default traces that are very
rare and don't affect performance, because `./configure
--enable-trace-backends=simple` seems to enable some traces by default, e.g.
`pr_manager_run`, and I don't know how to get rid of them.
"""
)
parser.add_argument(
    '-U', '--tmux-args', default='',
    help='Pass extra parameters to the program running on the `-u` tmux split'
)
parser.add_argument(
    '-u', '--tmux', default=False, action='store_true',
    help="""\
Create a tmUx split the window. You must already be inside of a `tmux` session
to use this option:
* on the main window, run the emulator as usual
* on the split:
** if on QEMU and `-d` is given, GDB
** if on gem5, the gem5 terminal
"""
)
parser.add_argument(
    '-x', '--graphic', default=False, action='store_true',
    help='Run in graphic mode. Mnemonic: X11'
)
parser.add_argument(
    '-V', '--vnc', default=False, action='store_true',
    help="""\
Run QEMU with VNC instead of the default SDL. Connect to it with:
`vinagre localhost:5900`.
"""
)
parser.add_argument(
    'extra_emulator_args', nargs='*',
    help='Extra options to append at the end of the emulator command line'
)
args = common.setup(parser)

# Common qemu / gem5 logic.
# nokaslr:
# * https://unix.stackexchange.com/questions/397939/turning-off-kaslr-to-debug-linux-kernel-using-qemu-and-gdb
# * https://stackoverflow.com/questions/44612822/unable-to-debug-kernel-with-qemu-gdb/49840927#49840927
#   Turned on by default since v4.12
kernel_cli_extra = 'console_msg_format=syslog nokaslr norandmaps panic=-1 printk.devkmsg=on printk.time=y'
if args.kernel_cli_extra is not None:
    kernel_cli_extra += ' {}'.format(args.kernel_cli_extra)
env = os.environ.copy()
kernel_cli_extra_after_dash = ''
extra_emulator_args = args.extra_emulator_args.copy()
extra_qemu_args = []
if args.debug_vm:
    debug_vm = ['gdb', '-q', '-ex', 'start', '--args']
else:
    debug_vm = []
if args.debug:
    extra_qemu_args.append('-S')
if args.kernel_cli_extra_after_dash_base64 is not None:
    kernel_cli_extra_after_dash += ' lkmc_eval_base64="{}"'.format(common.base64_encode(args.kernel_cli_extra_after_dash_base64))
if args.kernel_cli_extra_after_dash is not None:
    kernel_cli_extra_after_dash += ' {}'.format(args.kernel_cli_extra_after_dash)
if args.kgdb:
    kernel_cli_extra += ' kgdbwait'
if args.vnc:
    vnc = ['-vnc', ':0']
else:
    vnc = []
if args.initrd or args.initramfs:
    ramfs = True
else:
    ramfs = False
if args.eval is not None:
    if ramfs:
        initarg = 'rdinit'
    else:
        initarg = 'init'
    kernel_cli_extra += ' {}=/eval_base64.sh'.format(initarg)
    kernel_cli_extra_after_dash += ' lkmc_eval="{}"'.format(common.base64_encode(args.eval))
if not args.graphic:
    if args.arch == 'x86_64':
        kernel_cli_extra += ' console=ttyS0'
    extra_qemu_args.append('-nographic')
if kernel_cli_extra_after_dash:
    kernel_cli_extra += " -{}".format(kernel_cli_extra_after_dash)

# A dummy value that is already turned on by default and does not produce large output,
# just to prevent QEMU from emitting a warning that '' is not valid.
trace_type = 'pr_manager_run'

if args.gem5:
    memory = '{}B'.format(args.memory)
    gem5_exe_args = shlex.split(args.gem5_exe_args)
    if args.trace is not None:
        gem5_exe_args.append('--debug-flags={}'.format(args.trace))
    env['M5_PATH'] = common.gem5_system_dir
    cmd = (
        debug_vm +
        [
            common.executable,
            '--debug-file=trace.txt',
        ] +
        gem5_exe_args +
        [
            '-d', common.m5out_dir
        ]
    )
    if args.gem5_biglittle:
        # TODO port
        # if args.gem5_restore_last_checkpoint is not None:
        #     cmd += [
        #         '--restore-from='${common.m5out_dir}/$(ls -crt "common.m5out_dir" | grep -E "common.gem5_cpt_pref" | tail -n "$gem5_restore_last_checkpoint" | head -n 1
        #     ]
        cmd += [
            os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'),
            '--big-cpus', '2',
            '--cpu-type', 'atomic',
            '--disk', common.ext2_file,
            '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'),
            '--kernel', common.vmlinux,
            '--little-cpus', '2'
        ]
    else:
        # TODO port
        #if [ -n "$gem5_restore_last_checkpoint" ]; then
        #    latest_cpt_basename="$(ls -crt "common.m5out_dir" | grep -E "common.gem5_cpt_pref" | tail -n "$gem5_restore_last_checkpoint" | head -n 1)"
        #    n="$(ls -1 "common.m5out_dir" | grep -E "common.gem5_cpt_pref" | sort -k 2 -n -t . | grep -n "$latest_cpt_basename" | cut -d : -f 1)"
        #    cmd += -r ${n} \\
        cmd += [
            os.path.join(common.gem5_src_dir, 'configs', 'example', 'fs.py'),
            '--disk-image', common.ext2_file,
            '--kernel', common.vmlinux,
            '--mem-size', memory,
            '--num-cpus', str(args.cpus),
            '--script', common.gem5_readfile_file,
        ]
        if args.arch == 'x86_64':
            if args.kvm:
                cmd += ['--cpu-type', 'X86KvmCPU']
            cmd += ['--command-line', 'earlyprintk=ttyS0 console=ttyS0 lpj=7999923 root=/dev/sda {}'.format(kernel_cli_extra)]
        elif args.arch == 'arm' or args.arch == 'aarch64':
            # TODO why is it mandatory to pass mem= here? Not true for QEMU.
            # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value?
            cmd += [
                '--command-line', 'earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli_extra),
                '--dtb-file', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)),
                '--machine-type', 'VExpress_GEM5_V1',
            ]
else:
    os.makedirs(common.run_dir, exist_ok=True)
    if args.debug_vm:
        serial_monitor = []
    else:
        serial_monitor = ['-serial', 'mon:stdio']
    if args.kvm:
        extra_emulator_args.append('-enable-kvm')
    if args.kgdb:
        extra_qemu_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.gdb_port)])
    if args.prebuilt:
        common.mkdir()
        qemu_executable = "qemu-system-{}".format(args.arch)
    else:
        qemu_executable = common.qemu_executable
    extra_emulator_args = extra_qemu_args + extra_emulator_args
    cmd = (
        debug_vm +
        [
            qemu_executable,
            '-device', 'rtl8139,netdev=net0',
            '-gdb', 'tcp::{}'.format(common.gdb_port),
            '-kernel', common.linux_image,
            '-m', args.memory,
            '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port),
            '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port),
            '-no-reboot',
            '-smp', str(args.cpus),
            '-trace', 'enable={},file={}'.format(trace_type, os.path.join(common.run_dir, 'trace.bin')),
            '-virtfs', 'local,path={},mount_tag=host_scratch,security_model=mapped,id=host_scratch'.format(common.p9_dir),
            '-virtfs', 'local,path={},mount_tag=host_out,security_model=mapped,id=host_out'.format(common.build_dir),
        ] +
        serial_monitor +
        vnc
    )
    if args.initrd:
        extra_emulator_args.extend(['-initrd', os.path.join(common.images_dir, 'rootfs.cpio')])
    rr = args.qemu_record or args.qemu_replay
    if ramfs:
        # TODO why is this needed, and why any string works.
        root = 'root=/dev/anything'
    else:
        if rr:
            driveif = 'none'
            rrid = ',id=img-direct'
            root = 'root=/dev/sda'
            snapshot = ''
        else:
            driveif = 'virtio'
            root = 'root=/dev/vda'
            rrid = ''
            snapshot = ',snapshot'
        extra_emulator_args.extend([
            '-drive',
            'file={},format=qcow2,if={}{}{}'.format(common.qcow2_file, driveif, snapshot, rrid)
        ])
        if rr:
            extra_emulator_args.extend([
            '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay',
            '-device', 'ide-hd,drive=img-blkreplay'
        ])
    if rr:
        extra_emulator_args.extend([
            '-object', 'filter-replay,id=replay,netdev=net0',
            '-icount', 'shift=7,rr={},rrfile={}'.format('record' if args.qemu_record else 'replay', common.qemu_rrfile),
        ])
        virtio_gpu_pci = []
    else:
        virtio_gpu_pci = ['-device', 'virtio-gpu-pci']
    if args.arch == 'x86_64':
        if args.kgdb:
            kernel_cli_extra += ' kgdboc=ttyS0,115200'
        cmd.extend([
            '-M', 'pc',
            '-append', '{} nopat {}'.format(root, kernel_cli_extra),
            '-device', 'edu',
        ])
    elif args.arch == 'arm' or args.arch == 'aarch64':
        if args.kgdb:
            kernel_cli_extra += ' kgdboc=ttyAMA0,115200'
        if args.arch == 'arm':
            cpu = 'cortex-a15'
        else:
            cpu = 'cortex-a57'
        # highmem=off needed since v3.0.0 due to:
        # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
        cmd = (
            cmd +
            [
                '-M', 'virt,highmem=off',
                '-append', '{} {}'.format(root, kernel_cli_extra),
                '-cpu', cpu,
            ] +
            virtio_gpu_pci
        )

if args.tmux:
    if args.gem5:
        subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
            'sleep 2;./gem5-shell -n {} {}' \
            .format(args.run_id, args.tmux_args)
        ])
    elif args.debug:
        # TODO find a nicer way to forward all those args automatically.
        # Part of me wants to: https://github.com/jonathanslenders/pymux
        # but it cannot be used as a library properly it seems, and it is
        # slower than tmux.
        subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
            "sleep 2;./rungdb -a '{}' -L '{}' -n '{}' {}" \
            .format(args.arch, args.linux_build_id, args.run_id, args.tmux_args)
        ])

cmd += extra_emulator_args
common.print_cmd(cmd)
# Otherwise Ctrl + C gives an ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
signal.signal(signal.SIGINT, signal.SIG_IGN)
subprocess.Popen(cmd, env=env).wait()
signal.signal(signal.SIGINT, signal.SIG_DFL)

#cmd="time \\
#${cmd}${extra_emulator_args}"
#if [ -z "$debug_vm" ]; then
#  cmd="${cmd}\
#|& tee >(ts -s %.s > ${common.termout_file})\
#"
#fi
#"${common.root_dir}/eeval" "$cmd" "${common.run_dir}/run.sh"
#cmd_out=$?
#if [ "$cmd_out" -ne 0 ]; then
#  exit "$cmd_out"
#fi

# TODO
## Check if guest panicked.
#if args.gem5:
#    # We have to do some parsing here because gem5 exits with status 0 even when panic happens.
#    # Grepping for '^panic: ' does not work because some errors don't show that message.
#    panic_msg = '--- BEGIN LIBC BACKTRACE ---$'
#else:
#    panic_msg = 'Kernel panic - not syncing'
#if grep -E -e "$panic_msg" -q "common.termout_file"; then
#    print('Simulation error detected by parsing logs. Exiting with status 1.', file=sys.stderr)
#    sys.exit(1)