common: convert scripts to python

.gitignore

@@ -11,3 +11,7 @@ gitignore*
 /images-*.zip
 /out
 /out.*
+
+# Python trash.
+*.pyc
+__pycache__

README.adoc

@@ -9553,22 +9553,15 @@ Otherwise, it becomes very difficult to keep everything working across path refa
                              Mnemonic: `fast`.
 |`-g` |                     |Enable gem5 build or force its rebuild.
 |`-h` |                     |Show this help message.
-|`-L` |`VARIANT`            |Linux kernel build variant.
 |`-I` |                     |Enable initramfs for the current build.
 |`-i` |                     |Enable initrd for the current build.
 |`-K` |`KERNEL_CONFIG_FILE` |Use `KERNEL_CONFIG_FILE` as the exact Linux kernel
                              configuration. Ignore the default kernel config fragments,
                              but still add options explicitly passed with `-C` and `-c`.
                              on top of it.
-|`-M` |`VARIANT`            |gem5 build variant.
 |`-p` |                     |Pass extra arguments to the `rootfs_post_build_script`.
-|`-Q` |`VARIANT``           |QEMU build variant.
 |`-S` |                     |Don't build QEMU with SDL support.
                              Graphics such as X11 won't work, only the terminal.
-|`-s` |                     |Add a custom suffix to the build.
-                             E.g., doing `./build -s mysuf` puts all the build output
-                             into `out/x86_64-mysuf`. This allows keep multiple builds around
-                             when you checkout between branches.
 |`-v` |                     |Do a verbose build.
 |===
 
@@ -9595,7 +9588,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
 [options="header"]
 |===
 |Name   |Argument name |Description
-|`-a`   |`ARCH`        |Run architecture `ARCH`.
 |`-c`   |`NCPUS`       |Emulate `NCPUS` guest CPUs.
 |`-D`   |              |Run GDB on the emulator itself.
 |`-d`   |              |Wait for GDB to connect before starting execution.
@@ -9618,7 +9610,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
 |`-G`   |              |Pass extra options to the gem5 executable.
                         Do not confuse with the arguments passed to config scripts,
                         like `fs.py`. Example: `./run -G '--debug-flags=Exec --debug' -g`
-|`-g`   |              |Use gem5 instead of QEMU.
 |`-h`   |              |Show this help message.
 |`-I`   |              |Run with initramfs.
 |`-i`   |              |Run with initrd.
@@ -9632,8 +9623,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
                         The default is the minimum amount that boots all archs without extra
                         options added. Anything lower will lead some arch to fail to boot.
                         Any
-|`-N`   |`VARIANT`     |gem5 source input variant.
-|`-n`   |              |Run ID.
 |`-R`   |              |Replay a QEMU run record deterministically.
 |`-r`   |              |Record a QEMU run record for later replay with `-R`.
 |`-P`   |              |Run the downloaded prebuilt images.

common

@@ -1,204 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-common_abspath() (
-  echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
-)
-
-# Benchmark a command.
-#
-# $1: command to benchmark
-# $2: where to append write results to. Default: /dev/null.
-#
-# Result format:
-#
-# cmd <command run>
-# time <time in seconds to finish>
-# exit_status <exit status>
-common_bench_cmd() (
-  cmd="$1"
-  results_file="${2:-/dev/null}"
-  printf 'cmd ' >> "$results_file"
-  env time --append -f 'time %e' --output="$results_file" "${common_root_dir}/eeval" -a "$cmd" "$results_file"
-  printf "exit_status $?\n" >> "$results_file"
-)
-
-
-# Handle options common across multiple scripts.
-common_getopts_case() {
-  case "$1" in
-    a)
-      common_arch="$OPTARG"
-      ;;
-    g)
-      common_gem5=true
-      ;;
-    L)
-      common_linux_variant="$OPTARG"
-      ;;
-    M)
-      common_gem5_variant="$OPTARG"
-      ;;
-    N)
-      common_gem5_worktree="$OPTARG"
-      ;;
-    n)
-      common_run_id="$OPTARG"
-      ;;
-    Q)
-      common_qemu_variant="$OPTARG"
-      ;;
-    s)
-      common_suffix="$OPTARG"
-      ;;
-    t)
-      common_gem5_build_type="$OPTARG"
-      ;;
-    ?)
-      exit 2
-      ;;
-  esac
-}
-common_getopts_flags='a:gL:M:N:n:Q:s:t:'
-
-# Setup several variables and do other initialization common to most scripts.
-# Typically done after getting inputs from the command line arguments.
-common_setup() {
-  case "$common_arch" in
-    a|arm)
-      common_arch=arm
-      common_gem5_arch=ARM
-      ;;
-    A|aarch64)
-      common_arch=aarch64
-      common_gem5_arch=ARM
-      ;;
-    m|mips64)
-      common_arch=mips64
-      ;;
-    x|x86_64)
-      common_arch=x86_64
-      common_gem5_arch=X86
-      ;;
-    *)
-      printf "unknown arch: ${common_arch}\n" 1>&2
-      exit 2
-      ;;
-  esac
-  common_buildroot_dir="${common_root_dir}/buildroot"
-  common_arch_dir="$common_arch"
-  if [ -n "$common_suffix" ]; then
-    common_arch_dir="${common_arch_dir}-${common_suffix}"
-  fi
-  common_out_arch_dir="${common_out_dir}/${common_arch_dir}"
-    common_buildroot_out_dir="${common_out_arch_dir}/buildroot"
-      common_build_dir="${common_buildroot_out_dir}/build"
-        common_linux_custom_dir="${common_build_dir}/linux-custom"
-          common_linux_variant_dir="${common_linux_custom_dir}.${common_linux_variant}"
-            common_vmlinux="${common_linux_variant_dir}/vmlinux"
-        common_qemu_custom_dir="${common_build_dir}/host-qemu-custom"
-          common_qemu_guest_variant_dir="${common_qemu_custom_dir}.${common_qemu_variant}"
-          common_qemu_variant_dir="${common_qemu_custom_dir}.${common_qemu_variant}"
-            common_qemu_exec="${common_qemu_variant_dir}/${common_arch}-softmmu/qemu-system-${common_arch}"
-        common_qemu_guest_custom_dir="${common_build_dir}/qemu-custom"
-      common_host_dir="${common_buildroot_out_dir}/host"
-      common_images_dir="${common_buildroot_out_dir}/images"
-        common_qcow2_file="${common_images_dir}/rootfs.ext2.qcow2"
-      common_staging_dir="${common_buildroot_out_dir}/staging"
-      common_target_dir="${common_buildroot_out_dir}/target"
-    common_gem5_run_dir="${common_out_arch_dir}/gem5/${common_run_id}"
-      common_m5out_dir="${common_gem5_run_dir}/m5out"
-        common_trace_txt_file="${common_m5out_dir}/trace.txt"
-      common_gem5_termout_file="${common_gem5_run_dir}/termout.txt"
-    common_qemu_run_dir="${common_out_arch_dir}/qemu/${common_run_id}"
-      common_qemu_termout_file="${common_qemu_run_dir}/termout.txt"
-      common_qemu_rrfile="${common_qemu_run_dir}/rrfile"
-  common_gem5_out_dir="${common_dir}/gem5/${common_gem5_variant}"
-    common_gem5_m5term="${common_gem5_out_dir}/m5term"
-    common_gem5_build_dir="${common_gem5_out_dir}/build"
-      common_gem5_exec="${common_gem5_build_dir}/${common_gem5_arch}/gem5.${common_gem5_build_type}"
-    common_gem5_system_dir="${common_gem5_out_dir}/system"
-  if [ -n "$common_gem5_worktree" ]; then
-    common_gem5_src_dir="${common_gem5_non_default_src_root_dir}/${common_gem5_worktree}"
-  else
-    common_gem5_src_dir="$common_gem5_default_src_dir"
-  fi
-  if "$common_gem5"; then
-    common_exec="$common_gem5_exec"
-    common_run_dir="$common_gem5_run_dir"
-    common_termout_file="$common_gem5_termout_file"
-  else
-    common_exec="$common_qemu_exec"
-    common_run_dir="$common_qemu_run_dir"
-    common_termout_file="$common_qemu_termout_file"
-  fi
-  case "$common_arch" in
-    arm)
-      common_linux_image=arch/arm/boot/zImage
-      ;;
-    aarch64)
-      common_linux_image=arch/arm64/boot/Image
-      ;;
-    mips64)
-      common_linux_image=vmlinux
-      ;;
-    x86_64)
-      common_linux_image=arch/x86/boot/bzImage
-      ;;
-  esac
-  common_linux_image="${common_linux_variant_dir}/${common_linux_image}"
-
-  # Ports.
-  common_run_id_number="$(echo "$common_run_id" | cut -d . -f 2)"
-  if "$common_gem5"; then
-    common_gem5_telnet_port="$((3456 + $common_run_id_number))"
-    common_gdb_port="$((7000 + $common_run_id_number))"
-  else
-    common_qemu_base_port="$((45454 + 10 * $common_run_id_number))"
-    common_qemu_monitor_port="$(($common_qemu_base_port + 0))"
-    common_qemu_hostfwd_generic_port="$(($common_qemu_base_port + 1))"
-    common_qemu_hostfwd_ssh_port="$(($common_qemu_base_port + 2))"
-    common_qemu_gdb_port="$(($common_qemu_base_port + 3))"
-    common_gdb_port="$common_qemu_gdb_port"
-  fi
-}
-
-common_mkdir() (
-  mkdir -p \
-    "$common_build_dir" \
-    "$common_gem5_out_dir" \
-    "$common_gem5_run_dir" \
-    "$common_qemu_run_dir" \
-    "$common_9p_dir" \
-  ;
-)
-
-# Default paths.
-common_root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-  common_data_dir="${common_root_dir}/data"
-    common_9p_dir="${common_data_dir}/9p"
-    common_gem5_non_default_src_root_dir="${common_data_dir}/gem5"
-    common_gem5_readfile_file="${common_data_dir}/readfile"
-  common_gem5_default_src_dir="${common_root_dir}/gem5/gem5"
-  common_out_dir="${common_root_dir}/out"
-    common_bench_boot="${common_out_dir}/bench-boot.txt"
-    common_dir="${common_out_dir}/common"
-
-# Other default variables.
-common_arch=x86_64
-common_gem5=false
-common_gem5_build_type=opt
-common_gem5_cpt_pref='^cpt\.'
-common_gem5_variant=default
-common_gem5_worktree=
-common_linux_variant=default
-common_qemu_variant=default
-common_run_id=0
-common_sha="$(git log -1 --format="%H")"
-common_suffix=
-
-f="${common_data_dir}/config"
-if [ -f "$f" ]; then
-  . "$f"
-fi

common.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+import os
+import sys
+
+this = sys.modules[__name__]
+
+# TODO
+## Benchmark a command.
+##
+## $1: command to benchmark
+## $2: where to append write results to. Default: /dev/null.
+##
+## Result format:
+##
+## cmd <command run>
+## time <time in seconds to finish>
+## exit_status <exit status>
+#bench_cmd() (
+#    cmd = "$1"
+#    results_file = "${2:-/dev/null}"
+#    printf 'cmd ' >> "$results_file"
+#    env time --append -f 'time %e' --output = "$results_file" "${root_dir}/eeval" -a "$cmd" "$results_file"
+#    printf "exit_status $?\n" >> "$results_file"
+#)
+
+def get_argparse(**kwargs):
+    """
+    Return an argument parser with common arguments set.
+    """
+    default_build_id='default'
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawTextHelpFormatter,
+        **kwargs
+    )
+    parser.add_argument(
+        '-a', '--arch', default='x86_64',
+        help='CPU architecture. Default: %(default)s'
+    )
+    parser.add_argument(
+        '-g', '--gem5', default=False, action='store_true',
+        help='Use gem5 instead of QEMU'
+    )
+    parser.add_argument(
+        '-L', '--linux-build-id', default=default_build_id,
+        help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s'
+    )
+    parser.add_argument(
+        '-M', '--gem5-build-id', default=default_build_id,
+        help='gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
+    )
+    parser.add_argument(
+        '-N', '--gem5-worktree',
+        help="""\
+gem5 worktree to use for build and Python scripts at runtime.
+If not given, just use the submodule source.
+"""
+    )
+    parser.add_argument(
+        '-n', '--run-id', default='0',
+        help="""\
+ID for run outputs such as gem5's m5out. Allows you to do multiple runs,
+and then inspect separate outputs later in different output directories.
+Default: %(default)s
+"""
+    )
+    parser.add_argument(
+        '--port-offset', type=int,
+        help="""\
+Increase the ports to be used such as for GDB by an offset to run multiple
+instances in parallel.
+Default: %(default)s
+"""
+    )
+    parser.add_argument(
+        '-Q', '--qemu-build-id', default=default_build_id,
+        help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
+    )
+    parser.add_argument(
+        '-s', '--suffix',
+        help="""\
+Add a custom suffix to the build. E.g., doing `./build -s mysuf` puts all
+the build output into `out/x86_64-mysuf`. This allows keep multiple builds
+around when you checkout between branches.
+"""
+    )
+    parser.add_argument(
+        '-t', '--gem5-build-type', default='opt',
+        help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
+    )
+    return parser
+
+def setup(parser):
+    """
+    Parse the command line arguments, and setup several variables based on them.
+    Typically done after getting inputs from the command line arguments.
+    """
+    global this
+    args = parser.parse_args()
+    if args.arch == 'a' or args.arch == 'arm':
+        args.arch = 'arm'
+        gem5_arch = 'ARM'
+    elif args.arch == 'A' or args.arch == 'aarch64':
+        args.arch = 'aarch64'
+        gem5_arch = 'ARM'
+    elif args.arch == 'x' or args.arch == 'x86_64':
+        args.arch = 'x86_64'
+        gem5_arch = 'X86'
+    else:
+        print('unknown arch: {}'.format(args.arch), file=sys.stderr)
+        sys.exit(2)
+    this.buildroot_dir = os.path.join(root_dir, 'buildroot')
+    this.arch_dir = args.arch
+    if args.suffix is not None:
+        this.arch_dir = '{}-{}'.format(arch_dir, args.suffix)
+    global out_arch_dir
+    this.out_arch_dir = os.path.join(this.out_dir, this.arch_dir)
+    this.buildroot_out_dir = os.path.join(this.out_arch_dir, 'buildroot')
+    this.build_dir = os.path.join(this.buildroot_out_dir, 'build')
+    this.linux_custom_dir = os.path.join(this.build_dir, 'linux-custom')
+    this.linux_variant_dir = '{}.{}'.format(this.linux_custom_dir, args.linux_build_id)
+    this.vmlinux = os.path.join(this.linux_variant_dir, "vmlinux")
+    this.qemu_custom_dir = os.path.join(this.build_dir, 'host-qemu-custom')
+    this.qemu_guest_variant_dir = os.path.join(this.qemu_custom_dir, args.qemu_build_id)
+    this.qemu_variant_dir = '{}.{}'.format(this.qemu_custom_dir, args.qemu_build_id)
+    this.qemu_executable = os.path.join(this.qemu_variant_dir, '{}-softmmu'.format(args.arch), 'qemu-system-{}'.format(args.arch))
+    this.qemu_guest_custom_dir = os.path.join(this.build_dir, 'qemu-custom')
+    this.host_dir = os.path.join(this.buildroot_out_dir, 'host')
+    this.images_dir = os.path.join(this.buildroot_out_dir, 'images')
+    this.qcow2_file = os.path.join(this.images_dir, 'rootfs.ext2.qcow2')
+    this.staging_dir = os.path.join(this.buildroot_out_dir, 'staging')
+    this.target_dir = os.path.join(this.buildroot_out_dir, 'target')
+    this.gem5_run_dir = os.path.join(this.out_arch_dir, 'gem5', str(args.run_id))
+    this.m5out_dir = os.path.join(this.gem5_run_dir, 'm5out')
+    this.trace_txt_file = os.path.join(this.m5out_dir, 'trace.txt')
+    this.gem5_termout_file = os.path.join(this.gem5_run_dir, 'termout.txt')
+    this.qemu_run_dir = os.path.join(this.out_arch_dir, 'qemu', str(args.run_id))
+    this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt')
+    this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile')
+    this.gem5_out_dir = os.path.join(this.common_dir, 'gem5', args.gem5_build_id)
+    this.gem5_m5term = os.path.join(this.gem5_out_dir, 'm5term')
+    this.gem5_build_dir = os.path.join(this.gem5_out_dir, 'build')
+    this.gem5_executable = os.path.join(this.gem5_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type))
+    this.gem5_system_dir = os.path.join(this.gem5_out_dir, 'system')
+    if args.gem5_worktree is not None:
+        this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree)
+    else:
+        this.gem5_src_dir = this.gem5_default_src_dir
+    if args.gem5:
+        this.executable = this.gem5_executable
+        this.run_dir = this.gem5_run_dir
+        this.termout_file = this.gem5_termout_file
+    else:
+        this.executable = this.qemu_executable
+        this.run_dir = this.qemu_run_dir
+        this.termout_file = this.qemu_termout_file
+    if args.arch == 'arm':
+        this.linux_image = os.path.join('arch', 'arm', 'boot', 'zImage')
+    elif args.arch == 'aarch64':
+        this.linux_image = os.path.join('arch', 'arm64', 'boot', 'Image')
+    elif args.arch == 'x86_64':
+        this.linux_image = os.path.join('arch', 'x86', 'boot', 'bzImage')
+    this.linux_image = os.path.join(this.linux_variant_dir, linux_image)
+
+    # Ports.
+    if args.port_offset is None:
+        args.port_offset = int(args.run_id)
+    if args.gem5:
+        this.gem5_telnet_port = 3456 + args.port_offset
+        this.gdb_port = 7000 + args.port_offset
+    else:
+        this.qemu_base_port = 45454 + 10 * args.port_offset
+        this.qemu_monitor_port = this.qemu_base_port + 0
+        this.qemu_hostfwd_generic_port = this.qemu_base_port + 1
+        this.qemu_hostfwd_ssh_port = this.qemu_base_port + 2
+        this.qemu_gdb_port = this.qemu_base_port + 3
+        this.gdb_port = this.qemu_gdb_port
+    return args
+
+def mkdir():
+    global this
+    os.makedirs(this.build_dir, exist_ok=True)
+    os.mkdirs(this.gem5_out_dir, exit_ok=True)
+    os.mkdirs(this.gem5_run_dir, exit_ok=True)
+    os.mkdirs(this.qemu_run_dir, exit_ok=True)
+    os.mkdirs(this.p9_dir, exit_ok=True)
+
+# Default paths.
+root_dir = os.path.dirname(os.path.abspath(__file__))
+data_dir = os.path.join(root_dir, 'data')
+p9_dir = os.path.join(data_dir, '9p')
+gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
+gem5_readfile_file = os.path.join(data_dir, 'readfile')
+gem5_default_src_dir = os.path.join(root_dir, 'gem5', 'gem5')
+out_dir = os.path.join(root_dir, 'out')
+bench_boot = os.path.join(out_dir, 'bench-boot.txt')
+common_dir = os.path.join(out_dir, 'common')
+
+# Other default variables.
+gem5_cpt_pref = '^cpt\.'
+sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
+
+# Config file. TODO move to decent python setup.
+config_file = os.path.join(data_dir, 'config')
+if os.path.exists(config_file):
+    exec(open(config_file).read())
+    with open(config_file) as f:
+        exec(f.read())

getvar

@@ -1,16 +1,8 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
+import common
-while getopts "h${common_getopts_flags}" OPT; do
+parser = common.get_argparse(
-  case "$OPT" in
+    description='https://github.com/cirosantilli/linux-kernel-module-cheat#getvar'
-    h)
+)
-      echo "https://github.com/cirosantilli/linux-kernel-module-cheat#getvar" 2>&1
+parser.add_argument('variable')
-      exit
+args = common.setup(parser)
-      ;;
+print(getattr(common, args.variable))
-    ?)
-      common_getopts_case "$OPT"
-      ;;
-  esac
-done
-shift "$(($OPTIND - 1))"
-common_setup
-eval "echo \$common_${1}"

run

@@ -176,7 +176,7 @@ if "$common_gem5"; then
   gem5_common="\
 M5_PATH='${common_gem5_system_dir}' \\
 ${debug_vm}\
-'${common_exec}' \\
+'${common_executable}' \\
 --debug-file=trace.txt \\
 ${gem5opts}\
 -d '${common_m5out_dir}' \\
@@ -248,14 +248,14 @@ else
   fi
   if "$prebuilt"; then
     common_mkdir
-    qemu_exec="qemu-system-${common_arch}"
+    qemu_executable="qemu-system-${common_arch}"
   else
-    qemu_exec="${common_qemu_exec}"
+    qemu_executable="${common_qemu_exec}"
   fi
   extra_flags="${extra_flags_qemu}${extra_flags}"
   qemu_common="\
 ${debug_vm}\
-${qemu_exec} \\
+${qemu_executable} \\
 -device rtl8139,netdev=net0 \\
 -gdb 'tcp::${common_gdb_port}' \\
 -kernel '${common_linux_image}' \\