From 9b30ac217ff519bc822626a3175d25be72517846 Mon Sep 17 00:00:00 2001
From: Ciro Santilli <ciro.santilli@gmail.com>
Date: Thu, 23 Aug 2018 09:25:21 +0100
Subject: [PATCH] common: convert scripts to python

---
 .gitignore  |   4 +
 README.adoc |  11 ---
 common      | 204 --------------------------------------------------
 common.py   | 210 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 getvar      |  24 ++----
 run         |   8 +-
 6 files changed, 226 insertions(+), 235 deletions(-)
 delete mode 100644 common
 create mode 100644 common.py

diff --git a/.gitignore b/.gitignore
index 3e05d6e..ea666c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,7 @@ gitignore*
 /images-*.zip
 /out
 /out.*
+
+# Python trash.
+*.pyc
+__pycache__
diff --git a/README.adoc b/README.adoc
index 52db887..df021e0 100644
--- a/README.adoc
+++ b/README.adoc
@@ -9553,22 +9553,15 @@ Otherwise, it becomes very difficult to keep everything working across path refa
                              Mnemonic: `fast`.
 |`-g` |                     |Enable gem5 build or force its rebuild.
 |`-h` |                     |Show this help message.
-|`-L` |`VARIANT`            |Linux kernel build variant.
 |`-I` |                     |Enable initramfs for the current build.
 |`-i` |                     |Enable initrd for the current build.
 |`-K` |`KERNEL_CONFIG_FILE` |Use `KERNEL_CONFIG_FILE` as the exact Linux kernel
                              configuration. Ignore the default kernel config fragments,
                              but still add options explicitly passed with `-C` and `-c`.
                              on top of it.
-|`-M` |`VARIANT`            |gem5 build variant.
 |`-p` |                     |Pass extra arguments to the `rootfs_post_build_script`.
-|`-Q` |`VARIANT``           |QEMU build variant.
 |`-S` |                     |Don't build QEMU with SDL support.
                              Graphics such as X11 won't work, only the terminal.
-|`-s` |                     |Add a custom suffix to the build.
-                             E.g., doing `./build -s mysuf` puts all the build output
-                             into `out/x86_64-mysuf`. This allows keep multiple builds around
-                             when you checkout between branches.
 |`-v` |                     |Do a verbose build.
 |===
 
@@ -9595,7 +9588,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
 [options="header"]
 |===
 |Name   |Argument name |Description
-|`-a`   |`ARCH`        |Run architecture `ARCH`.
 |`-c`   |`NCPUS`       |Emulate `NCPUS` guest CPUs.
 |`-D`   |              |Run GDB on the emulator itself.
 |`-d`   |              |Wait for GDB to connect before starting execution.
@@ -9618,7 +9610,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
 |`-G`   |              |Pass extra options to the gem5 executable.
                         Do not confuse with the arguments passed to config scripts,
                         like `fs.py`. Example: `./run -G '--debug-flags=Exec --debug' -g`
-|`-g`   |              |Use gem5 instead of QEMU.
 |`-h`   |              |Show this help message.
 |`-I`   |              |Run with initramfs.
 |`-i`   |              |Run with initrd.
@@ -9632,8 +9623,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
                         The default is the minimum amount that boots all archs without extra
                         options added. Anything lower will lead some arch to fail to boot.
                         Any
-|`-N`   |`VARIANT`     |gem5 source input variant.
-|`-n`   |              |Run ID.
 |`-R`   |              |Replay a QEMU run record deterministically.
 |`-r`   |              |Record a QEMU run record for later replay with `-R`.
 |`-P`   |              |Run the downloaded prebuilt images.
diff --git a/common b/common
deleted file mode 100644
index b9dcd05..0000000
--- a/common
+++ /dev/null
@@ -1,204 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-common_abspath() (
-  echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
-)
-
-# Benchmark a command.
-#
-# $1: command to benchmark
-# $2: where to append write results to. Default: /dev/null.
-#
-# Result format:
-#
-# cmd <command run>
-# time <time in seconds to finish>
-# exit_status <exit status>
-common_bench_cmd() (
-  cmd="$1"
-  results_file="${2:-/dev/null}"
-  printf 'cmd ' >> "$results_file"
-  env time --append -f 'time %e' --output="$results_file" "${common_root_dir}/eeval" -a "$cmd" "$results_file"
-  printf "exit_status $?\n" >> "$results_file"
-)
-
-
-# Handle options common across multiple scripts.
-common_getopts_case() {
-  case "$1" in
-    a)
-      common_arch="$OPTARG"
-      ;;
-    g)
-      common_gem5=true
-      ;;
-    L)
-      common_linux_variant="$OPTARG"
-      ;;
-    M)
-      common_gem5_variant="$OPTARG"
-      ;;
-    N)
-      common_gem5_worktree="$OPTARG"
-      ;;
-    n)
-      common_run_id="$OPTARG"
-      ;;
-    Q)
-      common_qemu_variant="$OPTARG"
-      ;;
-    s)
-      common_suffix="$OPTARG"
-      ;;
-    t)
-      common_gem5_build_type="$OPTARG"
-      ;;
-    ?)
-      exit 2
-      ;;
-  esac
-}
-common_getopts_flags='a:gL:M:N:n:Q:s:t:'
-
-# Setup several variables and do other initialization common to most scripts.
-# Typically done after getting inputs from the command line arguments.
-common_setup() {
-  case "$common_arch" in
-    a|arm)
-      common_arch=arm
-      common_gem5_arch=ARM
-      ;;
-    A|aarch64)
-      common_arch=aarch64
-      common_gem5_arch=ARM
-      ;;
-    m|mips64)
-      common_arch=mips64
-      ;;
-    x|x86_64)
-      common_arch=x86_64
-      common_gem5_arch=X86
-      ;;
-    *)
-      printf "unknown arch: ${common_arch}\n" 1>&2
-      exit 2
-      ;;
-  esac
-  common_buildroot_dir="${common_root_dir}/buildroot"
-  common_arch_dir="$common_arch"
-  if [ -n "$common_suffix" ]; then
-    common_arch_dir="${common_arch_dir}-${common_suffix}"
-  fi
-  common_out_arch_dir="${common_out_dir}/${common_arch_dir}"
-    common_buildroot_out_dir="${common_out_arch_dir}/buildroot"
-      common_build_dir="${common_buildroot_out_dir}/build"
-        common_linux_custom_dir="${common_build_dir}/linux-custom"
-          common_linux_variant_dir="${common_linux_custom_dir}.${common_linux_variant}"
-            common_vmlinux="${common_linux_variant_dir}/vmlinux"
-        common_qemu_custom_dir="${common_build_dir}/host-qemu-custom"
-          common_qemu_guest_variant_dir="${common_qemu_custom_dir}.${common_qemu_variant}"
-          common_qemu_variant_dir="${common_qemu_custom_dir}.${common_qemu_variant}"
-            common_qemu_exec="${common_qemu_variant_dir}/${common_arch}-softmmu/qemu-system-${common_arch}"
-        common_qemu_guest_custom_dir="${common_build_dir}/qemu-custom"
-      common_host_dir="${common_buildroot_out_dir}/host"
-      common_images_dir="${common_buildroot_out_dir}/images"
-        common_qcow2_file="${common_images_dir}/rootfs.ext2.qcow2"
-      common_staging_dir="${common_buildroot_out_dir}/staging"
-      common_target_dir="${common_buildroot_out_dir}/target"
-    common_gem5_run_dir="${common_out_arch_dir}/gem5/${common_run_id}"
-      common_m5out_dir="${common_gem5_run_dir}/m5out"
-        common_trace_txt_file="${common_m5out_dir}/trace.txt"
-      common_gem5_termout_file="${common_gem5_run_dir}/termout.txt"
-    common_qemu_run_dir="${common_out_arch_dir}/qemu/${common_run_id}"
-      common_qemu_termout_file="${common_qemu_run_dir}/termout.txt"
-      common_qemu_rrfile="${common_qemu_run_dir}/rrfile"
-  common_gem5_out_dir="${common_dir}/gem5/${common_gem5_variant}"
-    common_gem5_m5term="${common_gem5_out_dir}/m5term"
-    common_gem5_build_dir="${common_gem5_out_dir}/build"
-      common_gem5_exec="${common_gem5_build_dir}/${common_gem5_arch}/gem5.${common_gem5_build_type}"
-    common_gem5_system_dir="${common_gem5_out_dir}/system"
-  if [ -n "$common_gem5_worktree" ]; then
-    common_gem5_src_dir="${common_gem5_non_default_src_root_dir}/${common_gem5_worktree}"
-  else
-    common_gem5_src_dir="$common_gem5_default_src_dir"
-  fi
-  if "$common_gem5"; then
-    common_exec="$common_gem5_exec"
-    common_run_dir="$common_gem5_run_dir"
-    common_termout_file="$common_gem5_termout_file"
-  else
-    common_exec="$common_qemu_exec"
-    common_run_dir="$common_qemu_run_dir"
-    common_termout_file="$common_qemu_termout_file"
-  fi
-  case "$common_arch" in
-    arm)
-      common_linux_image=arch/arm/boot/zImage
-      ;;
-    aarch64)
-      common_linux_image=arch/arm64/boot/Image
-      ;;
-    mips64)
-      common_linux_image=vmlinux
-      ;;
-    x86_64)
-      common_linux_image=arch/x86/boot/bzImage
-      ;;
-  esac
-  common_linux_image="${common_linux_variant_dir}/${common_linux_image}"
-
-  # Ports.
-  common_run_id_number="$(echo "$common_run_id" | cut -d . -f 2)"
-  if "$common_gem5"; then
-    common_gem5_telnet_port="$((3456 + $common_run_id_number))"
-    common_gdb_port="$((7000 + $common_run_id_number))"
-  else
-    common_qemu_base_port="$((45454 + 10 * $common_run_id_number))"
-    common_qemu_monitor_port="$(($common_qemu_base_port + 0))"
-    common_qemu_hostfwd_generic_port="$(($common_qemu_base_port + 1))"
-    common_qemu_hostfwd_ssh_port="$(($common_qemu_base_port + 2))"
-    common_qemu_gdb_port="$(($common_qemu_base_port + 3))"
-    common_gdb_port="$common_qemu_gdb_port"
-  fi
-}
-
-common_mkdir() (
-  mkdir -p \
-    "$common_build_dir" \
-    "$common_gem5_out_dir" \
-    "$common_gem5_run_dir" \
-    "$common_qemu_run_dir" \
-    "$common_9p_dir" \
-  ;
-)
-
-# Default paths.
-common_root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-  common_data_dir="${common_root_dir}/data"
-    common_9p_dir="${common_data_dir}/9p"
-    common_gem5_non_default_src_root_dir="${common_data_dir}/gem5"
-    common_gem5_readfile_file="${common_data_dir}/readfile"
-  common_gem5_default_src_dir="${common_root_dir}/gem5/gem5"
-  common_out_dir="${common_root_dir}/out"
-    common_bench_boot="${common_out_dir}/bench-boot.txt"
-    common_dir="${common_out_dir}/common"
-
-# Other default variables.
-common_arch=x86_64
-common_gem5=false
-common_gem5_build_type=opt
-common_gem5_cpt_pref='^cpt\.'
-common_gem5_variant=default
-common_gem5_worktree=
-common_linux_variant=default
-common_qemu_variant=default
-common_run_id=0
-common_sha="$(git log -1 --format="%H")"
-common_suffix=
-
-f="${common_data_dir}/config"
-if [ -f "$f" ]; then
-  . "$f"
-fi
diff --git a/common.py b/common.py
new file mode 100644
index 0000000..72035b1
--- /dev/null
+++ b/common.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+import os
+import sys
+
+this = sys.modules[__name__]
+
+# TODO
+## Benchmark a command.
+##
+## $1: command to benchmark
+## $2: where to append write results to. Default: /dev/null.
+##
+## Result format:
+##
+## cmd <command run>
+## time <time in seconds to finish>
+## exit_status <exit status>
+#bench_cmd() (
+#    cmd = "$1"
+#    results_file = "${2:-/dev/null}"
+#    printf 'cmd ' >> "$results_file"
+#    env time --append -f 'time %e' --output = "$results_file" "${root_dir}/eeval" -a "$cmd" "$results_file"
+#    printf "exit_status $?\n" >> "$results_file"
+#)
+
+def get_argparse(**kwargs):
+    """
+    Return an argument parser with common arguments set.
+    """
+    default_build_id='default'
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawTextHelpFormatter,
+        **kwargs
+    )
+    parser.add_argument(
+        '-a', '--arch', default='x86_64',
+        help='CPU architecture. Default: %(default)s'
+    )
+    parser.add_argument(
+        '-g', '--gem5', default=False, action='store_true',
+        help='Use gem5 instead of QEMU'
+    )
+    parser.add_argument(
+        '-L', '--linux-build-id', default=default_build_id,
+        help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s'
+    )
+    parser.add_argument(
+        '-M', '--gem5-build-id', default=default_build_id,
+        help='gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
+    )
+    parser.add_argument(
+        '-N', '--gem5-worktree',
+        help="""\
+gem5 worktree to use for build and Python scripts at runtime.
+If not given, just use the submodule source.
+"""
+    )
+    parser.add_argument(
+        '-n', '--run-id', default='0',
+        help="""\
+ID for run outputs such as gem5's m5out. Allows you to do multiple runs,
+and then inspect separate outputs later in different output directories.
+Default: %(default)s
+"""
+    )
+    parser.add_argument(
+        '--port-offset', type=int,
+        help="""\
+Increase the ports to be used such as for GDB by an offset to run multiple
+instances in parallel.
+Default: %(default)s
+"""
+    )
+    parser.add_argument(
+        '-Q', '--qemu-build-id', default=default_build_id,
+        help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
+    )
+    parser.add_argument(
+        '-s', '--suffix',
+        help="""\
+Add a custom suffix to the build. E.g., doing `./build -s mysuf` puts all
+the build output into `out/x86_64-mysuf`. This allows keep multiple builds
+around when you checkout between branches.
+"""
+    )
+    parser.add_argument(
+        '-t', '--gem5-build-type', default='opt',
+        help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
+    )
+    return parser
+
+def setup(parser):
+    """
+    Parse the command line arguments, and setup several variables based on them.
+    Typically done after getting inputs from the command line arguments.
+    """
+    global this
+    args = parser.parse_args()
+    if args.arch == 'a' or args.arch == 'arm':
+        args.arch = 'arm'
+        gem5_arch = 'ARM'
+    elif args.arch == 'A' or args.arch == 'aarch64':
+        args.arch = 'aarch64'
+        gem5_arch = 'ARM'
+    elif args.arch == 'x' or args.arch == 'x86_64':
+        args.arch = 'x86_64'
+        gem5_arch = 'X86'
+    else:
+        print('unknown arch: {}'.format(args.arch), file=sys.stderr)
+        sys.exit(2)
+    this.buildroot_dir = os.path.join(root_dir, 'buildroot')
+    this.arch_dir = args.arch
+    if args.suffix is not None:
+        this.arch_dir = '{}-{}'.format(arch_dir, args.suffix)
+    global out_arch_dir
+    this.out_arch_dir = os.path.join(this.out_dir, this.arch_dir)
+    this.buildroot_out_dir = os.path.join(this.out_arch_dir, 'buildroot')
+    this.build_dir = os.path.join(this.buildroot_out_dir, 'build')
+    this.linux_custom_dir = os.path.join(this.build_dir, 'linux-custom')
+    this.linux_variant_dir = '{}.{}'.format(this.linux_custom_dir, args.linux_build_id)
+    this.vmlinux = os.path.join(this.linux_variant_dir, "vmlinux")
+    this.qemu_custom_dir = os.path.join(this.build_dir, 'host-qemu-custom')
+    this.qemu_guest_variant_dir = os.path.join(this.qemu_custom_dir, args.qemu_build_id)
+    this.qemu_variant_dir = '{}.{}'.format(this.qemu_custom_dir, args.qemu_build_id)
+    this.qemu_executable = os.path.join(this.qemu_variant_dir, '{}-softmmu'.format(args.arch), 'qemu-system-{}'.format(args.arch))
+    this.qemu_guest_custom_dir = os.path.join(this.build_dir, 'qemu-custom')
+    this.host_dir = os.path.join(this.buildroot_out_dir, 'host')
+    this.images_dir = os.path.join(this.buildroot_out_dir, 'images')
+    this.qcow2_file = os.path.join(this.images_dir, 'rootfs.ext2.qcow2')
+    this.staging_dir = os.path.join(this.buildroot_out_dir, 'staging')
+    this.target_dir = os.path.join(this.buildroot_out_dir, 'target')
+    this.gem5_run_dir = os.path.join(this.out_arch_dir, 'gem5', str(args.run_id))
+    this.m5out_dir = os.path.join(this.gem5_run_dir, 'm5out')
+    this.trace_txt_file = os.path.join(this.m5out_dir, 'trace.txt')
+    this.gem5_termout_file = os.path.join(this.gem5_run_dir, 'termout.txt')
+    this.qemu_run_dir = os.path.join(this.out_arch_dir, 'qemu', str(args.run_id))
+    this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt')
+    this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile')
+    this.gem5_out_dir = os.path.join(this.common_dir, 'gem5', args.gem5_build_id)
+    this.gem5_m5term = os.path.join(this.gem5_out_dir, 'm5term')
+    this.gem5_build_dir = os.path.join(this.gem5_out_dir, 'build')
+    this.gem5_executable = os.path.join(this.gem5_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type))
+    this.gem5_system_dir = os.path.join(this.gem5_out_dir, 'system')
+    if args.gem5_worktree is not None:
+        this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree)
+    else:
+        this.gem5_src_dir = this.gem5_default_src_dir
+    if args.gem5:
+        this.executable = this.gem5_executable
+        this.run_dir = this.gem5_run_dir
+        this.termout_file = this.gem5_termout_file
+    else:
+        this.executable = this.qemu_executable
+        this.run_dir = this.qemu_run_dir
+        this.termout_file = this.qemu_termout_file
+    if args.arch == 'arm':
+        this.linux_image = os.path.join('arch', 'arm', 'boot', 'zImage')
+    elif args.arch == 'aarch64':
+        this.linux_image = os.path.join('arch', 'arm64', 'boot', 'Image')
+    elif args.arch == 'x86_64':
+        this.linux_image = os.path.join('arch', 'x86', 'boot', 'bzImage')
+    this.linux_image = os.path.join(this.linux_variant_dir, linux_image)
+
+    # Ports.
+    if args.port_offset is None:
+        args.port_offset = int(args.run_id)
+    if args.gem5:
+        this.gem5_telnet_port = 3456 + args.port_offset
+        this.gdb_port = 7000 + args.port_offset
+    else:
+        this.qemu_base_port = 45454 + 10 * args.port_offset
+        this.qemu_monitor_port = this.qemu_base_port + 0
+        this.qemu_hostfwd_generic_port = this.qemu_base_port + 1
+        this.qemu_hostfwd_ssh_port = this.qemu_base_port + 2
+        this.qemu_gdb_port = this.qemu_base_port + 3
+        this.gdb_port = this.qemu_gdb_port
+    return args
+
+def mkdir():
+    global this
+    os.makedirs(this.build_dir, exist_ok=True)
+    os.mkdirs(this.gem5_out_dir, exit_ok=True)
+    os.mkdirs(this.gem5_run_dir, exit_ok=True)
+    os.mkdirs(this.qemu_run_dir, exit_ok=True)
+    os.mkdirs(this.p9_dir, exit_ok=True)
+
+# Default paths.
+root_dir = os.path.dirname(os.path.abspath(__file__))
+data_dir = os.path.join(root_dir, 'data')
+p9_dir = os.path.join(data_dir, '9p')
+gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
+gem5_readfile_file = os.path.join(data_dir, 'readfile')
+gem5_default_src_dir = os.path.join(root_dir, 'gem5', 'gem5')
+out_dir = os.path.join(root_dir, 'out')
+bench_boot = os.path.join(out_dir, 'bench-boot.txt')
+common_dir = os.path.join(out_dir, 'common')
+
+# Other default variables.
+gem5_cpt_pref = '^cpt\.'
+sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
+
+# Config file. TODO move to decent python setup.
+config_file = os.path.join(data_dir, 'config')
+if os.path.exists(config_file):
+    exec(open(config_file).read())
+    with open(config_file) as f:
+        exec(f.read())
diff --git a/getvar b/getvar
index cad9bb5..2f19d24 100755
--- a/getvar
+++ b/getvar
@@ -1,16 +1,8 @@
-#!/usr/bin/env bash
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
-while getopts "h${common_getopts_flags}" OPT; do
-  case "$OPT" in
-    h)
-      echo "https://github.com/cirosantilli/linux-kernel-module-cheat#getvar" 2>&1
-      exit
-      ;;
-    ?)
-      common_getopts_case "$OPT"
-      ;;
-  esac
-done
-shift "$(($OPTIND - 1))"
-common_setup
-eval "echo \$common_${1}"
+#!/usr/bin/env python3
+import common
+parser = common.get_argparse(
+    description='https://github.com/cirosantilli/linux-kernel-module-cheat#getvar'
+)
+parser.add_argument('variable')
+args = common.setup(parser)
+print(getattr(common, args.variable))
diff --git a/run b/run
index 6bcbd3c..956c703 100755
--- a/run
+++ b/run
@@ -176,7 +176,7 @@ if "$common_gem5"; then
   gem5_common="\
 M5_PATH='${common_gem5_system_dir}' \\
 ${debug_vm}\
-'${common_exec}' \\
+'${common_executable}' \\
 --debug-file=trace.txt \\
 ${gem5opts}\
 -d '${common_m5out_dir}' \\
@@ -248,14 +248,14 @@ else
   fi
   if "$prebuilt"; then
     common_mkdir
-    qemu_exec="qemu-system-${common_arch}"
+    qemu_executable="qemu-system-${common_arch}"
   else
-    qemu_exec="${common_qemu_exec}"
+    qemu_executable="${common_qemu_exec}"
   fi
   extra_flags="${extra_flags_qemu}${extra_flags}"
   qemu_common="\
 ${debug_vm}\
-${qemu_exec} \\
+${qemu_executable} \\
 -device rtl8139,netdev=net0 \\
 -gdb 'tcp::${common_gdb_port}' \\
 -kernel '${common_linux_image}' \\